diff --git a/api4/api.go b/api4/api.go index 7517d814a6..0a01d7e03e 100644 --- a/api4/api.go +++ b/api4/api.go @@ -125,6 +125,8 @@ type Routes struct { Cloud *mux.Router // 'api/v4/cloud' Imports *mux.Router // 'api/v4/imports' + Exports *mux.Router // 'api/v4/exports' + Export *mux.Router // 'api/v4/exports/{export_name:.+\\.zip}' } type API struct { @@ -238,6 +240,8 @@ func Init(configservice configservice.ConfigService, globalOptionsFunc app.AppOp api.BaseRoutes.Cloud = api.BaseRoutes.ApiRoot.PathPrefix("/cloud").Subrouter() api.BaseRoutes.Imports = api.BaseRoutes.ApiRoot.PathPrefix("/imports").Subrouter() + api.BaseRoutes.Exports = api.BaseRoutes.ApiRoot.PathPrefix("/exports").Subrouter() + api.BaseRoutes.Export = api.BaseRoutes.Exports.PathPrefix("/{export_name:.+\\.zip}").Subrouter() api.InitUser() api.InitBot() @@ -276,6 +280,7 @@ func Init(configservice configservice.ConfigService, globalOptionsFunc app.AppOp api.InitAction() api.InitCloud() api.InitImport() + api.InitExport() root.Handle("/api/v4/{anything:.*}", http.HandlerFunc(api.Handle404)) @@ -344,6 +349,8 @@ func InitLocal(configservice configservice.ConfigService, globalOptionsFunc app. api.BaseRoutes.Upload = api.BaseRoutes.Uploads.PathPrefix("/{upload_id:[A-Za-z0-9]+}").Subrouter() api.BaseRoutes.Imports = api.BaseRoutes.ApiRoot.PathPrefix("/imports").Subrouter() + api.BaseRoutes.Exports = api.BaseRoutes.ApiRoot.PathPrefix("/exports").Subrouter() + api.BaseRoutes.Export = api.BaseRoutes.Exports.PathPrefix("/{export_name:.+\\.zip}").Subrouter() api.BaseRoutes.Jobs = api.BaseRoutes.ApiRoot.PathPrefix("/jobs").Subrouter() @@ -363,6 +370,7 @@ func InitLocal(configservice configservice.ConfigService, globalOptionsFunc app. api.InitRoleLocal() api.InitUploadLocal() api.InitImportLocal() + api.InitExportLocal() api.InitJobLocal() root.Handle("/api/v4/{anything:.*}", http.HandlerFunc(api.Handle404)) diff --git a/api4/export.go b/api4/export.go new file mode 100644 index 0000000000..b43999cdbe --- /dev/null +++ b/api4/export.go @@ -0,0 +1,86 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package api4 + +import ( + "encoding/json" + "net/http" + "path/filepath" + "time" + + "github.com/mattermost/mattermost-server/v5/audit" + "github.com/mattermost/mattermost-server/v5/model" +) + +func (api *API) InitExport() { + api.BaseRoutes.Exports.Handle("", api.ApiSessionRequired(listExports)).Methods("GET") + api.BaseRoutes.Export.Handle("", api.ApiSessionRequired(deleteExport)).Methods("DELETE") + api.BaseRoutes.Export.Handle("", api.ApiSessionRequired(downloadExport)).Methods("GET") +} + +func listExports(c *Context, w http.ResponseWriter, r *http.Request) { + if !c.IsSystemAdmin() { + c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM) + return + } + + exports, appErr := c.App.ListExports() + if appErr != nil { + c.Err = appErr + return + } + + data, err := json.Marshal(exports) + if err != nil { + c.Err = model.NewAppError("listImports", "app.export.marshal.app_error", nil, err.Error(), http.StatusInternalServerError) + return + } + + w.Write(data) +} + +func deleteExport(c *Context, w http.ResponseWriter, r *http.Request) { + auditRec := c.MakeAuditRecord("deleteExport", audit.Fail) + defer c.LogAuditRec(auditRec) + auditRec.AddMeta("export_name", c.Params.ExportName) + + if !c.IsSystemAdmin() { + c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM) + return + } + + if err := c.App.DeleteExport(c.Params.ExportName); err != nil { + c.Err = err + return + } + + auditRec.Success() + ReturnStatusOK(w) +} + +func downloadExport(c *Context, w http.ResponseWriter, r *http.Request) { + if !c.IsSystemAdmin() { + c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM) + return + } + + filePath := filepath.Join(*c.App.Config().ExportSettings.Directory, c.Params.ExportName) + if ok, err := c.App.FileExists(filePath); err != nil { + c.Err = err + return + } else if !ok { + c.Err = model.NewAppError("downloadExport", "api.export.export_not_found.app_error", nil, "", http.StatusNotFound) + return + } + + file, err := c.App.FileReader(filePath) + if err != nil { + c.Err = err + return + } + defer file.Close() + + w.Header().Set("Content-Type", "application/zip") + http.ServeContent(w, r, c.Params.ExportName, time.Time{}, file) +} diff --git a/api4/export_local.go b/api4/export_local.go new file mode 100644 index 0000000000..b3a243abbc --- /dev/null +++ b/api4/export_local.go @@ -0,0 +1,10 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package api4 + +func (api *API) InitExportLocal() { + api.BaseRoutes.Exports.Handle("", api.ApiLocal(listExports)).Methods("GET") + api.BaseRoutes.Export.Handle("", api.ApiLocal(deleteExport)).Methods("DELETE") + api.BaseRoutes.Export.Handle("", api.ApiLocal(downloadExport)).Methods("GET") +} diff --git a/api4/export_test.go b/api4/export_test.go new file mode 100644 index 0000000000..e3330eaacf --- /dev/null +++ b/api4/export_test.go @@ -0,0 +1,213 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package api4 + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/mattermost/mattermost-server/v5/model" + "github.com/mattermost/mattermost-server/v5/utils/fileutils" + + "github.com/stretchr/testify/require" +) + +func TestListExports(t *testing.T) { + th := Setup(t) + defer th.TearDown() + + t.Run("no permissions", func(t *testing.T) { + exports, resp := th.Client.ListExports() + require.Error(t, resp.Error) + require.Equal(t, "api.context.permissions.app_error", resp.Error.Id) + require.Nil(t, exports) + }) + + th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) { + exports, resp := c.ListExports() + require.Nil(t, resp.Error) + require.Empty(t, exports) + }, "no exports") + + dataDir, found := fileutils.FindDir("data") + require.True(t, found) + + th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) { + exportDir := filepath.Join(dataDir, *th.App.Config().ExportSettings.Directory) + err := os.Mkdir(exportDir, 0700) + require.Nil(t, err) + defer os.RemoveAll(exportDir) + + f, err := os.Create(filepath.Join(exportDir, "export.zip")) + require.Nil(t, err) + f.Close() + + exports, resp := c.ListExports() + require.Nil(t, resp.Error) + require.Len(t, exports, 1) + require.Equal(t, exports[0], "export.zip") + }, "expected exports") + + th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) { + value := *th.App.Config().ExportSettings.Directory + th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ExportSettings.Directory = value + "new" }) + defer th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ExportSettings.Directory = value }) + + exportDir := filepath.Join(dataDir, value+"new") + err := os.Mkdir(exportDir, 0700) + require.Nil(t, err) + defer os.RemoveAll(exportDir) + + exports, resp := c.ListExports() + require.Nil(t, resp.Error) + require.Empty(t, exports) + + f, err := os.Create(filepath.Join(exportDir, "export.zip")) + require.Nil(t, err) + f.Close() + + exports, resp = c.ListExports() + require.Nil(t, resp.Error) + require.Len(t, exports, 1) + require.Equal(t, "export.zip", exports[0]) + }, "change export directory") +} + +func TestDeleteExport(t *testing.T) { + th := Setup(t) + defer th.TearDown() + + t.Run("no permissions", func(t *testing.T) { + ok, resp := th.Client.DeleteExport("export.zip") + require.Error(t, resp.Error) + require.Equal(t, "api.context.permissions.app_error", resp.Error.Id) + require.False(t, ok) + }) + + dataDir, found := fileutils.FindDir("data") + require.True(t, found) + exportDir := filepath.Join(dataDir, *th.App.Config().ExportSettings.Directory) + + th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) { + err := os.Mkdir(exportDir, 0700) + require.Nil(t, err) + defer os.RemoveAll(exportDir) + exportName := "export.zip" + f, err := os.Create(filepath.Join(exportDir, exportName)) + require.Nil(t, err) + f.Close() + + exports, resp := c.ListExports() + require.Nil(t, resp.Error) + require.Len(t, exports, 1) + require.Equal(t, exports[0], exportName) + + ok, resp := c.DeleteExport(exportName) + require.Nil(t, resp.Error) + require.True(t, ok) + + exports, resp = c.ListExports() + require.Nil(t, resp.Error) + require.Empty(t, exports) + + // verify idempotence + ok, resp = c.DeleteExport(exportName) + require.Nil(t, resp.Error) + require.True(t, ok) + }, "successfully delete export") +} + +func TestDownloadExport(t *testing.T) { + th := Setup(t) + defer th.TearDown() + + t.Run("no permissions", func(t *testing.T) { + var buf bytes.Buffer + n, resp := th.Client.DownloadExport("export.zip", &buf, 0) + require.Error(t, resp.Error) + require.Equal(t, "api.context.permissions.app_error", resp.Error.Id) + require.Zero(t, n) + }) + + dataDir, found := fileutils.FindDir("data") + require.True(t, found) + exportDir := filepath.Join(dataDir, *th.App.Config().ExportSettings.Directory) + + th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) { + var buf bytes.Buffer + n, resp := c.DownloadExport("export.zip", &buf, 0) + require.Error(t, resp.Error) + require.Equal(t, "api.export.export_not_found.app_error", resp.Error.Id) + require.Zero(t, n) + }, "not found") + + th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) { + err := os.Mkdir(exportDir, 0700) + require.Nil(t, err) + defer os.RemoveAll(exportDir) + + data := randomBytes(t, 1024*1024) + var buf bytes.Buffer + exportName := "export.zip" + err = ioutil.WriteFile(filepath.Join(exportDir, exportName), data, 0600) + require.Nil(t, err) + + n, resp := c.DownloadExport(exportName, &buf, 0) + require.Nil(t, resp.Error) + require.Equal(t, len(data), int(n)) + require.Equal(t, data, buf.Bytes()) + }, "full download") + + th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) { + err := os.Mkdir(exportDir, 0700) + require.Nil(t, err) + defer os.RemoveAll(exportDir) + + data := randomBytes(t, 1024*1024) + var buf bytes.Buffer + exportName := "export.zip" + err = ioutil.WriteFile(filepath.Join(exportDir, exportName), data, 0600) + require.Nil(t, err) + + offset := 1024 * 512 + n, resp := c.DownloadExport(exportName, &buf, int64(offset)) + require.Nil(t, resp.Error) + require.Equal(t, len(data)-offset, int(n)) + require.Equal(t, data[offset:], buf.Bytes()) + }, "download with offset") +} + +func BenchmarkDownloadExport(b *testing.B) { + th := Setup(b) + defer th.TearDown() + + dataDir, found := fileutils.FindDir("data") + require.True(b, found) + exportDir := filepath.Join(dataDir, *th.App.Config().ExportSettings.Directory) + + err := os.Mkdir(exportDir, 0700) + require.Nil(b, err) + defer os.RemoveAll(exportDir) + + exportName := "export.zip" + f, err := os.Create(filepath.Join(exportDir, exportName)) + require.Nil(b, err) + f.Close() + + err = os.Truncate(filepath.Join(exportDir, exportName), 1024*1024*1024) + require.Nil(b, err) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + outFilePath := filepath.Join(dataDir, fmt.Sprintf("export%d.zip", i)) + outFile, _ := os.Create(outFilePath) + th.SystemAdminClient.DownloadExport(exportName, outFile, 0) + outFile.Close() + os.Remove(outFilePath) + } +} diff --git a/app/app.go b/app/app.go index 0be32d7cc9..152f50056f 100644 --- a/app/app.go +++ b/app/app.go @@ -123,6 +123,10 @@ func (a *App) initJobs() { a.srv.Jobs.ImportDelete = jobsImportDeleteInterface(a) } + if jobsExportProcessInterface != nil { + a.srv.Jobs.ExportProcess = jobsExportProcessInterface(a) + } + if jobsActiveUsersInterface != nil { a.srv.Jobs.ActiveUsers = jobsActiveUsersInterface(a) } diff --git a/app/app_iface.go b/app/app_iface.go index 24030b248e..4ab47dc4f8 100644 --- a/app/app_iface.go +++ b/app/app_iface.go @@ -388,7 +388,7 @@ type AppIface interface { BuildPostReactions(postId string) (*[]ReactionImportData, *model.AppError) BuildPushNotificationMessage(contentsConfig string, post *model.Post, user *model.User, channel *model.Channel, channelName string, senderName string, explicitMention bool, channelWideMention bool, replyToThreadType string) (*model.PushNotification, *model.AppError) BuildSamlMetadataObject(idpMetadata []byte) (*model.SamlMetadataResponse, *model.AppError) - BulkExport(writer io.Writer, file string, pathToEmojiDir string, dirNameToExportEmoji string) *model.AppError + BulkExport(writer io.Writer, outPath string, opts BulkExportOpts) *model.AppError BulkImport(fileReader io.Reader, dryRun bool, workers int) (*model.AppError, int) BulkImportWithPath(fileReader io.Reader, dryRun bool, workers int, importPath string) (*model.AppError, int) CancelJob(jobId string) *model.AppError @@ -466,6 +466,7 @@ type AppIface interface { DeleteCommand(commandID string) *model.AppError DeleteEmoji(emoji *model.Emoji) *model.AppError DeleteEphemeralPost(userID, postId string) + DeleteExport(name string) *model.AppError DeleteFlaggedPosts(postId string) DeleteGroup(groupID string) (*model.Group, *model.AppError) DeleteGroupMember(groupID string, userID string) (*model.GroupMember, *model.AppError) @@ -782,6 +783,7 @@ type AppIface interface { LimitedClientConfig() map[string]string ListAllCommands(teamID string, T goi18n.TranslateFunc) ([]*model.Command, *model.AppError) ListDirectory(path string) ([]string, *model.AppError) + ListExports() ([]string, *model.AppError) ListImports() ([]string, *model.AppError) ListPluginKeys(pluginId string, page, perPage int) ([]string, *model.AppError) ListTeamCommands(teamID string) ([]*model.Command, *model.AppError) diff --git a/app/enterprise.go b/app/enterprise.go index 44e7638c5e..754d37da1f 100644 --- a/app/enterprise.go +++ b/app/enterprise.go @@ -120,6 +120,12 @@ func RegisterJobsImportDeleteInterface(f func(*App) tjobs.ImportDeleteInterface) jobsImportDeleteInterface = f } +var jobsExportProcessInterface func(*App) tjobs.ExportProcessInterface + +func RegisterJobsExportProcessInterface(f func(*App) tjobs.ExportProcessInterface) { + jobsExportProcessInterface = f +} + var productNoticesJobInterface func(*App) tjobs.ProductNoticesJobInterface func RegisterProductNoticesJobInterface(f func(*App) tjobs.ProductNoticesJobInterface) { diff --git a/app/export.go b/app/export.go index 6c8dca719c..ce515dca4e 100644 --- a/app/export.go +++ b/app/export.go @@ -4,6 +4,7 @@ package app import ( + "archive/zip" "encoding/json" "io" "net/http" @@ -18,6 +19,15 @@ import ( "github.com/mattermost/mattermost-server/v5/store" ) +type BulkExportOpts struct { + IncludeAttachments bool + CreateArchive bool +} + +// ExportDataDir is the name of the directory were to store additional data +// included with the export (e.g. file attachments). +const ExportDataDir = "data" + // We use this map to identify the exportable preferences. // Here we link the preference category and name, to the name of the relevant field in the import struct. var exportablePreferences = map[ComparablePreference]string{{ @@ -53,7 +63,19 @@ var exportablePreferences = map[ComparablePreference]string{{ }: "EmailInterval", } -func (a *App) BulkExport(writer io.Writer, file string, pathToEmojiDir string, dirNameToExportEmoji string) *model.AppError { +func (a *App) BulkExport(writer io.Writer, outPath string, opts BulkExportOpts) *model.AppError { + var zipWr *zip.Writer + if opts.CreateArchive { + var err error + zipWr = zip.NewWriter(writer) + defer zipWr.Close() + writer, err = zipWr.Create("import.jsonl") + if err != nil { + return model.NewAppError("BulkExport", "app.export.zip_create.error", + nil, "err="+err.Error(), http.StatusInternalServerError) + } + } + mlog.Info("Bulk export: exporting version") if err := a.exportVersion(writer); err != nil { return err @@ -75,25 +97,47 @@ func (a *App) BulkExport(writer io.Writer, file string, pathToEmojiDir string, d } mlog.Info("Bulk export: exporting posts") - if err := a.exportAllPosts(writer); err != nil { + attachments, err := a.exportAllPosts(writer, opts.IncludeAttachments) + if err != nil { return err } mlog.Info("Bulk export: exporting emoji") - if err := a.exportCustomEmoji(writer, file, pathToEmojiDir, dirNameToExportEmoji); err != nil { + emojiPaths, err := a.exportCustomEmoji(writer, outPath, "exported_emoji", !opts.CreateArchive) + if err != nil { return err } mlog.Info("Bulk export: exporting direct channels") - if err := a.exportAllDirectChannels(writer); err != nil { + if err = a.exportAllDirectChannels(writer); err != nil { return err } mlog.Info("Bulk export: exporting direct posts") - if err := a.exportAllDirectPosts(writer); err != nil { + directAttachments, err := a.exportAllDirectPosts(writer, opts.IncludeAttachments) + if err != nil { return err } + if opts.IncludeAttachments { + mlog.Info("Bulk export: exporting file attachments") + for _, attachment := range attachments { + if err := a.exportFile(outPath, *attachment.Path, zipWr); err != nil { + return err + } + } + for _, attachment := range directAttachments { + if err := a.exportFile(outPath, *attachment.Path, zipWr); err != nil { + return err + } + } + for _, emojiPath := range emojiPaths { + if err := a.exportFile(outPath, emojiPath, zipWr); err != nil { + return err + } + } + } + return nil } @@ -334,17 +378,18 @@ func (a *App) buildUserNotifyProps(notifyProps model.StringMap) *UserNotifyProps } } -func (a *App) exportAllPosts(writer io.Writer) *model.AppError { +func (a *App) exportAllPosts(writer io.Writer, withAttachments bool) ([]AttachmentImportData, *model.AppError) { + var attachments []AttachmentImportData afterId := strings.Repeat("0", 26) for { posts, nErr := a.Srv().Store.Post().GetParentsForExportAfter(1000, afterId) if nErr != nil { - return model.NewAppError("exportAllPosts", "app.post.get_posts.app_error", nil, nErr.Error(), http.StatusInternalServerError) + return nil, model.NewAppError("exportAllPosts", "app.post.get_posts.app_error", nil, nErr.Error(), http.StatusInternalServerError) } if len(posts) == 0 { - return nil + return attachments, nil } for _, post := range posts { @@ -357,33 +402,50 @@ func (a *App) exportAllPosts(writer io.Writer) *model.AppError { postLine := ImportLineForPost(post) - var err *model.AppError - postLine.Post.Replies, err = a.buildPostReplies(post.Id) + replies, replyAttachments, err := a.buildPostReplies(post.Id, withAttachments) if err != nil { - return err + return nil, err } + if withAttachments && len(replyAttachments) > 0 { + attachments = append(attachments, replyAttachments...) + } + + postLine.Post.Replies = &replies postLine.Post.Reactions = &[]ReactionImportData{} if post.HasReactions { postLine.Post.Reactions, err = a.BuildPostReactions(post.Id) if err != nil { - return err + return nil, err + } + } + + if len(post.FileIds) > 0 { + postAttachments, err := a.buildPostAttachments(post.Id) + if err != nil { + return nil, err + } + postLine.Post.Attachments = &postAttachments + + if withAttachments && len(postAttachments) > 0 { + attachments = append(attachments, postAttachments...) } } if err := a.exportWriteLine(writer, postLine); err != nil { - return err + return nil, err } } } } -func (a *App) buildPostReplies(postId string) (*[]ReplyImportData, *model.AppError) { +func (a *App) buildPostReplies(postId string, withAttachments bool) ([]ReplyImportData, []AttachmentImportData, *model.AppError) { var replies []ReplyImportData + var attachments []AttachmentImportData replyPosts, nErr := a.Srv().Store.Post().GetRepliesForExport(postId) if nErr != nil { - return nil, model.NewAppError("buildPostReplies", "app.post.get_posts.app_error", nil, nErr.Error(), http.StatusInternalServerError) + return nil, nil, model.NewAppError("buildPostReplies", "app.post.get_posts.app_error", nil, nErr.Error(), http.StatusInternalServerError) } for _, reply := range replyPosts { @@ -392,13 +454,24 @@ func (a *App) buildPostReplies(postId string) (*[]ReplyImportData, *model.AppErr var appErr *model.AppError replyImportObject.Reactions, appErr = a.BuildPostReactions(reply.Id) if appErr != nil { - return nil, appErr + return nil, nil, appErr } } + if len(reply.FileIds) > 0 { + postAttachments, appErr := a.buildPostAttachments(reply.Id) + if appErr != nil { + return nil, nil, appErr + } + replyImportObject.Attachments = &attachments + if withAttachments && len(postAttachments) > 0 { + attachments = append(attachments, postAttachments...) + } + } + replies = append(replies, *replyImportObject) } - return &replies, nil + return replies, attachments, nil } func (a *App) BuildPostReactions(postId string) (*[]ReactionImportData, *model.AppError) { @@ -426,13 +499,28 @@ func (a *App) BuildPostReactions(postId string) (*[]ReactionImportData, *model.A } -func (a *App) exportCustomEmoji(writer io.Writer, file string, pathToEmojiDir string, dirNameToExportEmoji string) *model.AppError { +func (a *App) buildPostAttachments(postId string) ([]AttachmentImportData, *model.AppError) { + infos, nErr := a.Srv().Store.FileInfo().GetForPost(postId, false, false, false) + if nErr != nil { + return nil, model.NewAppError("buildPostAttachments", "app.file_info.get_for_post.app_error", nil, nErr.Error(), http.StatusInternalServerError) + } + + attachments := make([]AttachmentImportData, 0, len(infos)) + for _, info := range infos { + attachments = append(attachments, AttachmentImportData{Path: &info.Path}) + } + + return attachments, nil +} + +func (a *App) exportCustomEmoji(writer io.Writer, outPath, exportDir string, exportFiles bool) ([]string, *model.AppError) { + var emojiPaths []string pageNumber := 0 for { customEmojiList, err := a.GetEmojiList(pageNumber, 100, model.EMOJI_SORT_BY_NAME) if err != nil { - return err + return nil, err } if len(customEmojiList) == 0 { @@ -441,42 +529,35 @@ func (a *App) exportCustomEmoji(writer io.Writer, file string, pathToEmojiDir st pageNumber++ - pathToDir := a.createDirForEmoji(file, dirNameToExportEmoji) + emojiPath := filepath.Join(*a.Config().FileSettings.Directory, "emoji") + pathToDir := filepath.Join(outPath, exportDir) + if exportFiles { + if _, err := os.Stat(pathToDir); os.IsNotExist(err) { + os.Mkdir(pathToDir, os.ModePerm) + } + } for _, emoji := range customEmojiList { - emojiImagePath := pathToEmojiDir + emoji.Id + "/image" - err := a.copyEmojiImages(emoji.Id, emojiImagePath, pathToDir) - if err != nil { - return model.NewAppError("BulkExport", "app.export.export_custom_emoji.copy_emoji_images.error", nil, "err="+err.Error(), http.StatusBadRequest) + emojiImagePath := filepath.Join(emojiPath, emoji.Id, "image") + filePath := filepath.Join(exportDir, emoji.Id, "image") + if exportFiles { + err := a.copyEmojiImages(emoji.Id, emojiImagePath, pathToDir) + if err != nil { + return nil, model.NewAppError("BulkExport", "app.export.export_custom_emoji.copy_emoji_images.error", nil, "err="+err.Error(), http.StatusBadRequest) + } + } else { + filePath = filepath.Join("emoji", emoji.Id, "image") + emojiPaths = append(emojiPaths, filePath) } - filePath := dirNameToExportEmoji + "/" + emoji.Id + "/image" - emojiImportObject := ImportLineFromEmoji(emoji, filePath) - if err := a.exportWriteLine(writer, emojiImportObject); err != nil { - return err + return nil, err } } } - return nil -} - -// Creates directory named 'exported_emoji' to copy the emoji files -// Directory and the file specified by admin share the same path -func (a *App) createDirForEmoji(file string, dirName string) string { - pathToFile, _ := filepath.Abs(file) - pathSlice := strings.Split(pathToFile, "/") - if len(pathSlice) > 0 { - pathSlice = pathSlice[:len(pathSlice)-1] - } - pathToDir := strings.Join(pathSlice, "/") + "/" + dirName - - if _, err := os.Stat(pathToDir); os.IsNotExist(err) { - os.Mkdir(pathToDir, os.ModePerm) - } - return pathToDir + return emojiPaths, nil } // Copies emoji files from 'data/emoji' dir to 'exported_emoji' dir @@ -543,12 +624,13 @@ func (a *App) exportAllDirectChannels(writer io.Writer) *model.AppError { return nil } -func (a *App) exportAllDirectPosts(writer io.Writer) *model.AppError { +func (a *App) exportAllDirectPosts(writer io.Writer, withAttachments bool) ([]AttachmentImportData, *model.AppError) { + var attachments []AttachmentImportData afterId := strings.Repeat("0", 26) for { posts, err := a.Srv().Store.Post().GetDirectPostParentsForExportAfter(1000, afterId) if err != nil { - return model.NewAppError("exportAllDirectPosts", "app.post.get_direct_posts.app_error", nil, err.Error(), http.StatusInternalServerError) + return nil, model.NewAppError("exportAllDirectPosts", "app.post.get_direct_posts.app_error", nil, err.Error(), http.StatusInternalServerError) } if len(posts) == 0 { @@ -563,18 +645,106 @@ func (a *App) exportAllDirectPosts(writer io.Writer) *model.AppError { continue } + // Handle attachments. + var postAttachments []AttachmentImportData + var err *model.AppError + if len(post.FileIds) > 0 { + postAttachments, err = a.buildPostAttachments(post.Id) + if err != nil { + return nil, err + } + + if withAttachments && len(postAttachments) > 0 { + attachments = append(attachments, postAttachments...) + } + } + // Do the Replies. - replies, err := a.buildPostReplies(post.Id) + replies, replyAttachments, err := a.buildPostReplies(post.Id, withAttachments) if err != nil { - return err + return nil, err + } + + if withAttachments && len(replyAttachments) > 0 { + attachments = append(attachments, replyAttachments...) } postLine := ImportLineForDirectPost(post) - postLine.DirectPost.Replies = replies + postLine.DirectPost.Replies = &replies + if len(postAttachments) > 0 { + postLine.DirectPost.Attachments = &postAttachments + } if err := a.exportWriteLine(writer, postLine); err != nil { - return err + return nil, err } } } + return attachments, nil +} + +func (a *App) exportFile(outPath, filePath string, zipWr *zip.Writer) *model.AppError { + var wr io.Writer + var err error + rd, appErr := a.FileReader(filePath) + if appErr != nil { + return appErr + } + defer rd.Close() + + if zipWr != nil { + wr, err = zipWr.CreateHeader(&zip.FileHeader{ + Name: filepath.Join(ExportDataDir, filePath), + Method: zip.Store, + }) + if err != nil { + return model.NewAppError("exportFileAttachment", "app.export.export_attachment.zip_create_header.error", + nil, "err="+err.Error(), http.StatusInternalServerError) + } + } else { + filePath = filepath.Join(outPath, ExportDataDir, filePath) + if err = os.MkdirAll(filepath.Dir(filePath), 0700); err != nil { + return model.NewAppError("exportFileAttachment", "app.export.export_attachment.mkdirall.error", + nil, "err="+err.Error(), http.StatusInternalServerError) + } + + wr, err = os.Create(filePath) + if err != nil { + return model.NewAppError("exportFileAttachment", "app.export.export_attachment.create_file.error", + nil, "err="+err.Error(), http.StatusInternalServerError) + } + defer wr.(*os.File).Close() + } + + if _, err := io.Copy(wr, rd); err != nil { + return model.NewAppError("exportFileAttachment", "app.export.export_attachment.copy_file.error", + nil, "err="+err.Error(), http.StatusInternalServerError) + } + return nil } + +func (a *App) ListExports() ([]string, *model.AppError) { + exports, appErr := a.ListDirectory(*a.Config().ExportSettings.Directory) + if appErr != nil { + return nil, appErr + } + + results := make([]string, len(exports)) + for i := range exports { + results[i] = filepath.Base(exports[i]) + } + + return results, nil +} + +func (a *App) DeleteExport(name string) *model.AppError { + filePath := filepath.Join(*a.Config().ExportSettings.Directory, name) + + if ok, err := a.FileExists(filePath); err != nil { + return err + } else if !ok { + return nil + } + + return a.RemoveFile(filePath) +} diff --git a/app/export_test.go b/app/export_test.go index 17e1e5929d..f4e63c2ffe 100644 --- a/app/export_test.go +++ b/app/export_test.go @@ -5,7 +5,9 @@ package app import ( "bytes" + "io/ioutil" "os" + "path/filepath" "sort" "testing" @@ -13,6 +15,8 @@ import ( "github.com/stretchr/testify/require" "github.com/mattermost/mattermost-server/v5/model" + "github.com/mattermost/mattermost-server/v5/utils" + "github.com/mattermost/mattermost-server/v5/utils/fileutils" ) func TestReactionsOfPost(t *testing.T) { @@ -111,16 +115,6 @@ func TestExportUserChannels(t *testing.T) { } } -func TestDirCreationForEmoji(t *testing.T) { - th := SetupWithStoreMock(t) - defer th.TearDown() - - pathToDir := th.App.createDirForEmoji("test.json", "exported_emoji_test") - defer os.Remove(pathToDir) - _, err := os.Stat(pathToDir) - require.False(t, os.IsNotExist(err), "Directory exported_emoji_test should exist") -} - func TestCopyEmojiImages(t *testing.T) { th := SetupWithStoreMock(t) defer th.TearDown() @@ -164,11 +158,13 @@ func TestExportCustomEmoji(t *testing.T) { require.Nil(t, err) defer os.Remove(filePath) - pathToEmojiDir := "../data/emoji/" dirNameToExportEmoji := "exported_emoji_test" defer os.RemoveAll("../" + dirNameToExportEmoji) - err = th.App.exportCustomEmoji(fileWriter, filePath, pathToEmojiDir, dirNameToExportEmoji) + outPath, err := filepath.Abs(filePath) + require.Nil(t, err) + + _, err = th.App.exportCustomEmoji(fileWriter, outPath, dirNameToExportEmoji, false) require.Nil(t, err, "should not have failed") } @@ -182,7 +178,7 @@ func TestExportAllUsers(t *testing.T) { require.Nil(t, err) var b bytes.Buffer - err = th1.App.BulkExport(&b, "somefile", "somePath", "someDir") + err = th1.App.BulkExport(&b, "somePath", BulkExportOpts{}) require.Nil(t, err) th2 := Setup(t) @@ -228,7 +224,7 @@ func TestExportDMChannel(t *testing.T) { th1.CreateDmChannel(th1.BasicUser2) var b bytes.Buffer - err := th1.App.BulkExport(&b, "somefile", "somePath", "someDir") + err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{}) require.Nil(t, err) channels, nErr := th1.App.Srv().Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000") @@ -264,7 +260,7 @@ func TestExportDMChannelToSelf(t *testing.T) { th1.CreateDmChannel(th1.BasicUser) var b bytes.Buffer - err := th1.App.BulkExport(&b, "somefile", "somePath", "someDir") + err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{}) require.Nil(t, err) channels, nErr := th1.App.Srv().Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000") @@ -302,7 +298,7 @@ func TestExportGMChannel(t *testing.T) { th1.CreateGroupChannel(user1, user2) var b bytes.Buffer - err := th1.App.BulkExport(&b, "somefile", "somePath", "someDir") + err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{}) require.Nil(t, err) channels, nErr := th1.App.Srv().Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000") @@ -334,7 +330,7 @@ func TestExportGMandDMChannels(t *testing.T) { th1.CreateGroupChannel(user1, user2) var b bytes.Buffer - err := th1.App.BulkExport(&b, "somefile", "somePath", "someDir") + err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{}) require.Nil(t, err) channels, nErr := th1.App.Srv().Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000") @@ -417,7 +413,7 @@ func TestExportDMandGMPost(t *testing.T) { assert.Equal(t, 4, len(posts)) var b bytes.Buffer - err = th1.App.BulkExport(&b, "somefile", "somePath", "someDir") + err = th1.App.BulkExport(&b, "somePath", BulkExportOpts{}) require.Nil(t, err) th1.TearDown() @@ -492,7 +488,7 @@ func TestExportPostWithProps(t *testing.T) { require.NotEmpty(t, posts[1].Props) var b bytes.Buffer - err = th1.App.BulkExport(&b, "somefile", "somePath", "someDir") + err = th1.App.BulkExport(&b, "somePath", BulkExportOpts{}) require.Nil(t, err) th1.TearDown() @@ -530,7 +526,7 @@ func TestExportDMPostWithSelf(t *testing.T) { th1.CreatePost(dmChannel) var b bytes.Buffer - err := th1.App.BulkExport(&b, "somefile", "somePath", "someDir") + err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{}) require.Nil(t, err) posts, nErr := th1.App.Srv().Store.Post().GetDirectPostParentsForExportAfter(1000, "0000000") @@ -557,3 +553,57 @@ func TestExportDMPostWithSelf(t *testing.T) { assert.Equal(t, 1, len((*posts[0].ChannelMembers))) assert.Equal(t, th1.BasicUser.Username, (*posts[0].ChannelMembers)[0]) } + +func TestBulkExport(t *testing.T) { + th := Setup(t) + testsDir, _ := fileutils.FindDir("tests") + + dir, err := ioutil.TempDir("", "import_test") + require.Nil(t, err) + defer os.RemoveAll(dir) + + extractImportFile := func(filePath string) *os.File { + importFile, err2 := os.Open(filePath) + require.Nil(t, err2) + defer importFile.Close() + + info, err2 := importFile.Stat() + require.Nil(t, err2) + + paths, err2 := utils.UnzipToPath(importFile, info.Size(), dir) + require.Nil(t, err2) + require.NotEmpty(t, paths) + + jsonFile, err2 := os.Open(filepath.Join(dir, "import.jsonl")) + require.Nil(t, err2) + + return jsonFile + } + + jsonFile := extractImportFile(filepath.Join(testsDir, "import_test.zip")) + defer jsonFile.Close() + + err, _ = th.App.BulkImportWithPath(jsonFile, false, 1, dir) + require.Nil(t, err) + + exportFile, err := os.Create(filepath.Join(dir, "export.zip")) + require.Nil(t, err) + defer exportFile.Close() + + opts := BulkExportOpts{ + IncludeAttachments: true, + CreateArchive: true, + } + err = th.App.BulkExport(exportFile, dir, opts) + require.Nil(t, err) + + th.TearDown() + th = Setup(t) + defer th.TearDown() + + jsonFile = extractImportFile(filepath.Join(dir, "export.zip")) + defer jsonFile.Close() + + err, _ = th.App.BulkImportWithPath(jsonFile, false, 1, filepath.Join(dir, "data")) + require.Nil(t, err) +} diff --git a/app/import_functions.go b/app/import_functions.go index aa417e1547..32433c21e4 100644 --- a/app/import_functions.go +++ b/app/import_functions.go @@ -9,7 +9,7 @@ import ( "crypto/sha1" "errors" "fmt" - "io" + "io/ioutil" "net/http" "os" "path" @@ -1117,10 +1117,15 @@ func (a *App) importAttachment(data *AttachmentImportData, post *model.Post, tea if file == nil || err != nil { return nil, model.NewAppError("BulkImport", "app.import.attachment.bad_file.error", map[string]interface{}{"FilePath": *data.Path}, "", http.StatusBadRequest) } + defer file.Close() timestamp := utils.TimeFromMillis(post.CreateAt) - buf := bytes.NewBuffer(nil) - _, _ = io.Copy(buf, file) + + fileData, err := ioutil.ReadAll(file) + if err != nil { + return nil, model.NewAppError("BulkImport", "app.import.attachment.read_file_data.error", map[string]interface{}{"FilePath": *data.Path}, "", http.StatusBadRequest) + } + // Go over existing files in the post and see if there already exists a file with the same name, size and hash. If so - skip it if post.Id != "" { oldFiles, err := a.GetFileInfosForPost(post.Id, true) @@ -1128,11 +1133,11 @@ func (a *App) importAttachment(data *AttachmentImportData, post *model.Post, tea return nil, model.NewAppError("BulkImport", "app.import.attachment.file_upload.error", map[string]interface{}{"FilePath": *data.Path}, "", http.StatusBadRequest) } for _, oldFile := range oldFiles { - if oldFile.Name != path.Base(file.Name()) || oldFile.Size != int64(buf.Len()) { + if oldFile.Name != path.Base(file.Name()) || oldFile.Size != int64(len(fileData)) { continue } // check md5 - newHash := sha1.Sum(buf.Bytes()) + newHash := sha1.Sum(fileData) oldFileData, err := a.GetFile(oldFile.Id) if err != nil { return nil, model.NewAppError("BulkImport", "app.import.attachment.file_upload.error", map[string]interface{}{"FilePath": *data.Path}, "", http.StatusBadRequest) @@ -1145,15 +1150,19 @@ func (a *App) importAttachment(data *AttachmentImportData, post *model.Post, tea } } } - fileInfo, appErr := a.DoUploadFile(timestamp, teamID, post.ChannelId, post.UserId, file.Name(), buf.Bytes()) + + mlog.Info("Uploading file with name", mlog.String("file_name", file.Name())) + + fileInfo, appErr := a.DoUploadFile(timestamp, teamID, post.ChannelId, post.UserId, file.Name(), fileData) if appErr != nil { mlog.Error("Failed to upload file:", mlog.Err(appErr)) return nil, appErr } - a.HandleImages([]string{fileInfo.PreviewPath}, []string{fileInfo.ThumbnailPath}, [][]byte{buf.Bytes()}) + if fileInfo.IsImage() { + a.HandleImages([]string{fileInfo.PreviewPath}, []string{fileInfo.ThumbnailPath}, [][]byte{fileData}) + } - mlog.Info("Uploading file with name", mlog.String("file_name", file.Name())) return fileInfo, nil } diff --git a/app/opentracing/opentracing_layer.go b/app/opentracing/opentracing_layer.go index 690a8f3786..8455aee48c 100644 --- a/app/opentracing/opentracing_layer.go +++ b/app/opentracing/opentracing_layer.go @@ -878,7 +878,7 @@ func (a *OpenTracingAppLayer) BuildSamlMetadataObject(idpMetadata []byte) (*mode return resultVar0, resultVar1 } -func (a *OpenTracingAppLayer) BulkExport(writer io.Writer, file string, pathToEmojiDir string, dirNameToExportEmoji string) *model.AppError { +func (a *OpenTracingAppLayer) BulkExport(writer io.Writer, outPath string, opts app.BulkExportOpts) *model.AppError { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.BulkExport") @@ -890,7 +890,7 @@ func (a *OpenTracingAppLayer) BulkExport(writer io.Writer, file string, pathToEm }() defer span.Finish() - resultVar0 := a.app.BulkExport(writer, file, pathToEmojiDir, dirNameToExportEmoji) + resultVar0 := a.app.BulkExport(writer, outPath, opts) if resultVar0 != nil { span.LogFields(spanlog.Error(resultVar0)) @@ -2756,6 +2756,28 @@ func (a *OpenTracingAppLayer) DeleteEphemeralPost(userID string, postId string) a.app.DeleteEphemeralPost(userID, postId) } +func (a *OpenTracingAppLayer) DeleteExport(name string) *model.AppError { + origCtx := a.ctx + span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.DeleteExport") + + a.ctx = newCtx + a.app.Srv().Store.SetContext(newCtx) + defer func() { + a.app.Srv().Store.SetContext(origCtx) + a.ctx = origCtx + }() + + defer span.Finish() + resultVar0 := a.app.DeleteExport(name) + + if resultVar0 != nil { + span.LogFields(spanlog.Error(resultVar0)) + ext.Error.Set(span, true) + } + + return resultVar0 +} + func (a *OpenTracingAppLayer) DeleteFlaggedPosts(postId string) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.DeleteFlaggedPosts") @@ -10423,6 +10445,28 @@ func (a *OpenTracingAppLayer) ListDirectory(path string) ([]string, *model.AppEr return resultVar0, resultVar1 } +func (a *OpenTracingAppLayer) ListExports() ([]string, *model.AppError) { + origCtx := a.ctx + span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ListExports") + + a.ctx = newCtx + a.app.Srv().Store.SetContext(newCtx) + defer func() { + a.app.Srv().Store.SetContext(origCtx) + a.ctx = origCtx + }() + + defer span.Finish() + resultVar0, resultVar1 := a.app.ListExports() + + if resultVar1 != nil { + span.LogFields(spanlog.Error(resultVar1)) + ext.Error.Set(span, true) + } + + return resultVar0, resultVar1 +} + func (a *OpenTracingAppLayer) ListImports() ([]string, *model.AppError) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ListImports") diff --git a/cmd/mattermost/commands/export.go b/cmd/mattermost/commands/export.go index 07e4fb0652..817eb3b56e 100644 --- a/cmd/mattermost/commands/export.go +++ b/cmd/mattermost/commands/export.go @@ -7,13 +7,15 @@ import ( "context" "fmt" "os" + "path/filepath" "time" + "github.com/mattermost/mattermost-server/v5/app" + "github.com/mattermost/mattermost-server/v5/audit" + "github.com/mattermost/mattermost-server/v5/model" + "github.com/pkg/errors" "github.com/spf13/cobra" - - "github.com/mattermost/mattermost-server/v5/audit" - "github.com/mattermost/mattermost-server/v5/model" ) var ExportCmd = &cobra.Command{ @@ -74,6 +76,8 @@ func init() { GlobalRelayZipExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.") BulkExportCmd.Flags().Bool("all-teams", true, "Export all teams from the server.") + BulkExportCmd.Flags().Bool("attachments", false, "Also export file attachments.") + BulkExportCmd.Flags().Bool("archive", false, "Outputs a single archive file.") ExportCmd.AddCommand(ScheduleExportCmd) ExportCmd.AddCommand(CsvExportCmd) @@ -202,25 +206,31 @@ func bulkExportCmdF(command *cobra.Command, args []string) error { return errors.New("Nothing to export. Please specify the --all-teams flag to export all teams.") } + attachments, err := command.Flags().GetBool("attachments") + if err != nil { + return errors.Wrap(err, "attachments flag error") + } + + archive, err := command.Flags().GetBool("archive") + if err != nil { + return errors.Wrap(err, "archive flag error") + } + fileWriter, err := os.Create(args[0]) if err != nil { return err } defer fileWriter.Close() - // Path to directory of custom emoji - pathToEmojiDir := "data/emoji/" - - customDataDir := a.Config().FileSettings.Directory - if customDataDir != nil && *customDataDir != "" { - pathToEmojiDir = *customDataDir + "emoji/" + outPath, err := filepath.Abs(args[0]) + if err != nil { + return err } - // Name of the directory to export custom emoji - dirNameToExportEmoji := "exported_emoji" - - // args[0] points to the filename/filepath passed with export bulk command - if err := a.BulkExport(fileWriter, args[0], pathToEmojiDir, dirNameToExportEmoji); err != nil { + var opts app.BulkExportOpts + opts.IncludeAttachments = attachments + opts.CreateArchive = archive + if err := a.BulkExport(fileWriter, filepath.Dir(outPath), opts); err != nil { CommandPrintErrorln(err.Error()) return err } diff --git a/cmd/mattermost/commands/import.go b/cmd/mattermost/commands/import.go index 46c9b9351a..691495fe76 100644 --- a/cmd/mattermost/commands/import.go +++ b/cmd/mattermost/commands/import.go @@ -38,6 +38,7 @@ func init() { BulkImportCmd.Flags().Bool("apply", false, "Save the import data to the database. Use with caution - this cannot be reverted.") BulkImportCmd.Flags().Bool("validate", false, "Validate the import data without making any changes to the system.") BulkImportCmd.Flags().Int("workers", 2, "How many workers to run whilst doing the import.") + BulkImportCmd.Flags().String("import-path", "", "A path to the data directory to import files from.") ImportCmd.AddCommand( BulkImportCmd, @@ -118,6 +119,11 @@ func bulkImportCmdF(command *cobra.Command, args []string) error { return errors.New("Workers flag error") } + importPath, err := command.Flags().GetString("import-path") + if err != nil { + return errors.New("import-path flag error") + } + if len(args) != 1 { return errors.New("Incorrect number of arguments.") } @@ -143,7 +149,7 @@ func bulkImportCmdF(command *cobra.Command, args []string) error { CommandPrettyPrintln("") - if err, lineNumber := a.BulkImport(fileReader, !apply, workers); err != nil { + if err, lineNumber := a.BulkImportWithPath(fileReader, !apply, workers, importPath); err != nil { CommandPrintErrorln(err.Error()) if lineNumber != 0 { CommandPrintErrorln(fmt.Sprintf("Error occurred on data file line %v", lineNumber)) diff --git a/i18n/en.json b/i18n/en.json index 825d6eab38..230bffaa20 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -1324,6 +1324,10 @@ "id": "api.emoji.upload.open.app_error", "translation": "Unable to create the emoji. An error occurred when trying to open the attached image." }, + { + "id": "api.export.export_not_found.app_error", + "translation": "Unable to find export file." + }, { "id": "api.file.append_file.app_error", "translation": "Unable to append data to the file." @@ -4086,6 +4090,22 @@ "id": "app.emoji.get_list.internal_error", "translation": "Unable to get the emoji." }, + { + "id": "app.export.export_attachment.copy_file.error", + "translation": "Failed to copy file during export." + }, + { + "id": "app.export.export_attachment.create_file.error", + "translation": "Failed to created file during export." + }, + { + "id": "app.export.export_attachment.mkdirall.error", + "translation": "Failed to create directory during export." + }, + { + "id": "app.export.export_attachment.zip_create_header.error", + "translation": "Failed to create zip header during export." + }, { "id": "app.export.export_custom_emoji.copy_emoji_images.error", "translation": "Unable to copy custom emoji images" @@ -4098,6 +4118,14 @@ "id": "app.export.export_write_line.json_marshall.error", "translation": "An error occurred marshalling the JSON data for export." }, + { + "id": "app.export.marshal.app_error", + "translation": "Unable to marshal response." + }, + { + "id": "app.export.zip_create.error", + "translation": "Failed to add file to zip archive during export." + }, { "id": "app.file_info.get.app_error", "translation": "Unable to get the file info." @@ -4146,6 +4174,10 @@ "id": "app.import.attachment.file_upload.error", "translation": "Error uploading the file: \"{{.FilePath}}\"" }, + { + "id": "app.import.attachment.read_file_data.error", + "translation": "Failed to read file attachment during import." + }, { "id": "app.import.bulk_import.file_scan.error", "translation": "Error reading import data file." @@ -7274,6 +7306,14 @@ "id": "model.config.is_valid.encrypt_sql.app_error", "translation": "Invalid at rest encrypt key for SQL settings. Must be 32 chars or more." }, + { + "id": "model.config.is_valid.export.directory.app_error", + "translation": "Value for Directory should not be empty." + }, + { + "id": "model.config.is_valid.export.retention_days_too_low.app_error", + "translation": "Invalid value for RetentionDays. Value should be greater than 0" + }, { "id": "model.config.is_valid.file_driver.app_error", "translation": "Invalid driver name for file settings. Must be 'local' or 'amazons3'." diff --git a/imports/placeholder.go b/imports/placeholder.go index 59c508f340..ba3d6966db 100644 --- a/imports/placeholder.go +++ b/imports/placeholder.go @@ -27,4 +27,7 @@ import ( // This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty. _ "github.com/mattermost/mattermost-server/v5/jobs/import_delete" + + // This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty. + _ "github.com/mattermost/mattermost-server/v5/jobs/export_process" ) diff --git a/jobs/export_process/worker.go b/jobs/export_process/worker.go new file mode 100644 index 0000000000..9aca933bf9 --- /dev/null +++ b/jobs/export_process/worker.go @@ -0,0 +1,138 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package export_process + +import ( + "io" + "path/filepath" + + "github.com/mattermost/mattermost-server/v5/app" + "github.com/mattermost/mattermost-server/v5/jobs" + tjobs "github.com/mattermost/mattermost-server/v5/jobs/interfaces" + "github.com/mattermost/mattermost-server/v5/mlog" + "github.com/mattermost/mattermost-server/v5/model" +) + +func init() { + app.RegisterJobsExportProcessInterface(func(a *app.App) tjobs.ExportProcessInterface { + return &ExportProcessInterfaceImpl{a} + }) +} + +type ExportProcessInterfaceImpl struct { + app *app.App +} + +type ExportProcessWorker struct { + name string + stopChan chan struct{} + stoppedChan chan struct{} + jobsChan chan model.Job + jobServer *jobs.JobServer + app *app.App +} + +func (i *ExportProcessInterfaceImpl) MakeWorker() model.Worker { + return &ExportProcessWorker{ + name: "ExportProcess", + stopChan: make(chan struct{}), + stoppedChan: make(chan struct{}), + jobsChan: make(chan model.Job), + jobServer: i.app.Srv().Jobs, + app: i.app, + } +} + +func (w *ExportProcessWorker) JobChannel() chan<- model.Job { + return w.jobsChan +} + +func (w *ExportProcessWorker) Run() { + mlog.Debug("Worker started", mlog.String("worker", w.name)) + + defer func() { + mlog.Debug("Worker finished", mlog.String("worker", w.name)) + close(w.stoppedChan) + }() + + for { + select { + case <-w.stopChan: + mlog.Debug("Worker received stop signal", mlog.String("worker", w.name)) + return + case job := <-w.jobsChan: + mlog.Debug("Worker received a new candidate job.", mlog.String("worker", w.name)) + w.doJob(&job) + } + } +} + +func (w *ExportProcessWorker) Stop() { + mlog.Debug("Worker stopping", mlog.String("worker", w.name)) + close(w.stopChan) + <-w.stoppedChan +} + +func (w *ExportProcessWorker) doJob(job *model.Job) { + if claimed, err := w.jobServer.ClaimJob(job); err != nil { + mlog.Warn("Worker experienced an error while trying to claim job", + mlog.String("worker", w.name), + mlog.String("job_id", job.Id), + mlog.String("error", err.Error())) + return + } else if !claimed { + return + } + + opts := app.BulkExportOpts{ + CreateArchive: true, + } + + includeAttachments, ok := job.Data["include_attachments"] + if ok && includeAttachments == "true" { + opts.IncludeAttachments = true + } + + outPath := *w.app.Config().ExportSettings.Directory + exportFilename := model.NewId() + "_export.zip" + + rd, wr := io.Pipe() + + errCh := make(chan *model.AppError, 1) + go func() { + defer close(errCh) + _, appErr := w.app.WriteFile(rd, filepath.Join(outPath, exportFilename)) + errCh <- appErr + }() + + appErr := w.app.BulkExport(wr, outPath, opts) + if err := wr.Close(); err != nil { + mlog.Warn("Worker: error closing writer") + } + if appErr != nil { + w.setJobError(job, appErr) + return + } + + if appErr := <-errCh; appErr != nil { + w.setJobError(job, appErr) + return + } + + mlog.Info("Worker: Job is complete", mlog.String("worker", w.name), mlog.String("job_id", job.Id)) + w.setJobSuccess(job) +} + +func (w *ExportProcessWorker) setJobSuccess(job *model.Job) { + if err := w.app.Srv().Jobs.SetJobSuccess(job); err != nil { + mlog.Error("Worker: Failed to set success for job", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) + w.setJobError(job, err) + } +} + +func (w *ExportProcessWorker) setJobError(job *model.Job, appError *model.AppError) { + if err := w.app.Srv().Jobs.SetJobError(job, appError); err != nil { + mlog.Error("Worker: Failed to set job error", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) + } +} diff --git a/jobs/import_process/worker.go b/jobs/import_process/worker.go index cd6fffb281..6ddf011bd5 100644 --- a/jobs/import_process/worker.go +++ b/jobs/import_process/worker.go @@ -165,7 +165,7 @@ func (w *ImportProcessWorker) doJob(job *model.Job) { } // do the actual import. - appErr, lineNumber := w.app.BulkImportWithPath(jsonFile, false, runtime.NumCPU(), dir) + appErr, lineNumber := w.app.BulkImportWithPath(jsonFile, false, runtime.NumCPU(), filepath.Join(dir, app.ExportDataDir)) if appErr != nil { job.Data["line_number"] = strconv.Itoa(lineNumber) w.setJobError(job, appErr) diff --git a/jobs/interfaces/export_process_interface.go b/jobs/interfaces/export_process_interface.go new file mode 100644 index 0000000000..f2f1502a53 --- /dev/null +++ b/jobs/interfaces/export_process_interface.go @@ -0,0 +1,12 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package interfaces + +import ( + "github.com/mattermost/mattermost-server/v5/model" +) + +type ExportProcessInterface interface { + MakeWorker() model.Worker +} diff --git a/jobs/jobs_watcher.go b/jobs/jobs_watcher.go index 2ca31e0096..ab5dc3ee30 100644 --- a/jobs/jobs_watcher.go +++ b/jobs/jobs_watcher.go @@ -163,6 +163,13 @@ func (watcher *Watcher) PollAndNotify() { default: } } + } else if job.Type == model.JOB_TYPE_EXPORT_PROCESS { + if watcher.workers.ExportProcess != nil { + select { + case watcher.workers.ExportProcess.JobChannel() <- *job: + default: + } + } } else if job.Type == model.JOB_TYPE_CLOUD { if watcher.workers.Cloud != nil { select { diff --git a/jobs/server.go b/jobs/server.go index 4b30a6a88e..5235fe3d9d 100644 --- a/jobs/server.go +++ b/jobs/server.go @@ -32,6 +32,7 @@ type JobServer struct { ActiveUsers tjobs.ActiveUsersJobInterface ImportProcess tjobs.ImportProcessInterface ImportDelete tjobs.ImportDeleteInterface + ExportProcess tjobs.ExportProcessInterface Cloud ejobs.CloudJobInterface } diff --git a/jobs/workers.go b/jobs/workers.go index acce58f8ac..bbb91c6659 100644 --- a/jobs/workers.go +++ b/jobs/workers.go @@ -29,6 +29,7 @@ type Workers struct { ActiveUsers model.Worker ImportProcess model.Worker ImportDelete model.Worker + ExportProcess model.Worker Cloud model.Worker listenerId string @@ -92,6 +93,10 @@ func (srv *JobServer) InitWorkers() *Workers { workers.ImportDelete = importDeleteInterface.MakeWorker() } + if exportProcessInterface := srv.ExportProcess; exportProcessInterface != nil { + workers.ExportProcess = exportProcessInterface.MakeWorker() + } + if cloudInterface := srv.Cloud; cloudInterface != nil { workers.Cloud = cloudInterface.MakeWorker() } @@ -155,6 +160,10 @@ func (workers *Workers) Start() *Workers { go workers.ImportDelete.Run() } + if workers.ExportProcess != nil { + go workers.ExportProcess.Run() + } + if workers.Cloud != nil { go workers.Cloud.Run() } @@ -276,6 +285,10 @@ func (workers *Workers) Stop() *Workers { workers.ImportDelete.Stop() } + if workers.ExportProcess != nil { + workers.ExportProcess.Stop() + } + if workers.Cloud != nil { workers.Cloud.Stop() } diff --git a/model/client4.go b/model/client4.go index 273f109595..808860c254 100644 --- a/model/client4.go +++ b/model/client4.go @@ -34,6 +34,7 @@ const ( HEADER_CLOUD_TOKEN = "X-Cloud-Token" HEADER_REQUESTED_WITH = "X-Requested-With" HEADER_REQUESTED_WITH_XML = "XMLHttpRequest" + HEADER_RANGE = "Range" STATUS = "status" STATUS_OK = "OK" STATUS_FAIL = "FAIL" @@ -550,6 +551,14 @@ func (c *Client4) GetImportsRoute() string { return "/imports" } +func (c *Client4) GetExportsRoute() string { + return "/exports" +} + +func (c *Client4) GetExportRoute(name string) string { + return fmt.Sprintf(c.GetExportsRoute()+"/%v", name) +} + func (c *Client4) DoApiGet(url string, etag string) (*http.Response, *AppError) { return c.DoApiRequest(http.MethodGet, c.ApiUrl+url, "", etag) } @@ -575,21 +584,25 @@ func (c *Client4) DoApiDelete(url string) (*http.Response, *AppError) { } func (c *Client4) DoApiRequest(method, url, data, etag string) (*http.Response, *AppError) { - return c.doApiRequestReader(method, url, strings.NewReader(data), etag) + return c.doApiRequestReader(method, url, strings.NewReader(data), map[string]string{HEADER_ETAG_CLIENT: etag}) +} + +func (c *Client4) DoApiRequestWithHeaders(method, url, data string, headers map[string]string) (*http.Response, *AppError) { + return c.doApiRequestReader(method, url, strings.NewReader(data), headers) } func (c *Client4) doApiRequestBytes(method, url string, data []byte, etag string) (*http.Response, *AppError) { - return c.doApiRequestReader(method, url, bytes.NewReader(data), etag) + return c.doApiRequestReader(method, url, bytes.NewReader(data), map[string]string{HEADER_ETAG_CLIENT: etag}) } -func (c *Client4) doApiRequestReader(method, url string, data io.Reader, etag string) (*http.Response, *AppError) { +func (c *Client4) doApiRequestReader(method, url string, data io.Reader, headers map[string]string) (*http.Response, *AppError) { rq, err := http.NewRequest(method, url, data) if err != nil { return nil, NewAppError(url, "model.client.connecting.app_error", nil, err.Error(), http.StatusBadRequest) } - if etag != "" { - rq.Header.Set(HEADER_ETAG_CLIENT, etag) + for k, v := range headers { + rq.Header.Set(k, v) } if c.AuthToken != "" { @@ -5668,7 +5681,7 @@ func (c *Client4) GetUploadsForUser(userId string) ([]*UploadSession, *Response) // a FileInfo object. func (c *Client4) UploadData(uploadId string, data io.Reader) (*FileInfo, *Response) { url := c.GetUploadRoute(uploadId) - r, err := c.doApiRequestReader("POST", c.ApiUrl+url, data, "") + r, err := c.doApiRequestReader("POST", c.ApiUrl+url, data, nil) if err != nil { return nil, BuildErrorResponse(r, err) } @@ -5816,6 +5829,43 @@ func (c *Client4) ListImports() ([]string, *Response) { return ArrayFromJson(r.Body), BuildResponse(r) } +func (c *Client4) ListExports() ([]string, *Response) { + r, err := c.DoApiGet(c.GetExportsRoute(), "") + if err != nil { + return nil, BuildErrorResponse(r, err) + } + defer closeBody(r) + return ArrayFromJson(r.Body), BuildResponse(r) +} + +func (c *Client4) DeleteExport(name string) (bool, *Response) { + r, err := c.DoApiDelete(c.GetExportRoute(name)) + if err != nil { + return false, BuildErrorResponse(r, err) + } + defer closeBody(r) + return CheckStatusOK(r), BuildResponse(r) +} + +func (c *Client4) DownloadExport(name string, wr io.Writer, offset int64) (int64, *Response) { + var headers map[string]string + if offset > 0 { + headers = map[string]string{ + HEADER_RANGE: fmt.Sprintf("bytes=%d-", offset), + } + } + r, appErr := c.DoApiRequestWithHeaders(http.MethodGet, c.ApiUrl+c.GetExportRoute(name), "", headers) + if appErr != nil { + return 0, BuildErrorResponse(r, appErr) + } + defer closeBody(r) + n, err := io.Copy(wr, r.Body) + if err != nil { + return n, BuildErrorResponse(r, NewAppError("DownloadExport", "model.client.copy.app_error", nil, err.Error(), r.StatusCode)) + } + return n, BuildResponse(r) +} + func (c *Client4) GetThreadMentionsForUserPerChannel(userId, teamId string) (map[string]int64, *Response) { url := c.GetUserThreadsRoute(userId, teamId) r, appErr := c.DoApiGet(url+"/mention_counts", "") diff --git a/model/config.go b/model/config.go index 650a41ace1..535fba9c53 100644 --- a/model/config.go +++ b/model/config.go @@ -122,6 +122,9 @@ const ( IMPORT_SETTINGS_DEFAULT_DIRECTORY = "./import" IMPORT_SETTINGS_DEFAULT_RETENTION_DAYS = 30 + EXPORT_SETTINGS_DEFAULT_DIRECTORY = "./export" + EXPORT_SETTINGS_DEFAULT_RETENTION_DAYS = 30 + EMAIL_SETTINGS_DEFAULT_FEEDBACK_ORGANIZATION = "" SUPPORT_SETTINGS_DEFAULT_TERMS_OF_SERVICE_LINK = "https://about.mattermost.com/default-terms/" @@ -2936,6 +2939,37 @@ func (s *ImportSettings) SetDefaults() { } } +// ExportSettings defines configuration settings for file exports. +type ExportSettings struct { + // The directory where to store the exported files. + Directory *string + // The number of days to retain the exported files before deleting them. + RetentionDays *int +} + +func (s *ExportSettings) isValid() *AppError { + if *s.Directory == "" { + return NewAppError("Config.IsValid", "model.config.is_valid.export.directory.app_error", nil, "", http.StatusBadRequest) + } + + if *s.RetentionDays <= 0 { + return NewAppError("Config.IsValid", "model.config.is_valid.export.retention_days_too_low.app_error", nil, "", http.StatusBadRequest) + } + + return nil +} + +// SetDefaults applies the default settings to the struct. +func (s *ExportSettings) SetDefaults() { + if s.Directory == nil || *s.Directory == "" { + s.Directory = NewString(EXPORT_SETTINGS_DEFAULT_DIRECTORY) + } + + if s.RetentionDays == nil { + s.RetentionDays = NewInt(EXPORT_SETTINGS_DEFAULT_RETENTION_DAYS) + } +} + type ConfigFunc func() *Config const ConfigAccessTagType = "access" @@ -3013,6 +3047,7 @@ type Config struct { CloudSettings CloudSettings FeatureFlags *FeatureFlags `json:",omitempty"` ImportSettings ImportSettings + ExportSettings ExportSettings } func (o *Config) Clone() *Config { @@ -3121,6 +3156,7 @@ func (o *Config) SetDefaults() { o.FeatureFlags.SetDefaults() } o.ImportSettings.SetDefaults() + o.ExportSettings.SetDefaults() } func (o *Config) IsValid() *AppError { diff --git a/model/config_test.go b/model/config_test.go index 449999cfaf..0257c3f82e 100644 --- a/model/config_test.go +++ b/model/config_test.go @@ -1439,3 +1439,31 @@ func TestConfigImportSettingsIsValid(t *testing.T) { require.NotNil(t, err) require.Equal(t, "model.config.is_valid.import.retention_days_too_low.app_error", err.Id) } + +func TestConfigExportSettingsDefaults(t *testing.T) { + cfg := Config{} + cfg.SetDefaults() + + require.Equal(t, "./export", *cfg.ExportSettings.Directory) + require.Equal(t, 30, *cfg.ExportSettings.RetentionDays) +} + +func TestConfigExportSettingsIsValid(t *testing.T) { + cfg := Config{} + cfg.SetDefaults() + + err := cfg.ExportSettings.isValid() + require.Nil(t, err) + + *cfg.ExportSettings.Directory = "" + err = cfg.ExportSettings.isValid() + require.NotNil(t, err) + require.Equal(t, "model.config.is_valid.export.directory.app_error", err.Id) + + cfg.SetDefaults() + + *cfg.ExportSettings.RetentionDays = 0 + err = cfg.ExportSettings.isValid() + require.NotNil(t, err) + require.Equal(t, "model.config.is_valid.export.retention_days_too_low.app_error", err.Id) +} diff --git a/model/job.go b/model/job.go index b2cdaba122..31312add7a 100644 --- a/model/job.go +++ b/model/job.go @@ -24,6 +24,7 @@ const ( JOB_TYPE_ACTIVE_USERS = "active_users" JOB_TYPE_IMPORT_PROCESS = "import_process" JOB_TYPE_IMPORT_DELETE = "import_delete" + JOB_TYPE_EXPORT_PROCESS = "export_process" JOB_TYPE_CLOUD = "cloud" JOB_STATUS_PENDING = "pending" @@ -70,6 +71,7 @@ func (j *Job) IsValid() *AppError { case JOB_TYPE_ACTIVE_USERS: case JOB_TYPE_IMPORT_PROCESS: case JOB_TYPE_IMPORT_DELETE: + case JOB_TYPE_EXPORT_PROCESS: case JOB_TYPE_CLOUD: default: return NewAppError("Job.IsValid", "model.job.is_valid.type.app_error", nil, "id="+j.Id, http.StatusBadRequest) diff --git a/utils/archive.go b/utils/archive.go index cf8d740e4c..971f977423 100644 --- a/utils/archive.go +++ b/utils/archive.go @@ -38,12 +38,16 @@ func UnzipToPath(zipFile io.ReaderAt, size int64, outPath string) ([]string, err path := filepath.Join(outPath, filePath) paths[i] = path if f.FileInfo().IsDir() { - if err := os.Mkdir(path, 0744); err != nil { + if err := os.Mkdir(path, 0700); err != nil { return nil, fmt.Errorf("failed to create directory: %w", err) } continue } - + if _, err := os.Stat(filepath.Dir(path)); os.IsNotExist(err) { + if err = os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return nil, fmt.Errorf("failed to create directory: %w", err) + } + } outFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0600) if err != nil { return nil, fmt.Errorf("failed to create file: %w", err) diff --git a/web/params.go b/web/params.go index ed2ffcfefd..8ab509fdff 100644 --- a/web/params.go +++ b/web/params.go @@ -83,6 +83,7 @@ type Params struct { FilterParentTeamPermitted bool CategoryId string WarnMetricId string + ExportName string // Cloud InvoiceId string @@ -346,5 +347,9 @@ func ParamsFromRequest(r *http.Request) *Params { params.WarnMetricId = val } + if val, ok := props["export_name"]; ok { + params.ExportName = val + } + return params }