[MM-31247] Add support for compressed export files with attachments (#16614)

* Include filepaths for post attachments

* Cleanup

* Enable exporting file attachments

* Fix file import

* Enable zip export

* Support creating missing directories when unzipping

* Add test

* Add translations

* Export direct channel posts attachments

* Fix returned values order

Remove pointer to slice in return

* [MM-31597] Implement export process job (#16626)

* Implement export process job

* Add translations

* Remove unused value

* [MM-31249] Add /exports API endpoint (#16633)

* Implement API endpoints to list, download and delete export files

* Add endpoint for single resource

* Update i18n/en.json

Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com>

* Update i18n/en.json

Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com>

Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com>

* Fix var name

Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com>

api4/api.go

@@ -125,6 +125,8 @@ type Routes struct {
 	Cloud *mux.Router // 'api/v4/cloud'
 
 	Imports *mux.Router // 'api/v4/imports'
+	Exports *mux.Router // 'api/v4/exports'
+	Export  *mux.Router // 'api/v4/exports/{export_name:.+\\.zip}'
 }
 
 type API struct {
@@ -238,6 +240,8 @@ func Init(configservice configservice.ConfigService, globalOptionsFunc app.AppOp
 	api.BaseRoutes.Cloud = api.BaseRoutes.ApiRoot.PathPrefix("/cloud").Subrouter()
 
 	api.BaseRoutes.Imports = api.BaseRoutes.ApiRoot.PathPrefix("/imports").Subrouter()
+	api.BaseRoutes.Exports = api.BaseRoutes.ApiRoot.PathPrefix("/exports").Subrouter()
+	api.BaseRoutes.Export = api.BaseRoutes.Exports.PathPrefix("/{export_name:.+\\.zip}").Subrouter()
 
 	api.InitUser()
 	api.InitBot()
@@ -276,6 +280,7 @@ func Init(configservice configservice.ConfigService, globalOptionsFunc app.AppOp
 	api.InitAction()
 	api.InitCloud()
 	api.InitImport()
+	api.InitExport()
 
 	root.Handle("/api/v4/{anything:.*}", http.HandlerFunc(api.Handle404))
 
@@ -344,6 +349,8 @@ func InitLocal(configservice configservice.ConfigService, globalOptionsFunc app.
 	api.BaseRoutes.Upload = api.BaseRoutes.Uploads.PathPrefix("/{upload_id:[A-Za-z0-9]+}").Subrouter()
 
 	api.BaseRoutes.Imports = api.BaseRoutes.ApiRoot.PathPrefix("/imports").Subrouter()
+	api.BaseRoutes.Exports = api.BaseRoutes.ApiRoot.PathPrefix("/exports").Subrouter()
+	api.BaseRoutes.Export = api.BaseRoutes.Exports.PathPrefix("/{export_name:.+\\.zip}").Subrouter()
 
 	api.BaseRoutes.Jobs = api.BaseRoutes.ApiRoot.PathPrefix("/jobs").Subrouter()
 
@@ -363,6 +370,7 @@ func InitLocal(configservice configservice.ConfigService, globalOptionsFunc app.
 	api.InitRoleLocal()
 	api.InitUploadLocal()
 	api.InitImportLocal()
+	api.InitExportLocal()
 	api.InitJobLocal()
 
 	root.Handle("/api/v4/{anything:.*}", http.HandlerFunc(api.Handle404))

api4/export.go

@@ -0,0 +1,86 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package api4
+
+import (
+	"encoding/json"
+	"net/http"
+	"path/filepath"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v5/audit"
+	"github.com/mattermost/mattermost-server/v5/model"
+)
+
+func (api *API) InitExport() {
+	api.BaseRoutes.Exports.Handle("", api.ApiSessionRequired(listExports)).Methods("GET")
+	api.BaseRoutes.Export.Handle("", api.ApiSessionRequired(deleteExport)).Methods("DELETE")
+	api.BaseRoutes.Export.Handle("", api.ApiSessionRequired(downloadExport)).Methods("GET")
+}
+
+func listExports(c *Context, w http.ResponseWriter, r *http.Request) {
+	if !c.IsSystemAdmin() {
+		c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM)
+		return
+	}
+
+	exports, appErr := c.App.ListExports()
+	if appErr != nil {
+		c.Err = appErr
+		return
+	}
+
+	data, err := json.Marshal(exports)
+	if err != nil {
+		c.Err = model.NewAppError("listImports", "app.export.marshal.app_error", nil, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	w.Write(data)
+}
+
+func deleteExport(c *Context, w http.ResponseWriter, r *http.Request) {
+	auditRec := c.MakeAuditRecord("deleteExport", audit.Fail)
+	defer c.LogAuditRec(auditRec)
+	auditRec.AddMeta("export_name", c.Params.ExportName)
+
+	if !c.IsSystemAdmin() {
+		c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM)
+		return
+	}
+
+	if err := c.App.DeleteExport(c.Params.ExportName); err != nil {
+		c.Err = err
+		return
+	}
+
+	auditRec.Success()
+	ReturnStatusOK(w)
+}
+
+func downloadExport(c *Context, w http.ResponseWriter, r *http.Request) {
+	if !c.IsSystemAdmin() {
+		c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM)
+		return
+	}
+
+	filePath := filepath.Join(*c.App.Config().ExportSettings.Directory, c.Params.ExportName)
+	if ok, err := c.App.FileExists(filePath); err != nil {
+		c.Err = err
+		return
+	} else if !ok {
+		c.Err = model.NewAppError("downloadExport", "api.export.export_not_found.app_error", nil, "", http.StatusNotFound)
+		return
+	}
+
+	file, err := c.App.FileReader(filePath)
+	if err != nil {
+		c.Err = err
+		return
+	}
+	defer file.Close()
+
+	w.Header().Set("Content-Type", "application/zip")
+	http.ServeContent(w, r, c.Params.ExportName, time.Time{}, file)
+}

api4/export_local.go

@@ -0,0 +1,10 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package api4
+
+func (api *API) InitExportLocal() {
+	api.BaseRoutes.Exports.Handle("", api.ApiLocal(listExports)).Methods("GET")
+	api.BaseRoutes.Export.Handle("", api.ApiLocal(deleteExport)).Methods("DELETE")
+	api.BaseRoutes.Export.Handle("", api.ApiLocal(downloadExport)).Methods("GET")
+}

api4/export_test.go

@@ -0,0 +1,213 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package api4
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/utils/fileutils"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestListExports(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	t.Run("no permissions", func(t *testing.T) {
+		exports, resp := th.Client.ListExports()
+		require.Error(t, resp.Error)
+		require.Equal(t, "api.context.permissions.app_error", resp.Error.Id)
+		require.Nil(t, exports)
+	})
+
+	th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) {
+		exports, resp := c.ListExports()
+		require.Nil(t, resp.Error)
+		require.Empty(t, exports)
+	}, "no exports")
+
+	dataDir, found := fileutils.FindDir("data")
+	require.True(t, found)
+
+	th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) {
+		exportDir := filepath.Join(dataDir, *th.App.Config().ExportSettings.Directory)
+		err := os.Mkdir(exportDir, 0700)
+		require.Nil(t, err)
+		defer os.RemoveAll(exportDir)
+
+		f, err := os.Create(filepath.Join(exportDir, "export.zip"))
+		require.Nil(t, err)
+		f.Close()
+
+		exports, resp := c.ListExports()
+		require.Nil(t, resp.Error)
+		require.Len(t, exports, 1)
+		require.Equal(t, exports[0], "export.zip")
+	}, "expected exports")
+
+	th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) {
+		value := *th.App.Config().ExportSettings.Directory
+		th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ExportSettings.Directory = value + "new" })
+		defer th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ExportSettings.Directory = value })
+
+		exportDir := filepath.Join(dataDir, value+"new")
+		err := os.Mkdir(exportDir, 0700)
+		require.Nil(t, err)
+		defer os.RemoveAll(exportDir)
+
+		exports, resp := c.ListExports()
+		require.Nil(t, resp.Error)
+		require.Empty(t, exports)
+
+		f, err := os.Create(filepath.Join(exportDir, "export.zip"))
+		require.Nil(t, err)
+		f.Close()
+
+		exports, resp = c.ListExports()
+		require.Nil(t, resp.Error)
+		require.Len(t, exports, 1)
+		require.Equal(t, "export.zip", exports[0])
+	}, "change export directory")
+}
+
+func TestDeleteExport(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	t.Run("no permissions", func(t *testing.T) {
+		ok, resp := th.Client.DeleteExport("export.zip")
+		require.Error(t, resp.Error)
+		require.Equal(t, "api.context.permissions.app_error", resp.Error.Id)
+		require.False(t, ok)
+	})
+
+	dataDir, found := fileutils.FindDir("data")
+	require.True(t, found)
+	exportDir := filepath.Join(dataDir, *th.App.Config().ExportSettings.Directory)
+
+	th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) {
+		err := os.Mkdir(exportDir, 0700)
+		require.Nil(t, err)
+		defer os.RemoveAll(exportDir)
+		exportName := "export.zip"
+		f, err := os.Create(filepath.Join(exportDir, exportName))
+		require.Nil(t, err)
+		f.Close()
+
+		exports, resp := c.ListExports()
+		require.Nil(t, resp.Error)
+		require.Len(t, exports, 1)
+		require.Equal(t, exports[0], exportName)
+
+		ok, resp := c.DeleteExport(exportName)
+		require.Nil(t, resp.Error)
+		require.True(t, ok)
+
+		exports, resp = c.ListExports()
+		require.Nil(t, resp.Error)
+		require.Empty(t, exports)
+
+		// verify idempotence
+		ok, resp = c.DeleteExport(exportName)
+		require.Nil(t, resp.Error)
+		require.True(t, ok)
+	}, "successfully delete export")
+}
+
+func TestDownloadExport(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	t.Run("no permissions", func(t *testing.T) {
+		var buf bytes.Buffer
+		n, resp := th.Client.DownloadExport("export.zip", &buf, 0)
+		require.Error(t, resp.Error)
+		require.Equal(t, "api.context.permissions.app_error", resp.Error.Id)
+		require.Zero(t, n)
+	})
+
+	dataDir, found := fileutils.FindDir("data")
+	require.True(t, found)
+	exportDir := filepath.Join(dataDir, *th.App.Config().ExportSettings.Directory)
+
+	th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) {
+		var buf bytes.Buffer
+		n, resp := c.DownloadExport("export.zip", &buf, 0)
+		require.Error(t, resp.Error)
+		require.Equal(t, "api.export.export_not_found.app_error", resp.Error.Id)
+		require.Zero(t, n)
+	}, "not found")
+
+	th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) {
+		err := os.Mkdir(exportDir, 0700)
+		require.Nil(t, err)
+		defer os.RemoveAll(exportDir)
+
+		data := randomBytes(t, 1024*1024)
+		var buf bytes.Buffer
+		exportName := "export.zip"
+		err = ioutil.WriteFile(filepath.Join(exportDir, exportName), data, 0600)
+		require.Nil(t, err)
+
+		n, resp := c.DownloadExport(exportName, &buf, 0)
+		require.Nil(t, resp.Error)
+		require.Equal(t, len(data), int(n))
+		require.Equal(t, data, buf.Bytes())
+	}, "full download")
+
+	th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) {
+		err := os.Mkdir(exportDir, 0700)
+		require.Nil(t, err)
+		defer os.RemoveAll(exportDir)
+
+		data := randomBytes(t, 1024*1024)
+		var buf bytes.Buffer
+		exportName := "export.zip"
+		err = ioutil.WriteFile(filepath.Join(exportDir, exportName), data, 0600)
+		require.Nil(t, err)
+
+		offset := 1024 * 512
+		n, resp := c.DownloadExport(exportName, &buf, int64(offset))
+		require.Nil(t, resp.Error)
+		require.Equal(t, len(data)-offset, int(n))
+		require.Equal(t, data[offset:], buf.Bytes())
+	}, "download with offset")
+}
+
+func BenchmarkDownloadExport(b *testing.B) {
+	th := Setup(b)
+	defer th.TearDown()
+
+	dataDir, found := fileutils.FindDir("data")
+	require.True(b, found)
+	exportDir := filepath.Join(dataDir, *th.App.Config().ExportSettings.Directory)
+
+	err := os.Mkdir(exportDir, 0700)
+	require.Nil(b, err)
+	defer os.RemoveAll(exportDir)
+
+	exportName := "export.zip"
+	f, err := os.Create(filepath.Join(exportDir, exportName))
+	require.Nil(b, err)
+	f.Close()
+
+	err = os.Truncate(filepath.Join(exportDir, exportName), 1024*1024*1024)
+	require.Nil(b, err)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		outFilePath := filepath.Join(dataDir, fmt.Sprintf("export%d.zip", i))
+		outFile, _ := os.Create(outFilePath)
+		th.SystemAdminClient.DownloadExport(exportName, outFile, 0)
+		outFile.Close()
+		os.Remove(outFilePath)
+	}
+}

app/app.go

@@ -123,6 +123,10 @@ func (a *App) initJobs() {
 		a.srv.Jobs.ImportDelete = jobsImportDeleteInterface(a)
 	}
 
+	if jobsExportProcessInterface != nil {
+		a.srv.Jobs.ExportProcess = jobsExportProcessInterface(a)
+	}
+
 	if jobsActiveUsersInterface != nil {
 		a.srv.Jobs.ActiveUsers = jobsActiveUsersInterface(a)
 	}

app/app_iface.go

@@ -388,7 +388,7 @@ type AppIface interface {
 	BuildPostReactions(postId string) (*[]ReactionImportData, *model.AppError)
 	BuildPushNotificationMessage(contentsConfig string, post *model.Post, user *model.User, channel *model.Channel, channelName string, senderName string, explicitMention bool, channelWideMention bool, replyToThreadType string) (*model.PushNotification, *model.AppError)
 	BuildSamlMetadataObject(idpMetadata []byte) (*model.SamlMetadataResponse, *model.AppError)
-	BulkExport(writer io.Writer, file string, pathToEmojiDir string, dirNameToExportEmoji string) *model.AppError
+	BulkExport(writer io.Writer, outPath string, opts BulkExportOpts) *model.AppError
 	BulkImport(fileReader io.Reader, dryRun bool, workers int) (*model.AppError, int)
 	BulkImportWithPath(fileReader io.Reader, dryRun bool, workers int, importPath string) (*model.AppError, int)
 	CancelJob(jobId string) *model.AppError
@@ -466,6 +466,7 @@ type AppIface interface {
 	DeleteCommand(commandID string) *model.AppError
 	DeleteEmoji(emoji *model.Emoji) *model.AppError
 	DeleteEphemeralPost(userID, postId string)
+	DeleteExport(name string) *model.AppError
 	DeleteFlaggedPosts(postId string)
 	DeleteGroup(groupID string) (*model.Group, *model.AppError)
 	DeleteGroupMember(groupID string, userID string) (*model.GroupMember, *model.AppError)
@@ -782,6 +783,7 @@ type AppIface interface {
 	LimitedClientConfig() map[string]string
 	ListAllCommands(teamID string, T goi18n.TranslateFunc) ([]*model.Command, *model.AppError)
 	ListDirectory(path string) ([]string, *model.AppError)
+	ListExports() ([]string, *model.AppError)
 	ListImports() ([]string, *model.AppError)
 	ListPluginKeys(pluginId string, page, perPage int) ([]string, *model.AppError)
 	ListTeamCommands(teamID string) ([]*model.Command, *model.AppError)

app/enterprise.go

@@ -120,6 +120,12 @@ func RegisterJobsImportDeleteInterface(f func(*App) tjobs.ImportDeleteInterface)
 	jobsImportDeleteInterface = f
 }
 
+var jobsExportProcessInterface func(*App) tjobs.ExportProcessInterface
+
+func RegisterJobsExportProcessInterface(f func(*App) tjobs.ExportProcessInterface) {
+	jobsExportProcessInterface = f
+}
+
 var productNoticesJobInterface func(*App) tjobs.ProductNoticesJobInterface
 
 func RegisterProductNoticesJobInterface(f func(*App) tjobs.ProductNoticesJobInterface) {

app/export.go

@@ -4,6 +4,7 @@
 package app
 
 import (
+	"archive/zip"
 	"encoding/json"
 	"io"
 	"net/http"
@@ -18,6 +19,15 @@ import (
 	"github.com/mattermost/mattermost-server/v5/store"
 )
 
+type BulkExportOpts struct {
+	IncludeAttachments bool
+	CreateArchive      bool
+}
+
+// ExportDataDir is the name of the directory were to store additional data
+// included with the export (e.g. file attachments).
+const ExportDataDir = "data"
+
 // We use this map to identify the exportable preferences.
 // Here we link the preference category and name, to the name of the relevant field in the import struct.
 var exportablePreferences = map[ComparablePreference]string{{
@@ -53,7 +63,19 @@ var exportablePreferences = map[ComparablePreference]string{{
 }: "EmailInterval",
 }
 
-func (a *App) BulkExport(writer io.Writer, file string, pathToEmojiDir string, dirNameToExportEmoji string) *model.AppError {
+func (a *App) BulkExport(writer io.Writer, outPath string, opts BulkExportOpts) *model.AppError {
+	var zipWr *zip.Writer
+	if opts.CreateArchive {
+		var err error
+		zipWr = zip.NewWriter(writer)
+		defer zipWr.Close()
+		writer, err = zipWr.Create("import.jsonl")
+		if err != nil {
+			return model.NewAppError("BulkExport", "app.export.zip_create.error",
+				nil, "err="+err.Error(), http.StatusInternalServerError)
+		}
+	}
+
 	mlog.Info("Bulk export: exporting version")
 	if err := a.exportVersion(writer); err != nil {
 		return err
@@ -75,25 +97,47 @@ func (a *App) BulkExport(writer io.Writer, file string, pathToEmojiDir string, d
 	}
 
 	mlog.Info("Bulk export: exporting posts")
-	if err := a.exportAllPosts(writer); err != nil {
+	attachments, err := a.exportAllPosts(writer, opts.IncludeAttachments)
+	if err != nil {
 		return err
 	}
 
 	mlog.Info("Bulk export: exporting emoji")
-	if err := a.exportCustomEmoji(writer, file, pathToEmojiDir, dirNameToExportEmoji); err != nil {
+	emojiPaths, err := a.exportCustomEmoji(writer, outPath, "exported_emoji", !opts.CreateArchive)
+	if err != nil {
 		return err
 	}
 
 	mlog.Info("Bulk export: exporting direct channels")
-	if err := a.exportAllDirectChannels(writer); err != nil {
+	if err = a.exportAllDirectChannels(writer); err != nil {
 		return err
 	}
 
 	mlog.Info("Bulk export: exporting direct posts")
-	if err := a.exportAllDirectPosts(writer); err != nil {
+	directAttachments, err := a.exportAllDirectPosts(writer, opts.IncludeAttachments)
+	if err != nil {
 		return err
 	}
 
+	if opts.IncludeAttachments {
+		mlog.Info("Bulk export: exporting file attachments")
+		for _, attachment := range attachments {
+			if err := a.exportFile(outPath, *attachment.Path, zipWr); err != nil {
+				return err
+			}
+		}
+		for _, attachment := range directAttachments {
+			if err := a.exportFile(outPath, *attachment.Path, zipWr); err != nil {
+				return err
+			}
+		}
+		for _, emojiPath := range emojiPaths {
+			if err := a.exportFile(outPath, emojiPath, zipWr); err != nil {
+				return err
+			}
+		}
+	}
+
 	return nil
 }
 
@@ -334,17 +378,18 @@ func (a *App) buildUserNotifyProps(notifyProps model.StringMap) *UserNotifyProps
 	}
 }
 
-func (a *App) exportAllPosts(writer io.Writer) *model.AppError {
+func (a *App) exportAllPosts(writer io.Writer, withAttachments bool) ([]AttachmentImportData, *model.AppError) {
+	var attachments []AttachmentImportData
 	afterId := strings.Repeat("0", 26)
 
 	for {
 		posts, nErr := a.Srv().Store.Post().GetParentsForExportAfter(1000, afterId)
 		if nErr != nil {
-			return model.NewAppError("exportAllPosts", "app.post.get_posts.app_error", nil, nErr.Error(), http.StatusInternalServerError)
+			return nil, model.NewAppError("exportAllPosts", "app.post.get_posts.app_error", nil, nErr.Error(), http.StatusInternalServerError)
 		}
 
 		if len(posts) == 0 {
-			return nil
+			return attachments, nil
 		}
 
 		for _, post := range posts {
@@ -357,33 +402,50 @@ func (a *App) exportAllPosts(writer io.Writer) *model.AppError {
 
 			postLine := ImportLineForPost(post)
 
-			var err *model.AppError
-			postLine.Post.Replies, err = a.buildPostReplies(post.Id)
+			replies, replyAttachments, err := a.buildPostReplies(post.Id, withAttachments)
 			if err != nil {
-				return err
+				return nil, err
 			}
 
+			if withAttachments && len(replyAttachments) > 0 {
+				attachments = append(attachments, replyAttachments...)
+			}
+
+			postLine.Post.Replies = &replies
 			postLine.Post.Reactions = &[]ReactionImportData{}
 			if post.HasReactions {
 				postLine.Post.Reactions, err = a.BuildPostReactions(post.Id)
 				if err != nil {
-					return err
+					return nil, err
+				}
+			}
+
+			if len(post.FileIds) > 0 {
+				postAttachments, err := a.buildPostAttachments(post.Id)
+				if err != nil {
+					return nil, err
+				}
+				postLine.Post.Attachments = &postAttachments
+
+				if withAttachments && len(postAttachments) > 0 {
+					attachments = append(attachments, postAttachments...)
 				}
 			}
 
 			if err := a.exportWriteLine(writer, postLine); err != nil {
-				return err
+				return nil, err
 			}
 		}
 	}
 }
 
-func (a *App) buildPostReplies(postId string) (*[]ReplyImportData, *model.AppError) {
+func (a *App) buildPostReplies(postId string, withAttachments bool) ([]ReplyImportData, []AttachmentImportData, *model.AppError) {
 	var replies []ReplyImportData
+	var attachments []AttachmentImportData
 
 	replyPosts, nErr := a.Srv().Store.Post().GetRepliesForExport(postId)
 	if nErr != nil {
-		return nil, model.NewAppError("buildPostReplies", "app.post.get_posts.app_error", nil, nErr.Error(), http.StatusInternalServerError)
+		return nil, nil, model.NewAppError("buildPostReplies", "app.post.get_posts.app_error", nil, nErr.Error(), http.StatusInternalServerError)
 	}
 
 	for _, reply := range replyPosts {
@@ -392,13 +454,24 @@ func (a *App) buildPostReplies(postId string) (*[]ReplyImportData, *model.AppErr
 			var appErr *model.AppError
 			replyImportObject.Reactions, appErr = a.BuildPostReactions(reply.Id)
 			if appErr != nil {
-				return nil, appErr
+				return nil, nil, appErr
 			}
 		}
+		if len(reply.FileIds) > 0 {
+			postAttachments, appErr := a.buildPostAttachments(reply.Id)
+			if appErr != nil {
+				return nil, nil, appErr
+			}
+			replyImportObject.Attachments = &attachments
+			if withAttachments && len(postAttachments) > 0 {
+				attachments = append(attachments, postAttachments...)
+			}
+		}
+
 		replies = append(replies, *replyImportObject)
 	}
 
-	return &replies, nil
+	return replies, attachments, nil
 }
 
 func (a *App) BuildPostReactions(postId string) (*[]ReactionImportData, *model.AppError) {
@@ -426,13 +499,28 @@ func (a *App) BuildPostReactions(postId string) (*[]ReactionImportData, *model.A
 
 }
 
-func (a *App) exportCustomEmoji(writer io.Writer, file string, pathToEmojiDir string, dirNameToExportEmoji string) *model.AppError {
+func (a *App) buildPostAttachments(postId string) ([]AttachmentImportData, *model.AppError) {
+	infos, nErr := a.Srv().Store.FileInfo().GetForPost(postId, false, false, false)
+	if nErr != nil {
+		return nil, model.NewAppError("buildPostAttachments", "app.file_info.get_for_post.app_error", nil, nErr.Error(), http.StatusInternalServerError)
+	}
+
+	attachments := make([]AttachmentImportData, 0, len(infos))
+	for _, info := range infos {
+		attachments = append(attachments, AttachmentImportData{Path: &info.Path})
+	}
+
+	return attachments, nil
+}
+
+func (a *App) exportCustomEmoji(writer io.Writer, outPath, exportDir string, exportFiles bool) ([]string, *model.AppError) {
+	var emojiPaths []string
 	pageNumber := 0
 	for {
 		customEmojiList, err := a.GetEmojiList(pageNumber, 100, model.EMOJI_SORT_BY_NAME)
 
 		if err != nil {
-			return err
+			return nil, err
 		}
 
 		if len(customEmojiList) == 0 {
@@ -441,42 +529,35 @@ func (a *App) exportCustomEmoji(writer io.Writer, file string, pathToEmojiDir st
 
 		pageNumber++
 
-		pathToDir := a.createDirForEmoji(file, dirNameToExportEmoji)
+		emojiPath := filepath.Join(*a.Config().FileSettings.Directory, "emoji")
+		pathToDir := filepath.Join(outPath, exportDir)
+		if exportFiles {
+			if _, err := os.Stat(pathToDir); os.IsNotExist(err) {
+				os.Mkdir(pathToDir, os.ModePerm)
+			}
+		}
 
 		for _, emoji := range customEmojiList {
-			emojiImagePath := pathToEmojiDir + emoji.Id + "/image"
-			err := a.copyEmojiImages(emoji.Id, emojiImagePath, pathToDir)
-			if err != nil {
-				return model.NewAppError("BulkExport", "app.export.export_custom_emoji.copy_emoji_images.error", nil, "err="+err.Error(), http.StatusBadRequest)
+			emojiImagePath := filepath.Join(emojiPath, emoji.Id, "image")
+			filePath := filepath.Join(exportDir, emoji.Id, "image")
+			if exportFiles {
+				err := a.copyEmojiImages(emoji.Id, emojiImagePath, pathToDir)
+				if err != nil {
+					return nil, model.NewAppError("BulkExport", "app.export.export_custom_emoji.copy_emoji_images.error", nil, "err="+err.Error(), http.StatusBadRequest)
+				}
+			} else {
+				filePath = filepath.Join("emoji", emoji.Id, "image")
+				emojiPaths = append(emojiPaths, filePath)
 			}
 
-			filePath := dirNameToExportEmoji + "/" + emoji.Id + "/image"
-
 			emojiImportObject := ImportLineFromEmoji(emoji, filePath)
-
 			if err := a.exportWriteLine(writer, emojiImportObject); err != nil {
-				return err
+				return nil, err
 			}
 		}
 	}
 
-	return nil
-}
-
-// Creates directory named 'exported_emoji' to copy the emoji files
-// Directory and the file specified by admin share the same path
-func (a *App) createDirForEmoji(file string, dirName string) string {
-	pathToFile, _ := filepath.Abs(file)
-	pathSlice := strings.Split(pathToFile, "/")
-	if len(pathSlice) > 0 {
-		pathSlice = pathSlice[:len(pathSlice)-1]
-	}
-	pathToDir := strings.Join(pathSlice, "/") + "/" + dirName
-
-	if _, err := os.Stat(pathToDir); os.IsNotExist(err) {
-		os.Mkdir(pathToDir, os.ModePerm)
-	}
-	return pathToDir
+	return emojiPaths, nil
 }
 
 // Copies emoji files from 'data/emoji' dir to 'exported_emoji' dir
@@ -543,12 +624,13 @@ func (a *App) exportAllDirectChannels(writer io.Writer) *model.AppError {
 	return nil
 }
 
-func (a *App) exportAllDirectPosts(writer io.Writer) *model.AppError {
+func (a *App) exportAllDirectPosts(writer io.Writer, withAttachments bool) ([]AttachmentImportData, *model.AppError) {
+	var attachments []AttachmentImportData
 	afterId := strings.Repeat("0", 26)
 	for {
 		posts, err := a.Srv().Store.Post().GetDirectPostParentsForExportAfter(1000, afterId)
 		if err != nil {
-			return model.NewAppError("exportAllDirectPosts", "app.post.get_direct_posts.app_error", nil, err.Error(), http.StatusInternalServerError)
+			return nil, model.NewAppError("exportAllDirectPosts", "app.post.get_direct_posts.app_error", nil, err.Error(), http.StatusInternalServerError)
 		}
 
 		if len(posts) == 0 {
@@ -563,18 +645,106 @@ func (a *App) exportAllDirectPosts(writer io.Writer) *model.AppError {
 				continue
 			}
 
+			// Handle attachments.
+			var postAttachments []AttachmentImportData
+			var err *model.AppError
+			if len(post.FileIds) > 0 {
+				postAttachments, err = a.buildPostAttachments(post.Id)
+				if err != nil {
+					return nil, err
+				}
+
+				if withAttachments && len(postAttachments) > 0 {
+					attachments = append(attachments, postAttachments...)
+				}
+			}
+
 			// Do the Replies.
-			replies, err := a.buildPostReplies(post.Id)
+			replies, replyAttachments, err := a.buildPostReplies(post.Id, withAttachments)
 			if err != nil {
-				return err
+				return nil, err
+			}
+
+			if withAttachments && len(replyAttachments) > 0 {
+				attachments = append(attachments, replyAttachments...)
 			}
 
 			postLine := ImportLineForDirectPost(post)
-			postLine.DirectPost.Replies = replies
+			postLine.DirectPost.Replies = &replies
+			if len(postAttachments) > 0 {
+				postLine.DirectPost.Attachments = &postAttachments
+			}
 			if err := a.exportWriteLine(writer, postLine); err != nil {
-				return err
+				return nil, err
 			}
 		}
 	}
+	return attachments, nil
+}
+
+func (a *App) exportFile(outPath, filePath string, zipWr *zip.Writer) *model.AppError {
+	var wr io.Writer
+	var err error
+	rd, appErr := a.FileReader(filePath)
+	if appErr != nil {
+		return appErr
+	}
+	defer rd.Close()
+
+	if zipWr != nil {
+		wr, err = zipWr.CreateHeader(&zip.FileHeader{
+			Name:   filepath.Join(ExportDataDir, filePath),
+			Method: zip.Store,
+		})
+		if err != nil {
+			return model.NewAppError("exportFileAttachment", "app.export.export_attachment.zip_create_header.error",
+				nil, "err="+err.Error(), http.StatusInternalServerError)
+		}
+	} else {
+		filePath = filepath.Join(outPath, ExportDataDir, filePath)
+		if err = os.MkdirAll(filepath.Dir(filePath), 0700); err != nil {
+			return model.NewAppError("exportFileAttachment", "app.export.export_attachment.mkdirall.error",
+				nil, "err="+err.Error(), http.StatusInternalServerError)
+		}
+
+		wr, err = os.Create(filePath)
+		if err != nil {
+			return model.NewAppError("exportFileAttachment", "app.export.export_attachment.create_file.error",
+				nil, "err="+err.Error(), http.StatusInternalServerError)
+		}
+		defer wr.(*os.File).Close()
+	}
+
+	if _, err := io.Copy(wr, rd); err != nil {
+		return model.NewAppError("exportFileAttachment", "app.export.export_attachment.copy_file.error",
+			nil, "err="+err.Error(), http.StatusInternalServerError)
+	}
+
 	return nil
 }
+
+func (a *App) ListExports() ([]string, *model.AppError) {
+	exports, appErr := a.ListDirectory(*a.Config().ExportSettings.Directory)
+	if appErr != nil {
+		return nil, appErr
+	}
+
+	results := make([]string, len(exports))
+	for i := range exports {
+		results[i] = filepath.Base(exports[i])
+	}
+
+	return results, nil
+}
+
+func (a *App) DeleteExport(name string) *model.AppError {
+	filePath := filepath.Join(*a.Config().ExportSettings.Directory, name)
+
+	if ok, err := a.FileExists(filePath); err != nil {
+		return err
+	} else if !ok {
+		return nil
+	}
+
+	return a.RemoveFile(filePath)
+}

app/export_test.go

@@ -5,7 +5,9 @@ package app
 
 import (
 	"bytes"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"sort"
 	"testing"
 
@@ -13,6 +15,8 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/utils"
+	"github.com/mattermost/mattermost-server/v5/utils/fileutils"
 )
 
 func TestReactionsOfPost(t *testing.T) {
@@ -111,16 +115,6 @@ func TestExportUserChannels(t *testing.T) {
 	}
 }
 
-func TestDirCreationForEmoji(t *testing.T) {
-	th := SetupWithStoreMock(t)
-	defer th.TearDown()
-
-	pathToDir := th.App.createDirForEmoji("test.json", "exported_emoji_test")
-	defer os.Remove(pathToDir)
-	_, err := os.Stat(pathToDir)
-	require.False(t, os.IsNotExist(err), "Directory exported_emoji_test should exist")
-}
-
 func TestCopyEmojiImages(t *testing.T) {
 	th := SetupWithStoreMock(t)
 	defer th.TearDown()
@@ -164,11 +158,13 @@ func TestExportCustomEmoji(t *testing.T) {
 	require.Nil(t, err)
 	defer os.Remove(filePath)
 
-	pathToEmojiDir := "../data/emoji/"
 	dirNameToExportEmoji := "exported_emoji_test"
 	defer os.RemoveAll("../" + dirNameToExportEmoji)
 
-	err = th.App.exportCustomEmoji(fileWriter, filePath, pathToEmojiDir, dirNameToExportEmoji)
+	outPath, err := filepath.Abs(filePath)
+	require.Nil(t, err)
+
+	_, err = th.App.exportCustomEmoji(fileWriter, outPath, dirNameToExportEmoji, false)
 	require.Nil(t, err, "should not have failed")
 }
 
@@ -182,7 +178,7 @@ func TestExportAllUsers(t *testing.T) {
 	require.Nil(t, err)
 
 	var b bytes.Buffer
-	err = th1.App.BulkExport(&b, "somefile", "somePath", "someDir")
+	err = th1.App.BulkExport(&b, "somePath", BulkExportOpts{})
 	require.Nil(t, err)
 
 	th2 := Setup(t)
@@ -228,7 +224,7 @@ func TestExportDMChannel(t *testing.T) {
 	th1.CreateDmChannel(th1.BasicUser2)
 
 	var b bytes.Buffer
-	err := th1.App.BulkExport(&b, "somefile", "somePath", "someDir")
+	err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{})
 	require.Nil(t, err)
 
 	channels, nErr := th1.App.Srv().Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000")
@@ -264,7 +260,7 @@ func TestExportDMChannelToSelf(t *testing.T) {
 	th1.CreateDmChannel(th1.BasicUser)
 
 	var b bytes.Buffer
-	err := th1.App.BulkExport(&b, "somefile", "somePath", "someDir")
+	err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{})
 	require.Nil(t, err)
 
 	channels, nErr := th1.App.Srv().Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000")
@@ -302,7 +298,7 @@ func TestExportGMChannel(t *testing.T) {
 	th1.CreateGroupChannel(user1, user2)
 
 	var b bytes.Buffer
-	err := th1.App.BulkExport(&b, "somefile", "somePath", "someDir")
+	err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{})
 	require.Nil(t, err)
 
 	channels, nErr := th1.App.Srv().Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000")
@@ -334,7 +330,7 @@ func TestExportGMandDMChannels(t *testing.T) {
 	th1.CreateGroupChannel(user1, user2)
 
 	var b bytes.Buffer
-	err := th1.App.BulkExport(&b, "somefile", "somePath", "someDir")
+	err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{})
 	require.Nil(t, err)
 
 	channels, nErr := th1.App.Srv().Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000")
@@ -417,7 +413,7 @@ func TestExportDMandGMPost(t *testing.T) {
 	assert.Equal(t, 4, len(posts))
 
 	var b bytes.Buffer
-	err = th1.App.BulkExport(&b, "somefile", "somePath", "someDir")
+	err = th1.App.BulkExport(&b, "somePath", BulkExportOpts{})
 	require.Nil(t, err)
 
 	th1.TearDown()
@@ -492,7 +488,7 @@ func TestExportPostWithProps(t *testing.T) {
 	require.NotEmpty(t, posts[1].Props)
 
 	var b bytes.Buffer
-	err = th1.App.BulkExport(&b, "somefile", "somePath", "someDir")
+	err = th1.App.BulkExport(&b, "somePath", BulkExportOpts{})
 	require.Nil(t, err)
 
 	th1.TearDown()
@@ -530,7 +526,7 @@ func TestExportDMPostWithSelf(t *testing.T) {
 	th1.CreatePost(dmChannel)
 
 	var b bytes.Buffer
-	err := th1.App.BulkExport(&b, "somefile", "somePath", "someDir")
+	err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{})
 	require.Nil(t, err)
 
 	posts, nErr := th1.App.Srv().Store.Post().GetDirectPostParentsForExportAfter(1000, "0000000")
@@ -557,3 +553,57 @@ func TestExportDMPostWithSelf(t *testing.T) {
 	assert.Equal(t, 1, len((*posts[0].ChannelMembers)))
 	assert.Equal(t, th1.BasicUser.Username, (*posts[0].ChannelMembers)[0])
 }
+
+func TestBulkExport(t *testing.T) {
+	th := Setup(t)
+	testsDir, _ := fileutils.FindDir("tests")
+
+	dir, err := ioutil.TempDir("", "import_test")
+	require.Nil(t, err)
+	defer os.RemoveAll(dir)
+
+	extractImportFile := func(filePath string) *os.File {
+		importFile, err2 := os.Open(filePath)
+		require.Nil(t, err2)
+		defer importFile.Close()
+
+		info, err2 := importFile.Stat()
+		require.Nil(t, err2)
+
+		paths, err2 := utils.UnzipToPath(importFile, info.Size(), dir)
+		require.Nil(t, err2)
+		require.NotEmpty(t, paths)
+
+		jsonFile, err2 := os.Open(filepath.Join(dir, "import.jsonl"))
+		require.Nil(t, err2)
+
+		return jsonFile
+	}
+
+	jsonFile := extractImportFile(filepath.Join(testsDir, "import_test.zip"))
+	defer jsonFile.Close()
+
+	err, _ = th.App.BulkImportWithPath(jsonFile, false, 1, dir)
+	require.Nil(t, err)
+
+	exportFile, err := os.Create(filepath.Join(dir, "export.zip"))
+	require.Nil(t, err)
+	defer exportFile.Close()
+
+	opts := BulkExportOpts{
+		IncludeAttachments: true,
+		CreateArchive:      true,
+	}
+	err = th.App.BulkExport(exportFile, dir, opts)
+	require.Nil(t, err)
+
+	th.TearDown()
+	th = Setup(t)
+	defer th.TearDown()
+
+	jsonFile = extractImportFile(filepath.Join(dir, "export.zip"))
+	defer jsonFile.Close()
+
+	err, _ = th.App.BulkImportWithPath(jsonFile, false, 1, filepath.Join(dir, "data"))
+	require.Nil(t, err)
+}

app/import_functions.go

@@ -9,7 +9,7 @@ import (
 	"crypto/sha1"
 	"errors"
 	"fmt"
-	"io"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"path"
@@ -1117,10 +1117,15 @@ func (a *App) importAttachment(data *AttachmentImportData, post *model.Post, tea
 	if file == nil || err != nil {
 		return nil, model.NewAppError("BulkImport", "app.import.attachment.bad_file.error", map[string]interface{}{"FilePath": *data.Path}, "", http.StatusBadRequest)
 	}
+	defer file.Close()
 
 	timestamp := utils.TimeFromMillis(post.CreateAt)
-	buf := bytes.NewBuffer(nil)
-	_, _ = io.Copy(buf, file)
+
+	fileData, err := ioutil.ReadAll(file)
+	if err != nil {
+		return nil, model.NewAppError("BulkImport", "app.import.attachment.read_file_data.error", map[string]interface{}{"FilePath": *data.Path}, "", http.StatusBadRequest)
+	}
+
 	// Go over existing files in the post and see if there already exists a file with the same name, size and hash. If so - skip it
 	if post.Id != "" {
 		oldFiles, err := a.GetFileInfosForPost(post.Id, true)
@@ -1128,11 +1133,11 @@ func (a *App) importAttachment(data *AttachmentImportData, post *model.Post, tea
 			return nil, model.NewAppError("BulkImport", "app.import.attachment.file_upload.error", map[string]interface{}{"FilePath": *data.Path}, "", http.StatusBadRequest)
 		}
 		for _, oldFile := range oldFiles {
-			if oldFile.Name != path.Base(file.Name()) || oldFile.Size != int64(buf.Len()) {
+			if oldFile.Name != path.Base(file.Name()) || oldFile.Size != int64(len(fileData)) {
 				continue
 			}
 			// check md5
-			newHash := sha1.Sum(buf.Bytes())
+			newHash := sha1.Sum(fileData)
 			oldFileData, err := a.GetFile(oldFile.Id)
 			if err != nil {
 				return nil, model.NewAppError("BulkImport", "app.import.attachment.file_upload.error", map[string]interface{}{"FilePath": *data.Path}, "", http.StatusBadRequest)
@@ -1145,15 +1150,19 @@ func (a *App) importAttachment(data *AttachmentImportData, post *model.Post, tea
 			}
 		}
 	}
-	fileInfo, appErr := a.DoUploadFile(timestamp, teamID, post.ChannelId, post.UserId, file.Name(), buf.Bytes())
+
+	mlog.Info("Uploading file with name", mlog.String("file_name", file.Name()))
+
+	fileInfo, appErr := a.DoUploadFile(timestamp, teamID, post.ChannelId, post.UserId, file.Name(), fileData)
 	if appErr != nil {
 		mlog.Error("Failed to upload file:", mlog.Err(appErr))
 		return nil, appErr
 	}
 
-	a.HandleImages([]string{fileInfo.PreviewPath}, []string{fileInfo.ThumbnailPath}, [][]byte{buf.Bytes()})
+	if fileInfo.IsImage() {
+		a.HandleImages([]string{fileInfo.PreviewPath}, []string{fileInfo.ThumbnailPath}, [][]byte{fileData})
+	}
 
-	mlog.Info("Uploading file with name", mlog.String("file_name", file.Name()))
 	return fileInfo, nil
 }
 

app/opentracing/opentracing_layer.go

@@ -878,7 +878,7 @@ func (a *OpenTracingAppLayer) BuildSamlMetadataObject(idpMetadata []byte) (*mode
 	return resultVar0, resultVar1
 }
 
-func (a *OpenTracingAppLayer) BulkExport(writer io.Writer, file string, pathToEmojiDir string, dirNameToExportEmoji string) *model.AppError {
+func (a *OpenTracingAppLayer) BulkExport(writer io.Writer, outPath string, opts app.BulkExportOpts) *model.AppError {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.BulkExport")
 
@@ -890,7 +890,7 @@ func (a *OpenTracingAppLayer) BulkExport(writer io.Writer, file string, pathToEm
 	}()
 
 	defer span.Finish()
-	resultVar0 := a.app.BulkExport(writer, file, pathToEmojiDir, dirNameToExportEmoji)
+	resultVar0 := a.app.BulkExport(writer, outPath, opts)
 
 	if resultVar0 != nil {
 		span.LogFields(spanlog.Error(resultVar0))
@@ -2756,6 +2756,28 @@ func (a *OpenTracingAppLayer) DeleteEphemeralPost(userID string, postId string)
 	a.app.DeleteEphemeralPost(userID, postId)
 }
 
+func (a *OpenTracingAppLayer) DeleteExport(name string) *model.AppError {
+	origCtx := a.ctx
+	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.DeleteExport")
+
+	a.ctx = newCtx
+	a.app.Srv().Store.SetContext(newCtx)
+	defer func() {
+		a.app.Srv().Store.SetContext(origCtx)
+		a.ctx = origCtx
+	}()
+
+	defer span.Finish()
+	resultVar0 := a.app.DeleteExport(name)
+
+	if resultVar0 != nil {
+		span.LogFields(spanlog.Error(resultVar0))
+		ext.Error.Set(span, true)
+	}
+
+	return resultVar0
+}
+
 func (a *OpenTracingAppLayer) DeleteFlaggedPosts(postId string) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.DeleteFlaggedPosts")
@@ -10423,6 +10445,28 @@ func (a *OpenTracingAppLayer) ListDirectory(path string) ([]string, *model.AppEr
 	return resultVar0, resultVar1
 }
 
+func (a *OpenTracingAppLayer) ListExports() ([]string, *model.AppError) {
+	origCtx := a.ctx
+	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ListExports")
+
+	a.ctx = newCtx
+	a.app.Srv().Store.SetContext(newCtx)
+	defer func() {
+		a.app.Srv().Store.SetContext(origCtx)
+		a.ctx = origCtx
+	}()
+
+	defer span.Finish()
+	resultVar0, resultVar1 := a.app.ListExports()
+
+	if resultVar1 != nil {
+		span.LogFields(spanlog.Error(resultVar1))
+		ext.Error.Set(span, true)
+	}
+
+	return resultVar0, resultVar1
+}
+
 func (a *OpenTracingAppLayer) ListImports() ([]string, *model.AppError) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ListImports")

cmd/mattermost/commands/export.go

@@ -7,13 +7,15 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path/filepath"
 	"time"
 
+	"github.com/mattermost/mattermost-server/v5/app"
+	"github.com/mattermost/mattermost-server/v5/audit"
+	"github.com/mattermost/mattermost-server/v5/model"
+
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
-
-	"github.com/mattermost/mattermost-server/v5/audit"
-	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 var ExportCmd = &cobra.Command{
@@ -74,6 +76,8 @@ func init() {
 	GlobalRelayZipExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")
 
 	BulkExportCmd.Flags().Bool("all-teams", true, "Export all teams from the server.")
+	BulkExportCmd.Flags().Bool("attachments", false, "Also export file attachments.")
+	BulkExportCmd.Flags().Bool("archive", false, "Outputs a single archive file.")
 
 	ExportCmd.AddCommand(ScheduleExportCmd)
 	ExportCmd.AddCommand(CsvExportCmd)
@@ -202,25 +206,31 @@ func bulkExportCmdF(command *cobra.Command, args []string) error {
 		return errors.New("Nothing to export. Please specify the --all-teams flag to export all teams.")
 	}
 
+	attachments, err := command.Flags().GetBool("attachments")
+	if err != nil {
+		return errors.Wrap(err, "attachments flag error")
+	}
+
+	archive, err := command.Flags().GetBool("archive")
+	if err != nil {
+		return errors.Wrap(err, "archive flag error")
+	}
+
 	fileWriter, err := os.Create(args[0])
 	if err != nil {
 		return err
 	}
 	defer fileWriter.Close()
 
-	// Path to directory of custom emoji
-	pathToEmojiDir := "data/emoji/"
-
-	customDataDir := a.Config().FileSettings.Directory
-	if customDataDir != nil && *customDataDir != "" {
-		pathToEmojiDir = *customDataDir + "emoji/"
+	outPath, err := filepath.Abs(args[0])
+	if err != nil {
+		return err
 	}
 
-	// Name of the directory to export custom emoji
-	dirNameToExportEmoji := "exported_emoji"
-
-	// args[0] points to the filename/filepath passed with export bulk command
-	if err := a.BulkExport(fileWriter, args[0], pathToEmojiDir, dirNameToExportEmoji); err != nil {
+	var opts app.BulkExportOpts
+	opts.IncludeAttachments = attachments
+	opts.CreateArchive = archive
+	if err := a.BulkExport(fileWriter, filepath.Dir(outPath), opts); err != nil {
 		CommandPrintErrorln(err.Error())
 		return err
 	}

cmd/mattermost/commands/import.go

@@ -38,6 +38,7 @@ func init() {
 	BulkImportCmd.Flags().Bool("apply", false, "Save the import data to the database. Use with caution - this cannot be reverted.")
 	BulkImportCmd.Flags().Bool("validate", false, "Validate the import data without making any changes to the system.")
 	BulkImportCmd.Flags().Int("workers", 2, "How many workers to run whilst doing the import.")
+	BulkImportCmd.Flags().String("import-path", "", "A path to the data directory to import files from.")
 
 	ImportCmd.AddCommand(
 		BulkImportCmd,
@@ -118,6 +119,11 @@ func bulkImportCmdF(command *cobra.Command, args []string) error {
 		return errors.New("Workers flag error")
 	}
 
+	importPath, err := command.Flags().GetString("import-path")
+	if err != nil {
+		return errors.New("import-path flag error")
+	}
+
 	if len(args) != 1 {
 		return errors.New("Incorrect number of arguments.")
 	}
@@ -143,7 +149,7 @@ func bulkImportCmdF(command *cobra.Command, args []string) error {
 
 	CommandPrettyPrintln("")
 
-	if err, lineNumber := a.BulkImport(fileReader, !apply, workers); err != nil {
+	if err, lineNumber := a.BulkImportWithPath(fileReader, !apply, workers, importPath); err != nil {
 		CommandPrintErrorln(err.Error())
 		if lineNumber != 0 {
 			CommandPrintErrorln(fmt.Sprintf("Error occurred on data file line %v", lineNumber))

i18n/en.json

@@ -1324,6 +1324,10 @@
     "id": "api.emoji.upload.open.app_error",
     "translation": "Unable to create the emoji. An error occurred when trying to open the attached image."
   },
+  {
+    "id": "api.export.export_not_found.app_error",
+    "translation": "Unable to find export file."
+  },
   {
     "id": "api.file.append_file.app_error",
     "translation": "Unable to append data to the file."
@@ -4086,6 +4090,22 @@
     "id": "app.emoji.get_list.internal_error",
     "translation": "Unable to get the emoji."
   },
+  {
+    "id": "app.export.export_attachment.copy_file.error",
+    "translation": "Failed to copy file during export."
+  },
+  {
+    "id": "app.export.export_attachment.create_file.error",
+    "translation": "Failed to created file during export."
+  },
+  {
+    "id": "app.export.export_attachment.mkdirall.error",
+    "translation": "Failed to create directory during export."
+  },
+  {
+    "id": "app.export.export_attachment.zip_create_header.error",
+    "translation": "Failed to create zip header during export."
+  },
   {
     "id": "app.export.export_custom_emoji.copy_emoji_images.error",
     "translation": "Unable to copy custom emoji images"
@@ -4098,6 +4118,14 @@
     "id": "app.export.export_write_line.json_marshall.error",
     "translation": "An error occurred marshalling the JSON data for export."
   },
+  {
+    "id": "app.export.marshal.app_error",
+    "translation": "Unable to marshal response."
+  },
+  {
+    "id": "app.export.zip_create.error",
+    "translation": "Failed to add file to zip archive during export."
+  },
   {
     "id": "app.file_info.get.app_error",
     "translation": "Unable to get the file info."
@@ -4146,6 +4174,10 @@
     "id": "app.import.attachment.file_upload.error",
     "translation": "Error uploading the file: \"{{.FilePath}}\""
   },
+  {
+    "id": "app.import.attachment.read_file_data.error",
+    "translation": "Failed to read file attachment during import."
+  },
   {
     "id": "app.import.bulk_import.file_scan.error",
     "translation": "Error reading import data file."
@@ -7274,6 +7306,14 @@
     "id": "model.config.is_valid.encrypt_sql.app_error",
     "translation": "Invalid at rest encrypt key for SQL settings. Must be 32 chars or more."
   },
+  {
+    "id": "model.config.is_valid.export.directory.app_error",
+    "translation": "Value for Directory should not be empty."
+  },
+  {
+    "id": "model.config.is_valid.export.retention_days_too_low.app_error",
+    "translation": "Invalid value for RetentionDays. Value should be greater than 0"
+  },
   {
     "id": "model.config.is_valid.file_driver.app_error",
     "translation": "Invalid driver name for file settings. Must be 'local' or 'amazons3'."

imports/placeholder.go

@@ -27,4 +27,7 @@ import (
 
 	// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
 	_ "github.com/mattermost/mattermost-server/v5/jobs/import_delete"
+
+	// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
+	_ "github.com/mattermost/mattermost-server/v5/jobs/export_process"
 )

jobs/export_process/worker.go

@@ -0,0 +1,138 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package export_process
+
+import (
+	"io"
+	"path/filepath"
+
+	"github.com/mattermost/mattermost-server/v5/app"
+	"github.com/mattermost/mattermost-server/v5/jobs"
+	tjobs "github.com/mattermost/mattermost-server/v5/jobs/interfaces"
+	"github.com/mattermost/mattermost-server/v5/mlog"
+	"github.com/mattermost/mattermost-server/v5/model"
+)
+
+func init() {
+	app.RegisterJobsExportProcessInterface(func(a *app.App) tjobs.ExportProcessInterface {
+		return &ExportProcessInterfaceImpl{a}
+	})
+}
+
+type ExportProcessInterfaceImpl struct {
+	app *app.App
+}
+
+type ExportProcessWorker struct {
+	name        string
+	stopChan    chan struct{}
+	stoppedChan chan struct{}
+	jobsChan    chan model.Job
+	jobServer   *jobs.JobServer
+	app         *app.App
+}
+
+func (i *ExportProcessInterfaceImpl) MakeWorker() model.Worker {
+	return &ExportProcessWorker{
+		name:        "ExportProcess",
+		stopChan:    make(chan struct{}),
+		stoppedChan: make(chan struct{}),
+		jobsChan:    make(chan model.Job),
+		jobServer:   i.app.Srv().Jobs,
+		app:         i.app,
+	}
+}
+
+func (w *ExportProcessWorker) JobChannel() chan<- model.Job {
+	return w.jobsChan
+}
+
+func (w *ExportProcessWorker) Run() {
+	mlog.Debug("Worker started", mlog.String("worker", w.name))
+
+	defer func() {
+		mlog.Debug("Worker finished", mlog.String("worker", w.name))
+		close(w.stoppedChan)
+	}()
+
+	for {
+		select {
+		case <-w.stopChan:
+			mlog.Debug("Worker received stop signal", mlog.String("worker", w.name))
+			return
+		case job := <-w.jobsChan:
+			mlog.Debug("Worker received a new candidate job.", mlog.String("worker", w.name))
+			w.doJob(&job)
+		}
+	}
+}
+
+func (w *ExportProcessWorker) Stop() {
+	mlog.Debug("Worker stopping", mlog.String("worker", w.name))
+	close(w.stopChan)
+	<-w.stoppedChan
+}
+
+func (w *ExportProcessWorker) doJob(job *model.Job) {
+	if claimed, err := w.jobServer.ClaimJob(job); err != nil {
+		mlog.Warn("Worker experienced an error while trying to claim job",
+			mlog.String("worker", w.name),
+			mlog.String("job_id", job.Id),
+			mlog.String("error", err.Error()))
+		return
+	} else if !claimed {
+		return
+	}
+
+	opts := app.BulkExportOpts{
+		CreateArchive: true,
+	}
+
+	includeAttachments, ok := job.Data["include_attachments"]
+	if ok && includeAttachments == "true" {
+		opts.IncludeAttachments = true
+	}
+
+	outPath := *w.app.Config().ExportSettings.Directory
+	exportFilename := model.NewId() + "_export.zip"
+
+	rd, wr := io.Pipe()
+
+	errCh := make(chan *model.AppError, 1)
+	go func() {
+		defer close(errCh)
+		_, appErr := w.app.WriteFile(rd, filepath.Join(outPath, exportFilename))
+		errCh <- appErr
+	}()
+
+	appErr := w.app.BulkExport(wr, outPath, opts)
+	if err := wr.Close(); err != nil {
+		mlog.Warn("Worker: error closing writer")
+	}
+	if appErr != nil {
+		w.setJobError(job, appErr)
+		return
+	}
+
+	if appErr := <-errCh; appErr != nil {
+		w.setJobError(job, appErr)
+		return
+	}
+
+	mlog.Info("Worker: Job is complete", mlog.String("worker", w.name), mlog.String("job_id", job.Id))
+	w.setJobSuccess(job)
+}
+
+func (w *ExportProcessWorker) setJobSuccess(job *model.Job) {
+	if err := w.app.Srv().Jobs.SetJobSuccess(job); err != nil {
+		mlog.Error("Worker: Failed to set success for job", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
+		w.setJobError(job, err)
+	}
+}
+
+func (w *ExportProcessWorker) setJobError(job *model.Job, appError *model.AppError) {
+	if err := w.app.Srv().Jobs.SetJobError(job, appError); err != nil {
+		mlog.Error("Worker: Failed to set job error", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
+	}
+}

jobs/import_process/worker.go

@@ -165,7 +165,7 @@ func (w *ImportProcessWorker) doJob(job *model.Job) {
 	}
 
 	// do the actual import.
-	appErr, lineNumber := w.app.BulkImportWithPath(jsonFile, false, runtime.NumCPU(), dir)
+	appErr, lineNumber := w.app.BulkImportWithPath(jsonFile, false, runtime.NumCPU(), filepath.Join(dir, app.ExportDataDir))
 	if appErr != nil {
 		job.Data["line_number"] = strconv.Itoa(lineNumber)
 		w.setJobError(job, appErr)

jobs/interfaces/export_process_interface.go

@@ -0,0 +1,12 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package interfaces
+
+import (
+	"github.com/mattermost/mattermost-server/v5/model"
+)
+
+type ExportProcessInterface interface {
+	MakeWorker() model.Worker
+}

jobs/jobs_watcher.go

@@ -163,6 +163,13 @@ func (watcher *Watcher) PollAndNotify() {
 				default:
 				}
 			}
+		} else if job.Type == model.JOB_TYPE_EXPORT_PROCESS {
+			if watcher.workers.ExportProcess != nil {
+				select {
+				case watcher.workers.ExportProcess.JobChannel() <- *job:
+				default:
+				}
+			}
 		} else if job.Type == model.JOB_TYPE_CLOUD {
 			if watcher.workers.Cloud != nil {
 				select {

jobs/server.go

@@ -32,6 +32,7 @@ type JobServer struct {
 	ActiveUsers             tjobs.ActiveUsersJobInterface
 	ImportProcess           tjobs.ImportProcessInterface
 	ImportDelete            tjobs.ImportDeleteInterface
+	ExportProcess           tjobs.ExportProcessInterface
 	Cloud                   ejobs.CloudJobInterface
 }
 

jobs/workers.go

@@ -29,6 +29,7 @@ type Workers struct {
 	ActiveUsers              model.Worker
 	ImportProcess            model.Worker
 	ImportDelete             model.Worker
+	ExportProcess            model.Worker
 	Cloud                    model.Worker
 
 	listenerId string
@@ -92,6 +93,10 @@ func (srv *JobServer) InitWorkers() *Workers {
 		workers.ImportDelete = importDeleteInterface.MakeWorker()
 	}
 
+	if exportProcessInterface := srv.ExportProcess; exportProcessInterface != nil {
+		workers.ExportProcess = exportProcessInterface.MakeWorker()
+	}
+
 	if cloudInterface := srv.Cloud; cloudInterface != nil {
 		workers.Cloud = cloudInterface.MakeWorker()
 	}
@@ -155,6 +160,10 @@ func (workers *Workers) Start() *Workers {
 			go workers.ImportDelete.Run()
 		}
 
+		if workers.ExportProcess != nil {
+			go workers.ExportProcess.Run()
+		}
+
 		if workers.Cloud != nil {
 			go workers.Cloud.Run()
 		}
@@ -276,6 +285,10 @@ func (workers *Workers) Stop() *Workers {
 		workers.ImportDelete.Stop()
 	}
 
+	if workers.ExportProcess != nil {
+		workers.ExportProcess.Stop()
+	}
+
 	if workers.Cloud != nil {
 		workers.Cloud.Stop()
 	}

model/client4.go

@@ -34,6 +34,7 @@ const (
 	HEADER_CLOUD_TOKEN        = "X-Cloud-Token"
 	HEADER_REQUESTED_WITH     = "X-Requested-With"
 	HEADER_REQUESTED_WITH_XML = "XMLHttpRequest"
+	HEADER_RANGE              = "Range"
 	STATUS                    = "status"
 	STATUS_OK                 = "OK"
 	STATUS_FAIL               = "FAIL"
@@ -550,6 +551,14 @@ func (c *Client4) GetImportsRoute() string {
 	return "/imports"
 }
 
+func (c *Client4) GetExportsRoute() string {
+	return "/exports"
+}
+
+func (c *Client4) GetExportRoute(name string) string {
+	return fmt.Sprintf(c.GetExportsRoute()+"/%v", name)
+}
+
 func (c *Client4) DoApiGet(url string, etag string) (*http.Response, *AppError) {
 	return c.DoApiRequest(http.MethodGet, c.ApiUrl+url, "", etag)
 }
@@ -575,21 +584,25 @@ func (c *Client4) DoApiDelete(url string) (*http.Response, *AppError) {
 }
 
 func (c *Client4) DoApiRequest(method, url, data, etag string) (*http.Response, *AppError) {
-	return c.doApiRequestReader(method, url, strings.NewReader(data), etag)
+	return c.doApiRequestReader(method, url, strings.NewReader(data), map[string]string{HEADER_ETAG_CLIENT: etag})
+}
+
+func (c *Client4) DoApiRequestWithHeaders(method, url, data string, headers map[string]string) (*http.Response, *AppError) {
+	return c.doApiRequestReader(method, url, strings.NewReader(data), headers)
 }
 
 func (c *Client4) doApiRequestBytes(method, url string, data []byte, etag string) (*http.Response, *AppError) {
-	return c.doApiRequestReader(method, url, bytes.NewReader(data), etag)
+	return c.doApiRequestReader(method, url, bytes.NewReader(data), map[string]string{HEADER_ETAG_CLIENT: etag})
 }
 
-func (c *Client4) doApiRequestReader(method, url string, data io.Reader, etag string) (*http.Response, *AppError) {
+func (c *Client4) doApiRequestReader(method, url string, data io.Reader, headers map[string]string) (*http.Response, *AppError) {
 	rq, err := http.NewRequest(method, url, data)
 	if err != nil {
 		return nil, NewAppError(url, "model.client.connecting.app_error", nil, err.Error(), http.StatusBadRequest)
 	}
 
-	if etag != "" {
-		rq.Header.Set(HEADER_ETAG_CLIENT, etag)
+	for k, v := range headers {
+		rq.Header.Set(k, v)
 	}
 
 	if c.AuthToken != "" {
@@ -5668,7 +5681,7 @@ func (c *Client4) GetUploadsForUser(userId string) ([]*UploadSession, *Response)
 // a FileInfo object.
 func (c *Client4) UploadData(uploadId string, data io.Reader) (*FileInfo, *Response) {
 	url := c.GetUploadRoute(uploadId)
-	r, err := c.doApiRequestReader("POST", c.ApiUrl+url, data, "")
+	r, err := c.doApiRequestReader("POST", c.ApiUrl+url, data, nil)
 	if err != nil {
 		return nil, BuildErrorResponse(r, err)
 	}
@@ -5816,6 +5829,43 @@ func (c *Client4) ListImports() ([]string, *Response) {
 	return ArrayFromJson(r.Body), BuildResponse(r)
 }
 
+func (c *Client4) ListExports() ([]string, *Response) {
+	r, err := c.DoApiGet(c.GetExportsRoute(), "")
+	if err != nil {
+		return nil, BuildErrorResponse(r, err)
+	}
+	defer closeBody(r)
+	return ArrayFromJson(r.Body), BuildResponse(r)
+}
+
+func (c *Client4) DeleteExport(name string) (bool, *Response) {
+	r, err := c.DoApiDelete(c.GetExportRoute(name))
+	if err != nil {
+		return false, BuildErrorResponse(r, err)
+	}
+	defer closeBody(r)
+	return CheckStatusOK(r), BuildResponse(r)
+}
+
+func (c *Client4) DownloadExport(name string, wr io.Writer, offset int64) (int64, *Response) {
+	var headers map[string]string
+	if offset > 0 {
+		headers = map[string]string{
+			HEADER_RANGE: fmt.Sprintf("bytes=%d-", offset),
+		}
+	}
+	r, appErr := c.DoApiRequestWithHeaders(http.MethodGet, c.ApiUrl+c.GetExportRoute(name), "", headers)
+	if appErr != nil {
+		return 0, BuildErrorResponse(r, appErr)
+	}
+	defer closeBody(r)
+	n, err := io.Copy(wr, r.Body)
+	if err != nil {
+		return n, BuildErrorResponse(r, NewAppError("DownloadExport", "model.client.copy.app_error", nil, err.Error(), r.StatusCode))
+	}
+	return n, BuildResponse(r)
+}
+
 func (c *Client4) GetThreadMentionsForUserPerChannel(userId, teamId string) (map[string]int64, *Response) {
 	url := c.GetUserThreadsRoute(userId, teamId)
 	r, appErr := c.DoApiGet(url+"/mention_counts", "")

model/config.go

@@ -122,6 +122,9 @@ const (
 	IMPORT_SETTINGS_DEFAULT_DIRECTORY      = "./import"
 	IMPORT_SETTINGS_DEFAULT_RETENTION_DAYS = 30
 
+	EXPORT_SETTINGS_DEFAULT_DIRECTORY      = "./export"
+	EXPORT_SETTINGS_DEFAULT_RETENTION_DAYS = 30
+
 	EMAIL_SETTINGS_DEFAULT_FEEDBACK_ORGANIZATION = ""
 
 	SUPPORT_SETTINGS_DEFAULT_TERMS_OF_SERVICE_LINK = "https://about.mattermost.com/default-terms/"
@@ -2936,6 +2939,37 @@ func (s *ImportSettings) SetDefaults() {
 	}
 }
 
+// ExportSettings defines configuration settings for file exports.
+type ExportSettings struct {
+	// The directory where to store the exported files.
+	Directory *string
+	// The number of days to retain the exported files before deleting them.
+	RetentionDays *int
+}
+
+func (s *ExportSettings) isValid() *AppError {
+	if *s.Directory == "" {
+		return NewAppError("Config.IsValid", "model.config.is_valid.export.directory.app_error", nil, "", http.StatusBadRequest)
+	}
+
+	if *s.RetentionDays <= 0 {
+		return NewAppError("Config.IsValid", "model.config.is_valid.export.retention_days_too_low.app_error", nil, "", http.StatusBadRequest)
+	}
+
+	return nil
+}
+
+// SetDefaults applies the default settings to the struct.
+func (s *ExportSettings) SetDefaults() {
+	if s.Directory == nil || *s.Directory == "" {
+		s.Directory = NewString(EXPORT_SETTINGS_DEFAULT_DIRECTORY)
+	}
+
+	if s.RetentionDays == nil {
+		s.RetentionDays = NewInt(EXPORT_SETTINGS_DEFAULT_RETENTION_DAYS)
+	}
+}
+
 type ConfigFunc func() *Config
 
 const ConfigAccessTagType = "access"
@@ -3013,6 +3047,7 @@ type Config struct {
 	CloudSettings             CloudSettings
 	FeatureFlags              *FeatureFlags `json:",omitempty"`
 	ImportSettings            ImportSettings
+	ExportSettings            ExportSettings
 }
 
 func (o *Config) Clone() *Config {
@@ -3121,6 +3156,7 @@ func (o *Config) SetDefaults() {
 		o.FeatureFlags.SetDefaults()
 	}
 	o.ImportSettings.SetDefaults()
+	o.ExportSettings.SetDefaults()
 }
 
 func (o *Config) IsValid() *AppError {

model/config_test.go

@@ -1439,3 +1439,31 @@ func TestConfigImportSettingsIsValid(t *testing.T) {
 	require.NotNil(t, err)
 	require.Equal(t, "model.config.is_valid.import.retention_days_too_low.app_error", err.Id)
 }
+
+func TestConfigExportSettingsDefaults(t *testing.T) {
+	cfg := Config{}
+	cfg.SetDefaults()
+
+	require.Equal(t, "./export", *cfg.ExportSettings.Directory)
+	require.Equal(t, 30, *cfg.ExportSettings.RetentionDays)
+}
+
+func TestConfigExportSettingsIsValid(t *testing.T) {
+	cfg := Config{}
+	cfg.SetDefaults()
+
+	err := cfg.ExportSettings.isValid()
+	require.Nil(t, err)
+
+	*cfg.ExportSettings.Directory = ""
+	err = cfg.ExportSettings.isValid()
+	require.NotNil(t, err)
+	require.Equal(t, "model.config.is_valid.export.directory.app_error", err.Id)
+
+	cfg.SetDefaults()
+
+	*cfg.ExportSettings.RetentionDays = 0
+	err = cfg.ExportSettings.isValid()
+	require.NotNil(t, err)
+	require.Equal(t, "model.config.is_valid.export.retention_days_too_low.app_error", err.Id)
+}

model/job.go

@@ -24,6 +24,7 @@ const (
 	JOB_TYPE_ACTIVE_USERS                   = "active_users"
 	JOB_TYPE_IMPORT_PROCESS                 = "import_process"
 	JOB_TYPE_IMPORT_DELETE                  = "import_delete"
+	JOB_TYPE_EXPORT_PROCESS                 = "export_process"
 	JOB_TYPE_CLOUD                          = "cloud"
 
 	JOB_STATUS_PENDING          = "pending"
@@ -70,6 +71,7 @@ func (j *Job) IsValid() *AppError {
 	case JOB_TYPE_ACTIVE_USERS:
 	case JOB_TYPE_IMPORT_PROCESS:
 	case JOB_TYPE_IMPORT_DELETE:
+	case JOB_TYPE_EXPORT_PROCESS:
 	case JOB_TYPE_CLOUD:
 	default:
 		return NewAppError("Job.IsValid", "model.job.is_valid.type.app_error", nil, "id="+j.Id, http.StatusBadRequest)

utils/archive.go

@@ -38,12 +38,16 @@ func UnzipToPath(zipFile io.ReaderAt, size int64, outPath string) ([]string, err
 		path := filepath.Join(outPath, filePath)
 		paths[i] = path
 		if f.FileInfo().IsDir() {
-			if err := os.Mkdir(path, 0744); err != nil {
+			if err := os.Mkdir(path, 0700); err != nil {
 				return nil, fmt.Errorf("failed to create directory: %w", err)
 			}
 			continue
 		}
-
+		if _, err := os.Stat(filepath.Dir(path)); os.IsNotExist(err) {
+			if err = os.MkdirAll(filepath.Dir(path), 0700); err != nil {
+				return nil, fmt.Errorf("failed to create directory: %w", err)
+			}
+		}
 		outFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0600)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create file: %w", err)

web/params.go

@@ -83,6 +83,7 @@ type Params struct {
 	FilterParentTeamPermitted bool
 	CategoryId                string
 	WarnMetricId              string
+	ExportName                string
 
 	// Cloud
 	InvoiceId string
@@ -346,5 +347,9 @@ func ParamsFromRequest(r *http.Request) *Params {
 		params.WarnMetricId = val
 	}
 
+	if val, ok := props["export_name"]; ok {
+		params.ExportName = val
+	}
+
 	return params
 }