diff --git a/server/channels/app/export.go b/server/channels/app/export.go index 92707cc103..0258a5171e 100644 --- a/server/channels/app/export.go +++ b/server/channels/app/export.go @@ -207,9 +207,10 @@ func (a *App) BulkExport(ctx request.CTX, writer io.Writer, outPath string, job ctx.Logger().Info("Bulk export: exporting custom emojis") for _, emojiPath := range emojiPaths { if err := a.exportFile(ctx, outPath, emojiPath, zipWr); err != nil { - return err + ctx.Logger().Warn("Unable to export emoji", mlog.String("emoji_path", emojiPath), mlog.Err(err)) + } else { + totalExportedEmojis++ } - totalExportedEmojis++ if totalExportedEmojis%10 == 0 { ctx.Logger().Info("Bulk export: exporting emojis progress", mlog.Int("total_successfully_exported_emojis", totalExportedEmojis), mlog.Int("total_emojis_to_export", emojisLen)) } @@ -236,9 +237,10 @@ func (a *App) exportAttachments(ctx request.CTX, attachments []imports.Attachmen attachmentsLen := len(attachments) for _, attachment := range attachments { if err := a.exportFile(ctx, outPath, *attachment.Path, zipWr); err != nil { - return err + ctx.Logger().Warn("Unable to export file attachment", mlog.String("attachment_path", *attachment.Path), mlog.Err(err)) + } else { + totalExportedFiles++ } - totalExportedFiles++ if totalExportedFiles%10 == 0 { ctx.Logger().Info("Bulk export: exporting file attachments progress", mlog.Int("total_successfully_exported_files", totalExportedFiles), mlog.Int("total_files_to_export", attachmentsLen)) } diff --git a/server/channels/app/export_test.go b/server/channels/app/export_test.go index bee76b43e6..e746ed6fe2 100644 --- a/server/channels/app/export_test.go +++ b/server/channels/app/export_test.go @@ -14,6 +14,10 @@ import ( "testing" "time" + "github.com/mattermost/mattermost/server/v8/channels/testlib" + + "github.com/mattermost/mattermost/server/public/shared/mlog" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -843,6 +847,113 @@ func TestExportPostsWithThread(t *testing.T) { }) } +func TestExportFileWarnings(t *testing.T) { + testCases := []struct { + Description string + ConfigFunc func(cfg *model.Config) + }{ + { + "local", + func(cfg *model.Config) { + cfg.FileSettings.DriverName = model.NewPointer(model.ImageDriverLocal) + }, + }, + { + "s3", + func(cfg *model.Config) { + s3Host := os.Getenv("CI_MINIO_HOST") + if s3Host == "" { + s3Host = "localhost" + } + + s3Port := os.Getenv("CI_MINIO_PORT") + if s3Port == "" { + s3Port = "9000" + } + + s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port) + cfg.FileSettings.DriverName = model.NewPointer(model.ImageDriverS3) + cfg.FileSettings.AmazonS3AccessKeyId = model.NewPointer(model.MinioAccessKey) + cfg.FileSettings.AmazonS3SecretAccessKey = model.NewPointer(model.MinioSecretKey) + cfg.FileSettings.AmazonS3Bucket = model.NewPointer(model.MinioBucket) + cfg.FileSettings.AmazonS3PathPrefix = model.NewPointer("") + cfg.FileSettings.AmazonS3Endpoint = model.NewPointer(s3Endpoint) + cfg.FileSettings.AmazonS3Region = model.NewPointer("") + cfg.FileSettings.AmazonS3SSL = model.NewPointer(false) + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.Description, func(t *testing.T) { + th := Setup(t) + defer th.TearDown() + + th.App.UpdateConfig(testCase.ConfigFunc) + + // Create a buffer to capture logs + buffer := &mlog.Buffer{} + err := mlog.AddWriterTarget(th.TestLogger, buffer, true, mlog.StdAll...) + require.NoError(t, err) + + testsDir, _ := fileutils.FindDir("tests") + dir, err := os.MkdirTemp("", "import_test") + require.NoError(t, err) + defer os.RemoveAll(dir) + + extractImportFile := func(filePath string) *os.File { + importFile, err2 := os.Open(filePath) + require.NoError(t, err2) + defer importFile.Close() + + info, err2 := importFile.Stat() + require.NoError(t, err2) + + paths, err2 := utils.UnzipToPath(importFile, info.Size(), dir) + require.NoError(t, err2) + require.NotEmpty(t, paths) + + jsonFile, err2 := os.Open(filepath.Join(dir, "import.jsonl")) + require.NoError(t, err2) + + return jsonFile + } + + jsonFile := extractImportFile(filepath.Join(testsDir, "import_test.zip")) + defer jsonFile.Close() + + appErr, _ := th.App.BulkImportWithPath(th.Context, jsonFile, nil, false, true, 1, dir) + require.Nil(t, appErr) + + // delete one of the files + params := &model.SearchParams{Terms: "test3.png", SearchWithoutUserId: true} + results, err := th.App.Srv().Store().FileInfo().Search(th.Context, []*model.SearchParams{params}, "", "", 0, 20) + require.NoError(t, err) + require.Len(t, results.FileInfos, 1) + + for _, info2 := range results.FileInfos { + err2 := th.App.RemoveFile(info2.Path) + require.Nil(t, err2) + } + + exportFile, err := os.Create(filepath.Join(dir, "export.zip")) + require.NoError(t, err) + defer exportFile.Close() + + opts := model.BulkExportOpts{ + IncludeAttachments: true, + CreateArchive: true, + } + appErr = th.App.BulkExport(th.Context, exportFile, dir, nil, opts) + // should not get an error for the missing file + require.Nil(t, appErr) + + // should get a warning instead: + testlib.AssertLog(t, buffer, mlog.LvlWarn.Name, "Unable to export file attachment") + }) + } +} + func TestBulkExport(t *testing.T) { th := Setup(t) testsDir, _ := fileutils.FindDir("tests") diff --git a/server/cmd/mmctl/commands/importer/validate.go b/server/cmd/mmctl/commands/importer/validate.go index 147c1210c3..e8192b7097 100644 --- a/server/cmd/mmctl/commands/importer/validate.go +++ b/server/cmd/mmctl/commands/importer/validate.go @@ -815,7 +815,10 @@ func (v *Validator) validatePost(info ImportFileInfo, line imports.LineImportDat continue } - attachmentPath := path.Join("data", *attachment.Path) + attachmentPath := *attachment.Path + if _, ok := v.attachments[attachmentPath]; !ok { + attachmentPath = path.Join("data", *attachment.Path) + } if _, ok := v.attachments[attachmentPath]; !ok { helpful := "" @@ -949,7 +952,10 @@ func (v *Validator) validateDirectPost(info ImportFileInfo, line imports.LineImp continue } - attachmentPath := path.Join("data", *attachment.Path) + attachmentPath := *attachment.Path + if _, ok := v.attachments[attachmentPath]; !ok { + attachmentPath = path.Join("data", *attachment.Path) + } if _, ok := v.attachments[attachmentPath]; !ok { helpful := "" diff --git a/server/tests/import_test.zip b/server/tests/import_test.zip index b2c349843c..daddba9e0d 100644 Binary files a/server/tests/import_test.zip and b/server/tests/import_test.zip differ