diff --git a/app/export_test.go b/app/export_test.go index 066aec3771..7dbaa984b7 100644 --- a/app/export_test.go +++ b/app/export_test.go @@ -185,7 +185,7 @@ func TestExportAllUsers(t *testing.T) { defer th2.TearDown() err, i := th2.App.BulkImport(th2.Context, &b, nil, false, 5) assert.Nil(t, err) - assert.Equal(t, 0, i) + assert.EqualValues(t, 0, i) users1, err := th1.App.GetUsersFromProfiles(&model.UserGetOptions{ Page: 0, @@ -323,7 +323,7 @@ func TestExportDMChannelToSelf(t *testing.T) { // import the exported channel err, i := th2.App.BulkImport(th2.Context, &b, nil, false, 5) assert.Nil(t, err) - assert.Equal(t, 0, i) + assert.EqualValues(t, 0, i) channels, nErr = th2.App.Srv().Store().Channel().GetAllDirectChannelsForExportAfter(1000, "00000000") require.NoError(t, nErr) diff --git a/app/import.go b/app/import.go index 832524aa99..9510a91493 100644 --- a/app/import.go +++ b/app/import.go @@ -6,7 +6,6 @@ package app import ( "archive/zip" "bufio" - "bytes" "encoding/json" "fmt" "io" @@ -26,6 +25,7 @@ type ReactionImportData = imports.ReactionImportData // part of the app interfac const ( importMultiplePostsThreshold = 1000 maxScanTokenSize = 16 * 1024 * 1024 // Need to set a higher limit than default because some customers cross the limit. See MM-22314 + statusUpdateAfterLines = 8192 ) func stopOnError(c request.CTX, err imports.LineImportWorkerError) bool { @@ -41,7 +41,7 @@ func stopOnError(c request.CTX, err imports.LineImportWorkerError) bool { } } -func processAttachmentPaths(files *[]imports.AttachmentImportData, basePath string, filesMap map[string]*zip.File) error { +func processAttachmentPaths(c request.CTX, files *[]imports.AttachmentImportData, basePath string, filesMap map[string]*zip.File) error { if files == nil { return nil } @@ -61,20 +61,20 @@ func processAttachmentPaths(files *[]imports.AttachmentImportData, basePath stri return nil } -func processAttachments(line *imports.LineImportData, basePath string, filesMap map[string]*zip.File) error { +func processAttachments(c request.CTX, line *imports.LineImportData, basePath string, filesMap map[string]*zip.File) error { var ok bool switch line.Type { case "post", "direct_post": var replies []imports.ReplyImportData if line.Type == "direct_post" { - if err := processAttachmentPaths(line.DirectPost.Attachments, basePath, filesMap); err != nil { + if err := processAttachmentPaths(c, line.DirectPost.Attachments, basePath, filesMap); err != nil { return err } if line.DirectPost.Replies != nil { replies = *line.DirectPost.Replies } } else { - if err := processAttachmentPaths(line.Post.Attachments, basePath, filesMap); err != nil { + if err := processAttachmentPaths(c, line.Post.Attachments, basePath, filesMap); err != nil { return err } if line.Post.Replies != nil { @@ -82,7 +82,7 @@ func processAttachments(line *imports.LineImportData, basePath string, filesMap } } for _, reply := range replies { - if err := processAttachmentPaths(reply.Attachments, basePath, filesMap); err != nil { + if err := processAttachmentPaths(c, reply.Attachments, basePath, filesMap); err != nil { return err } } @@ -112,6 +112,15 @@ func processAttachments(line *imports.LineImportData, basePath string, filesMap } func (a *App) bulkImportWorker(c request.CTX, dryRun bool, wg *sync.WaitGroup, lines <-chan imports.LineImportWorkerData, errors chan<- imports.LineImportWorkerError) { + workerID := model.NewId() + processedLines := uint64(0) + + c.Logger().Info("Started new bulk import worker", mlog.String("bulk_import_worker_id", workerID)) + defer func() { + wg.Done() + c.Logger().Info("Bulk import worker finished", mlog.String("bulk_import_worker_id", workerID), mlog.Uint64("processed_lines", processedLines)) + }() + postLines := []imports.LineImportWorkerData{} directPostLines := []imports.LineImportWorkerData{} for line := range lines { @@ -143,6 +152,11 @@ func (a *App) bulkImportWorker(c request.CTX, dryRun bool, wg *sync.WaitGroup, l errors <- imports.LineImportWorkerError{Error: err, LineNumber: line.LineNumber} } } + + processedLines++ + if processedLines%statusUpdateAfterLines == 0 { + c.Logger().Info("Worker progress", mlog.String("bulk_import_worker_id", workerID), mlog.Uint64("processed_lines", processedLines)) + } } if len(postLines) > 0 { @@ -155,7 +169,6 @@ func (a *App) bulkImportWorker(c request.CTX, dryRun bool, wg *sync.WaitGroup, l errors <- imports.LineImportWorkerError{Error: err, LineNumber: errLine} } } - wg.Done() } func (a *App) BulkImport(c *request.Context, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun bool, workers int) (*model.AppError, int) { @@ -194,15 +207,17 @@ func (a *App) bulkImport(c request.CTX, jsonlReader io.Reader, attachmentsReader } for scanner.Scan() { - decoder := json.NewDecoder(bytes.NewReader(scanner.Bytes())) lineNumber++ + if lineNumber%statusUpdateAfterLines == 0 { + c.Logger().Info("Reader progress", mlog.Int("processed_lines", lineNumber)) + } var line imports.LineImportData - if err := decoder.Decode(&line); err != nil { + if err := json.Unmarshal(scanner.Bytes(), &line); err != nil { return model.NewAppError("BulkImport", "app.import.bulk_import.json_decode.error", nil, "", http.StatusBadRequest).Wrap(err), lineNumber } - if err := processAttachments(&line, importPath, attachedFiles); err != nil { + if err := processAttachments(c, &line, importPath, attachedFiles); err != nil { c.Logger().Warn("Error while processing import attachments. Objects might be broken.", mlog.Err(err)) } @@ -222,6 +237,12 @@ func (a *App) bulkImport(c request.CTX, jsonlReader io.Reader, attachmentsReader if line.Type != lastLineType { // Only clear the worker queue if is not the first data entry if lineNumber != 2 { + c.Logger().Info( + "Finished parsing segment, waiting for workers to finish", + mlog.String("old_segment", lastLineType), + mlog.String("new_segment", line.Type), + ) + // Changing type. Clear out the worker queue before continuing. close(linesChan) wg.Wait() @@ -235,6 +256,13 @@ func (a *App) bulkImport(c request.CTX, jsonlReader io.Reader, attachmentsReader } } + c.Logger().Info( + "Starting workers for new segment", + mlog.String("old_segment", lastLineType), + mlog.String("new_segment", line.Type), + mlog.Int("workers", workers), + ) + // Set up the workers and channel for this type. lastLineType = line.Type linesChan = make(chan imports.LineImportWorkerData, workers) @@ -290,7 +318,7 @@ func (a *App) importLine(c request.CTX, line imports.LineImportData, dryRun bool if line.Scheme == nil { return model.NewAppError("BulkImport", "app.import.import_line.null_scheme.error", nil, "", http.StatusBadRequest) } - return a.importScheme(line.Scheme, dryRun) + return a.importScheme(c, line.Scheme, dryRun) case line.Type == "team": if line.Team == nil { return model.NewAppError("BulkImport", "app.import.import_line.null_team.error", nil, "", http.StatusBadRequest) @@ -315,7 +343,7 @@ func (a *App) importLine(c request.CTX, line imports.LineImportData, dryRun bool if line.Emoji == nil { return model.NewAppError("BulkImport", "app.import.import_line.null_emoji.error", nil, "", http.StatusBadRequest) } - return a.importEmoji(line.Emoji, dryRun) + return a.importEmoji(c, line.Emoji, dryRun) default: return model.NewAppError("BulkImport", "app.import.import_line.unknown_line_type.error", map[string]any{"Type": line.Type}, "", http.StatusBadRequest) } diff --git a/app/import_functions.go b/app/import_functions.go index bf6d073f72..1f28b046d3 100644 --- a/app/import_functions.go +++ b/app/import_functions.go @@ -15,6 +15,7 @@ import ( "path" "strings" + "github.com/mattermost/logr/v2" "github.com/mattermost/mattermost-server/v6/app/imports" "github.com/mattermost/mattermost-server/v6/app/request" "github.com/mattermost/mattermost-server/v6/app/teams" @@ -25,13 +26,16 @@ import ( "github.com/mattermost/mattermost-server/v6/utils" ) -// // -- Bulk Import Functions -- // These functions import data directly into the database. Security and permission checks are bypassed but validity is // still enforced. -// +func (a *App) importScheme(c request.CTX, data *imports.SchemeImportData, dryRun bool) *model.AppError { + var fields []logr.Field + if data != nil && data.Name != nil { + fields = append(fields, mlog.String("schema_name", *data.Name)) + } + c.Logger().Info("Validating schema", fields...) -func (a *App) importScheme(data *imports.SchemeImportData, dryRun bool) *model.AppError { if err := imports.ValidateSchemeImportData(data); err != nil { return err } @@ -41,6 +45,8 @@ func (a *App) importScheme(data *imports.SchemeImportData, dryRun bool) *model.A return nil } + c.Logger().Info("Importing schema", fields...) + scheme, err := a.GetSchemeByName(*data.Name) if err != nil { scheme = new(model.Scheme) @@ -68,12 +74,12 @@ func (a *App) importScheme(data *imports.SchemeImportData, dryRun bool) *model.A if scheme.Scope == model.SchemeScopeTeam { data.DefaultTeamAdminRole.Name = &scheme.DefaultTeamAdminRole - if err := a.importRole(data.DefaultTeamAdminRole, dryRun, true); err != nil { + if err := a.importRole(c, data.DefaultTeamAdminRole, dryRun, true); err != nil { return err } data.DefaultTeamUserRole.Name = &scheme.DefaultTeamUserRole - if err := a.importRole(data.DefaultTeamUserRole, dryRun, true); err != nil { + if err := a.importRole(c, data.DefaultTeamUserRole, dryRun, true); err != nil { return err } @@ -83,19 +89,19 @@ func (a *App) importScheme(data *imports.SchemeImportData, dryRun bool) *model.A } } data.DefaultTeamGuestRole.Name = &scheme.DefaultTeamGuestRole - if err := a.importRole(data.DefaultTeamGuestRole, dryRun, true); err != nil { + if err := a.importRole(c, data.DefaultTeamGuestRole, dryRun, true); err != nil { return err } } if scheme.Scope == model.SchemeScopeTeam || scheme.Scope == model.SchemeScopeChannel { data.DefaultChannelAdminRole.Name = &scheme.DefaultChannelAdminRole - if err := a.importRole(data.DefaultChannelAdminRole, dryRun, true); err != nil { + if err := a.importRole(c, data.DefaultChannelAdminRole, dryRun, true); err != nil { return err } data.DefaultChannelUserRole.Name = &scheme.DefaultChannelUserRole - if err := a.importRole(data.DefaultChannelUserRole, dryRun, true); err != nil { + if err := a.importRole(c, data.DefaultChannelUserRole, dryRun, true); err != nil { return err } @@ -105,7 +111,7 @@ func (a *App) importScheme(data *imports.SchemeImportData, dryRun bool) *model.A } } data.DefaultChannelGuestRole.Name = &scheme.DefaultChannelGuestRole - if err := a.importRole(data.DefaultChannelGuestRole, dryRun, true); err != nil { + if err := a.importRole(c, data.DefaultChannelGuestRole, dryRun, true); err != nil { return err } } @@ -113,8 +119,15 @@ func (a *App) importScheme(data *imports.SchemeImportData, dryRun bool) *model.A return nil } -func (a *App) importRole(data *imports.RoleImportData, dryRun bool, isSchemeRole bool) *model.AppError { +func (a *App) importRole(c request.CTX, data *imports.RoleImportData, dryRun bool, isSchemeRole bool) *model.AppError { + var fields []logr.Field + if data != nil && data.Name != nil { + fields = append(fields, mlog.String("role_name", *data.Name)) + } + if !isSchemeRole { + c.Logger().Info("Validating role", fields...) + if err := imports.ValidateRoleImportData(data); err != nil { return err } @@ -125,6 +138,8 @@ func (a *App) importRole(data *imports.RoleImportData, dryRun bool, isSchemeRole return nil } + c.Logger().Info("Importing role", fields...) + role, err := a.GetRoleByName(context.Background(), *data.Name) if err != nil { role = new(model.Role) @@ -160,6 +175,12 @@ func (a *App) importRole(data *imports.RoleImportData, dryRun bool, isSchemeRole } func (a *App) importTeam(c request.CTX, data *imports.TeamImportData, dryRun bool) *model.AppError { + var fields []logr.Field + if data != nil && data.Name != nil { + fields = append(fields, mlog.String("team_name", *data.Name)) + } + c.Logger().Info("Validating team", fields...) + if err := imports.ValidateTeamImportData(data); err != nil { return err } @@ -169,6 +190,8 @@ func (a *App) importTeam(c request.CTX, data *imports.TeamImportData, dryRun boo return nil } + c.Logger().Info("Importing team", fields...) + var team *model.Team team, err := a.Srv().Store().Team().GetByName(*data.Name) @@ -228,6 +251,12 @@ func (a *App) importTeam(c request.CTX, data *imports.TeamImportData, dryRun boo } func (a *App) importChannel(c request.CTX, data *imports.ChannelImportData, dryRun bool) *model.AppError { + var fields []logr.Field + if data != nil && data.Name != nil { + fields = append(fields, mlog.String("channel_name", *data.Name)) + } + c.Logger().Info("Validating channel", fields...) + if err := imports.ValidateChannelImportData(data); err != nil { return err } @@ -237,6 +266,8 @@ func (a *App) importChannel(c request.CTX, data *imports.ChannelImportData, dryR return nil } + c.Logger().Info("Importing channel", fields...) + team, err := a.Srv().Store().Team().GetByName(*data.Team) if err != nil { return model.NewAppError("BulkImport", "app.import.import_channel.team_not_found.error", map[string]any{"TeamName": *data.Team}, "", http.StatusBadRequest).Wrap(err) @@ -293,6 +324,12 @@ func (a *App) importChannel(c request.CTX, data *imports.ChannelImportData, dryR } func (a *App) importUser(c request.CTX, data *imports.UserImportData, dryRun bool) *model.AppError { + var fields []logr.Field + if data != nil && data.Username != nil { + fields = append(fields, mlog.String("user_name", *data.Username)) + } + c.Logger().Info("Validating user", fields...) + if err := imports.ValidateUserImportData(data); err != nil { return err } @@ -302,6 +339,8 @@ func (a *App) importUser(c request.CTX, data *imports.UserImportData, dryRun boo return nil } + c.Logger().Info("Importing user", fields...) + // We want to avoid database writes if nothing has changed. hasUserChanged := false hasNotifyPropsChanged := false @@ -1214,6 +1253,8 @@ func (a *App) importAttachment(c request.CTX, data *imports.AttachmentImportData defer zipFile.Close() name = data.Data.Name file = zipFile.(io.Reader) + + c.Logger().Info("Preparing file upload from ZIP", mlog.String("file_name", name), mlog.Uint64("file_size", data.Data.UncompressedSize64)) } else { realFile, err := os.Open(*data.Path) if err != nil { @@ -1222,6 +1263,12 @@ func (a *App) importAttachment(c request.CTX, data *imports.AttachmentImportData defer realFile.Close() name = realFile.Name() file = realFile + + fields := []logr.Field{mlog.String("file_name", name)} + if info, err := realFile.Stat(); err != nil { + fields = append(fields, mlog.Int64("file_size", info.Size())) + } + c.Logger().Info("Preparing file upload from file system", fields...) } timestamp := utils.TimeFromMillis(post.CreateAt) @@ -1241,7 +1288,8 @@ func (a *App) importAttachment(c request.CTX, data *imports.AttachmentImportData if oldFile.Name != path.Base(name) || oldFile.Size != int64(len(fileData)) { continue } - // check md5 + + // check sha1 newHash := sha1.Sum(fileData) oldFileData, err := a.getFileIgnoreCloudLimit(oldFile.Id) if err != nil { @@ -1260,7 +1308,7 @@ func (a *App) importAttachment(c request.CTX, data *imports.AttachmentImportData fileInfo, appErr := a.DoUploadFile(c, timestamp, teamID, post.ChannelId, post.UserId, name, fileData) if appErr != nil { - mlog.Error("Failed to upload file:", mlog.Err(appErr)) + mlog.Error("Failed to upload file", mlog.Err(appErr), mlog.String("file_name", name)) return nil, appErr } @@ -1358,6 +1406,8 @@ func (a *App) importMultiplePostLines(c request.CTX, lines []imports.LineImportW return 0, nil } + c.Logger().Info("Validating post lines", mlog.Int("count", len(lines)), mlog.Int("first_line", lines[0].LineNumber)) + for _, line := range lines { if err := imports.ValidatePostImportData(line.Post, a.MaxPostSize()); err != nil { return line.LineNumber, err @@ -1369,6 +1419,8 @@ func (a *App) importMultiplePostLines(c request.CTX, lines []imports.LineImportW return 0, nil } + c.Logger().Info("Importing post lines", mlog.Int("count", len(lines)), mlog.Int("first_line", lines[0].LineNumber)) + usernames := []string{} teamNames := make([]string, len(lines)) postsData := make([]*imports.PostImportData, len(lines)) @@ -1855,7 +1907,13 @@ func (a *App) importMultipleDirectPostLines(c request.CTX, lines []imports.LineI return 0, nil } -func (a *App) importEmoji(data *imports.EmojiImportData, dryRun bool) *model.AppError { +func (a *App) importEmoji(c request.CTX, data *imports.EmojiImportData, dryRun bool) *model.AppError { + var fields []logr.Field + if data != nil && data.Name != nil { + fields = append(fields, mlog.String("emoji_name", *data.Name)) + } + c.Logger().Info("Validating emoji", fields...) + aerr := imports.ValidateEmojiImportData(data) if aerr != nil { if aerr.Id == "model.emoji.system_emoji_name.app_error" { @@ -1870,6 +1928,8 @@ func (a *App) importEmoji(data *imports.EmojiImportData, dryRun bool) *model.App return nil } + c.Logger().Info("Importing emoji", fields...) + var emoji *model.Emoji emoji, err := a.Srv().Store().Emoji().GetByName(context.Background(), *data.Name, true) diff --git a/app/import_functions_test.go b/app/import_functions_test.go index bb2271eeb6..864eef4a7f 100644 --- a/app/import_functions_test.go +++ b/app/import_functions_test.go @@ -65,7 +65,7 @@ func TestImportImportScheme(t *testing.T) { Description: ptrStr("description"), } - err := th.App.importScheme(&data, true) + err := th.App.importScheme(th.Context, &data, true) require.NotNil(t, err, "Should have failed to import.") _, nErr := th.App.Srv().Store().Scheme().GetByName(*data.Name) @@ -74,7 +74,7 @@ func TestImportImportScheme(t *testing.T) { // Try importing a valid scheme in dryRun mode. data.DisplayName = ptrStr("display name") - err = th.App.importScheme(&data, true) + err = th.App.importScheme(th.Context, &data, true) require.Nil(t, err, "Should have succeeded.") _, nErr = th.App.Srv().Store().Scheme().GetByName(*data.Name) @@ -83,7 +83,7 @@ func TestImportImportScheme(t *testing.T) { // Try importing an invalid scheme. data.DisplayName = nil - err = th.App.importScheme(&data, false) + err = th.App.importScheme(th.Context, &data, false) require.NotNil(t, err, "Should have failed to import.") _, nErr = th.App.Srv().Store().Scheme().GetByName(*data.Name) @@ -92,7 +92,7 @@ func TestImportImportScheme(t *testing.T) { // Try importing a valid scheme with all params set. data.DisplayName = ptrStr("display name") - err = th.App.importScheme(&data, false) + err = th.App.importScheme(th.Context, &data, false) require.Nil(t, err, "Should have succeeded.") scheme, nErr := th.App.Srv().Store().Scheme().GetByName(*data.Name) @@ -149,7 +149,7 @@ func TestImportImportScheme(t *testing.T) { data.DisplayName = ptrStr("new display name") data.Description = ptrStr("new description") - err = th.App.importScheme(&data, false) + err = th.App.importScheme(th.Context, &data, false) require.Nil(t, err, "Should have succeeded: %v", err) scheme, nErr = th.App.Srv().Store().Scheme().GetByName(*data.Name) @@ -205,7 +205,7 @@ func TestImportImportScheme(t *testing.T) { // Try changing the scope of the scheme and reimporting. data.Scope = ptrStr("channel") - err = th.App.importScheme(&data, false) + err = th.App.importScheme(th.Context, &data, false) require.NotNil(t, err, "Should have failed to import.") scheme, nErr = th.App.Srv().Store().Scheme().GetByName(*data.Name) @@ -252,7 +252,7 @@ func TestImportImportSchemeWithoutGuestRoles(t *testing.T) { Description: ptrStr("description"), } - err := th.App.importScheme(&data, true) + err := th.App.importScheme(th.Context, &data, true) require.NotNil(t, err, "Should have failed to import.") _, nErr := th.App.Srv().Store().Scheme().GetByName(*data.Name) @@ -261,7 +261,7 @@ func TestImportImportSchemeWithoutGuestRoles(t *testing.T) { // Try importing a valid scheme in dryRun mode. data.DisplayName = ptrStr("display name") - err = th.App.importScheme(&data, true) + err = th.App.importScheme(th.Context, &data, true) require.Nil(t, err, "Should have succeeded.") _, nErr = th.App.Srv().Store().Scheme().GetByName(*data.Name) @@ -270,7 +270,7 @@ func TestImportImportSchemeWithoutGuestRoles(t *testing.T) { // Try importing an invalid scheme. data.DisplayName = nil - err = th.App.importScheme(&data, false) + err = th.App.importScheme(th.Context, &data, false) require.NotNil(t, err, "Should have failed to import.") _, nErr = th.App.Srv().Store().Scheme().GetByName(*data.Name) @@ -279,7 +279,7 @@ func TestImportImportSchemeWithoutGuestRoles(t *testing.T) { // Try importing a valid scheme with all params set. data.DisplayName = ptrStr("display name") - err = th.App.importScheme(&data, false) + err = th.App.importScheme(th.Context, &data, false) require.Nil(t, err, "Should have succeeded.") scheme, nErr := th.App.Srv().Store().Scheme().GetByName(*data.Name) @@ -336,7 +336,7 @@ func TestImportImportSchemeWithoutGuestRoles(t *testing.T) { data.DisplayName = ptrStr("new display name") data.Description = ptrStr("new description") - err = th.App.importScheme(&data, false) + err = th.App.importScheme(th.Context, &data, false) require.Nil(t, err, "Should have succeeded: %v", err) scheme, nErr = th.App.Srv().Store().Scheme().GetByName(*data.Name) @@ -392,7 +392,7 @@ func TestImportImportSchemeWithoutGuestRoles(t *testing.T) { // Try changing the scope of the scheme and reimporting. data.Scope = ptrStr("channel") - err = th.App.importScheme(&data, false) + err = th.App.importScheme(th.Context, &data, false) require.NotNil(t, err, "Should have failed to import.") scheme, nErr = th.App.Srv().Store().Scheme().GetByName(*data.Name) @@ -414,7 +414,7 @@ func TestImportImportRole(t *testing.T) { Name: &rid1, } - err := th.App.importRole(&data, true, false) + err := th.App.importRole(th.Context, &data, true, false) require.NotNil(t, err, "Should have failed to import.") _, nErr := th.App.Srv().Store().Role().GetByName(context.Background(), rid1) @@ -423,7 +423,7 @@ func TestImportImportRole(t *testing.T) { // Try importing the valid role in dryRun mode. data.DisplayName = ptrStr("display name") - err = th.App.importRole(&data, true, false) + err = th.App.importRole(th.Context, &data, true, false) require.Nil(t, err, "Should have succeeded.") _, nErr = th.App.Srv().Store().Role().GetByName(context.Background(), rid1) @@ -432,7 +432,7 @@ func TestImportImportRole(t *testing.T) { // Try importing an invalid role. data.DisplayName = nil - err = th.App.importRole(&data, false, false) + err = th.App.importRole(th.Context, &data, false, false) require.NotNil(t, err, "Should have failed to import.") _, nErr = th.App.Srv().Store().Role().GetByName(context.Background(), rid1) @@ -443,7 +443,7 @@ func TestImportImportRole(t *testing.T) { data.Description = ptrStr("description") data.Permissions = &[]string{"invite_user", "add_user_to_team"} - err = th.App.importRole(&data, false, false) + err = th.App.importRole(th.Context, &data, false, false) require.Nil(t, err, "Should have succeeded.") role, nErr := th.App.Srv().Store().Role().GetByName(context.Background(), rid1) @@ -461,7 +461,7 @@ func TestImportImportRole(t *testing.T) { data.Description = ptrStr("description") data.Permissions = &[]string{"use_slash_commands"} - err = th.App.importRole(&data, false, true) + err = th.App.importRole(th.Context, &data, false, true) require.Nil(t, err, "Should have succeeded. %v", err) role, nErr = th.App.Srv().Store().Role().GetByName(context.Background(), rid1) @@ -480,7 +480,7 @@ func TestImportImportRole(t *testing.T) { DisplayName: ptrStr("new display name again"), } - err = th.App.importRole(&data2, false, false) + err = th.App.importRole(th.Context, &data2, false, false) require.Nil(t, err, "Should have succeeded.") role, nErr = th.App.Srv().Store().Role().GetByName(context.Background(), rid1) @@ -1384,7 +1384,7 @@ func TestImportImportUser(t *testing.T) { Description: ptrStr("description"), } - appErr = th.App.importScheme(teamSchemeData, false) + appErr = th.App.importScheme(th.Context, teamSchemeData, false) assert.Nil(t, appErr) teamScheme, nErr := th.App.Srv().Store().Scheme().GetByName(*teamSchemeData.Name) @@ -4151,7 +4151,7 @@ func TestImportImportEmoji(t *testing.T) { testImage := filepath.Join(testsDir, "test.png") data := imports.EmojiImportData{Name: ptrStr(model.NewId())} - appErr := th.App.importEmoji(&data, true) + appErr := th.App.importEmoji(th.Context, &data, true) assert.NotNil(t, appErr, "Invalid emoji should have failed dry run") emoji, nErr := th.App.Srv().Store().Emoji().GetByName(context.Background(), *data.Name, true) @@ -4159,35 +4159,35 @@ func TestImportImportEmoji(t *testing.T) { assert.Error(t, nErr) data.Image = ptrStr(testImage) - appErr = th.App.importEmoji(&data, true) + appErr = th.App.importEmoji(th.Context, &data, true) assert.Nil(t, appErr, "Valid emoji should have passed dry run") data = imports.EmojiImportData{Name: ptrStr(model.NewId())} - appErr = th.App.importEmoji(&data, false) + appErr = th.App.importEmoji(th.Context, &data, false) assert.NotNil(t, appErr, "Invalid emoji should have failed apply mode") data.Image = ptrStr("non-existent-file") - appErr = th.App.importEmoji(&data, false) + appErr = th.App.importEmoji(th.Context, &data, false) assert.NotNil(t, appErr, "Emoji with bad image file should have failed apply mode") data.Image = ptrStr(testImage) - appErr = th.App.importEmoji(&data, false) + appErr = th.App.importEmoji(th.Context, &data, false) assert.Nil(t, appErr, "Valid emoji should have succeeded apply mode") emoji, nErr = th.App.Srv().Store().Emoji().GetByName(context.Background(), *data.Name, true) assert.NotNil(t, emoji, "Emoji should have been imported") assert.NoError(t, nErr, "Emoji should have been imported without any error") - appErr = th.App.importEmoji(&data, false) + appErr = th.App.importEmoji(th.Context, &data, false) assert.Nil(t, appErr, "Second run should have succeeded apply mode") data = imports.EmojiImportData{Name: ptrStr("smiley"), Image: ptrStr(testImage)} - appErr = th.App.importEmoji(&data, false) + appErr = th.App.importEmoji(th.Context, &data, false) assert.Nil(t, appErr, "System emoji should not fail") largeImage := filepath.Join(testsDir, "large_image_file.jpg") data = imports.EmojiImportData{Name: ptrStr(model.NewId()), Image: ptrStr(largeImage)} - appErr = th.App.importEmoji(&data, false) + appErr = th.App.importEmoji(th.Context, &data, false) require.NotNil(t, appErr) require.ErrorIs(t, appErr.Unwrap(), utils.SizeLimitExceeded) } diff --git a/app/import_test.go b/app/import_test.go index 0ddb6834ec..60f8778165 100644 --- a/app/import_test.go +++ b/app/import_test.go @@ -17,7 +17,9 @@ import ( "github.com/stretchr/testify/require" "github.com/mattermost/mattermost-server/v6/app/imports" + "github.com/mattermost/mattermost-server/v6/app/request" "github.com/mattermost/mattermost-server/v6/model" + "github.com/mattermost/mattermost-server/v6/shared/mlog" "github.com/mattermost/mattermost-server/v6/utils" "github.com/mattermost/mattermost-server/v6/utils/fileutils" ) @@ -238,7 +240,7 @@ func TestImportBulkImport(t *testing.T) { {"type": "user", "user": {"username": "` + username + `", "email": "` + username + `@example.com", "teams": [{"name": "` + teamName + `","theme": "` + teamTheme1 + `", "channels": [{"name": "` + channelName + `"}]}]}} {"type": "post", "post": {"team": "` + teamName + `", "channel": "` + channelName + `", "user": "` + username + `", "message": "Hello World", "create_at": 123456789012, "attachments":[{"path": "` + testImage + `"}], "props":{"attachments":[{"id":0,"fallback":"[February 4th, 2020 2:46 PM] author: fallback","color":"D0D0D0","pretext":"","author_name":"author","author_link":"","title":"","title_link":"","text":"this post has props","fields":null,"image_url":"","thumb_url":"","footer":"Posted in #general","footer_icon":"","ts":"1580823992.000100"}]}}} {"type": "direct_channel", "direct_channel": {"members": ["` + username + `", "` + username + `"]}} -{"type": "direct_post", "direct_post": {"channel_members": ["` + username + `", "` + username + `"], "user": "` + username + `", "message": "Hello Direct Channel to myself", "create_at": 123456789014, "props":{"attachments":[{"id":0,"fallback":"[February 4th, 2020 2:46 PM] author: fallback","color":"D0D0D0","pretext":"","author_name":"author","author_link":"","title":"","title_link":"","text":"this post has props","fields":null,"image_url":"","thumb_url":"","footer":"Posted in #general","footer_icon":"","ts":"1580823992.000100"}]}}}}` +{"type": "direct_post", "direct_post": {"channel_members": ["` + username + `", "` + username + `"], "user": "` + username + `", "message": "Hello Direct Channel to myself", "create_at": 123456789014, "props":{"attachments":[{"id":0,"fallback":"[February 4th, 2020 2:46 PM] author: fallback","color":"D0D0D0","pretext":"","author_name":"author","author_link":"","title":"","title_link":"","text":"this post has props","fields":null,"image_url":"","thumb_url":"","footer":"Posted in #general","footer_icon":"","ts":"1580823992.000100"}]}}}` err, line := th.App.BulkImport(th.Context, strings.NewReader(data6), nil, false, 2) require.Nil(t, err, "BulkImport should have succeeded") @@ -285,6 +287,9 @@ func AssertFileIdsInPost(files []*model.FileInfo, th *TestHelper, t *testing.T) } func TestProcessAttachments(t *testing.T) { + logger, _ := mlog.NewLogger() + c := request.EmptyContext(logger) + genAttachments := func() *[]imports.AttachmentImportData { return &[]imports.AttachmentImportData{ { @@ -333,10 +338,11 @@ func TestProcessAttachments(t *testing.T) { Path: model.NewString("somedir/file.jpg"), }, } - err := processAttachments(&line, "", nil) + + err := processAttachments(c, &line, "", nil) require.NoError(t, err) require.Equal(t, expected, line.Post.Attachments) - err = processAttachments(&line2, "", nil) + err = processAttachments(c, &line2, "", nil) require.NoError(t, err) require.Equal(t, expected, line2.DirectPost.Attachments) }) @@ -352,27 +358,27 @@ func TestProcessAttachments(t *testing.T) { } t.Run("post attachments", func(t *testing.T) { - err := processAttachments(&line, "/tmp", nil) + err := processAttachments(c, &line, "/tmp", nil) require.NoError(t, err) require.Equal(t, expected, line.Post.Attachments) }) t.Run("direct post attachments", func(t *testing.T) { - err := processAttachments(&line2, "/tmp", nil) + err := processAttachments(c, &line2, "/tmp", nil) require.NoError(t, err) require.Equal(t, expected, line2.DirectPost.Attachments) }) t.Run("profile image", func(t *testing.T) { expected := "/tmp/profile.jpg" - err := processAttachments(&userLine, "/tmp", nil) + err := processAttachments(c, &userLine, "/tmp", nil) require.NoError(t, err) require.Equal(t, expected, *userLine.User.ProfileImage) }) t.Run("emoji", func(t *testing.T) { expected := "/tmp/emoji.png" - err := processAttachments(&emojiLine, "/tmp", nil) + err := processAttachments(c, &emojiLine, "/tmp", nil) require.NoError(t, err) require.Equal(t, expected, *emojiLine.Emoji.Image) }) @@ -383,11 +389,11 @@ func TestProcessAttachments(t *testing.T) { filesMap := map[string]*zip.File{ "/tmp/file.jpg": nil, } - err := processAttachments(&line, "", filesMap) + err := processAttachments(c, &line, "", filesMap) require.Error(t, err) filesMap["/tmp/somedir/file.jpg"] = nil - err = processAttachments(&line, "", filesMap) + err = processAttachments(c, &line, "", filesMap) require.NoError(t, err) }) @@ -395,11 +401,11 @@ func TestProcessAttachments(t *testing.T) { filesMap := map[string]*zip.File{ "/tmp/file.jpg": nil, } - err := processAttachments(&line2, "", filesMap) + err := processAttachments(c, &line2, "", filesMap) require.Error(t, err) filesMap["/tmp/somedir/file.jpg"] = nil - err = processAttachments(&line2, "", filesMap) + err = processAttachments(c, &line2, "", filesMap) require.NoError(t, err) }) @@ -407,11 +413,11 @@ func TestProcessAttachments(t *testing.T) { filesMap := map[string]*zip.File{ "/tmp/file.jpg": nil, } - err := processAttachments(&userLine, "", filesMap) + err := processAttachments(c, &userLine, "", filesMap) require.Error(t, err) filesMap["/tmp/profile.jpg"] = nil - err = processAttachments(&userLine, "", filesMap) + err = processAttachments(c, &userLine, "", filesMap) require.NoError(t, err) }) @@ -419,11 +425,11 @@ func TestProcessAttachments(t *testing.T) { filesMap := map[string]*zip.File{ "/tmp/file.jpg": nil, } - err := processAttachments(&emojiLine, "", filesMap) + err := processAttachments(c, &emojiLine, "", filesMap) require.Error(t, err) filesMap["/tmp/emoji.png"] = nil - err = processAttachments(&emojiLine, "", filesMap) + err = processAttachments(c, &emojiLine, "", filesMap) require.NoError(t, err) }) })