[MM-46417] Added more logging to the import (#21764)

app/export_test.go

@@ -185,7 +185,7 @@ func TestExportAllUsers(t *testing.T) {
 	defer th2.TearDown()
 	err, i := th2.App.BulkImport(th2.Context, &b, nil, false, 5)
 	assert.Nil(t, err)
-	assert.Equal(t, 0, i)
+	assert.EqualValues(t, 0, i)
 
 	users1, err := th1.App.GetUsersFromProfiles(&model.UserGetOptions{
 		Page:    0,
@@ -323,7 +323,7 @@ func TestExportDMChannelToSelf(t *testing.T) {
 	// import the exported channel
 	err, i := th2.App.BulkImport(th2.Context, &b, nil, false, 5)
 	assert.Nil(t, err)
-	assert.Equal(t, 0, i)
+	assert.EqualValues(t, 0, i)
 
 	channels, nErr = th2.App.Srv().Store().Channel().GetAllDirectChannelsForExportAfter(1000, "00000000")
 	require.NoError(t, nErr)

app/import.go

@@ -6,7 +6,6 @@ package app
 import (
 	"archive/zip"
 	"bufio"
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -26,6 +25,7 @@ type ReactionImportData = imports.ReactionImportData // part of the app interfac
 const (
 	importMultiplePostsThreshold = 1000
 	maxScanTokenSize             = 16 * 1024 * 1024 // Need to set a higher limit than default because some customers cross the limit. See MM-22314
+	statusUpdateAfterLines       = 8192
 )
 
 func stopOnError(c request.CTX, err imports.LineImportWorkerError) bool {
@@ -41,7 +41,7 @@ func stopOnError(c request.CTX, err imports.LineImportWorkerError) bool {
 	}
 }
 
-func processAttachmentPaths(files *[]imports.AttachmentImportData, basePath string, filesMap map[string]*zip.File) error {
+func processAttachmentPaths(c request.CTX, files *[]imports.AttachmentImportData, basePath string, filesMap map[string]*zip.File) error {
 	if files == nil {
 		return nil
 	}
@@ -61,20 +61,20 @@ func processAttachmentPaths(files *[]imports.AttachmentImportData, basePath stri
 	return nil
 }
 
-func processAttachments(line *imports.LineImportData, basePath string, filesMap map[string]*zip.File) error {
+func processAttachments(c request.CTX, line *imports.LineImportData, basePath string, filesMap map[string]*zip.File) error {
 	var ok bool
 	switch line.Type {
 	case "post", "direct_post":
 		var replies []imports.ReplyImportData
 		if line.Type == "direct_post" {
-			if err := processAttachmentPaths(line.DirectPost.Attachments, basePath, filesMap); err != nil {
+			if err := processAttachmentPaths(c, line.DirectPost.Attachments, basePath, filesMap); err != nil {
 				return err
 			}
 			if line.DirectPost.Replies != nil {
 				replies = *line.DirectPost.Replies
 			}
 		} else {
-			if err := processAttachmentPaths(line.Post.Attachments, basePath, filesMap); err != nil {
+			if err := processAttachmentPaths(c, line.Post.Attachments, basePath, filesMap); err != nil {
 				return err
 			}
 			if line.Post.Replies != nil {
@@ -82,7 +82,7 @@ func processAttachments(line *imports.LineImportData, basePath string, filesMap
 			}
 		}
 		for _, reply := range replies {
-			if err := processAttachmentPaths(reply.Attachments, basePath, filesMap); err != nil {
+			if err := processAttachmentPaths(c, reply.Attachments, basePath, filesMap); err != nil {
 				return err
 			}
 		}
@@ -112,6 +112,15 @@ func processAttachments(line *imports.LineImportData, basePath string, filesMap
 }
 
 func (a *App) bulkImportWorker(c request.CTX, dryRun bool, wg *sync.WaitGroup, lines <-chan imports.LineImportWorkerData, errors chan<- imports.LineImportWorkerError) {
+	workerID := model.NewId()
+	processedLines := uint64(0)
+
+	c.Logger().Info("Started new bulk import worker", mlog.String("bulk_import_worker_id", workerID))
+	defer func() {
+		wg.Done()
+		c.Logger().Info("Bulk import worker finished", mlog.String("bulk_import_worker_id", workerID), mlog.Uint64("processed_lines", processedLines))
+	}()
+
 	postLines := []imports.LineImportWorkerData{}
 	directPostLines := []imports.LineImportWorkerData{}
 	for line := range lines {
@@ -143,6 +152,11 @@ func (a *App) bulkImportWorker(c request.CTX, dryRun bool, wg *sync.WaitGroup, l
 				errors <- imports.LineImportWorkerError{Error: err, LineNumber: line.LineNumber}
 			}
 		}
+
+		processedLines++
+		if processedLines%statusUpdateAfterLines == 0 {
+			c.Logger().Info("Worker progress", mlog.String("bulk_import_worker_id", workerID), mlog.Uint64("processed_lines", processedLines))
+		}
 	}
 
 	if len(postLines) > 0 {
@@ -155,7 +169,6 @@ func (a *App) bulkImportWorker(c request.CTX, dryRun bool, wg *sync.WaitGroup, l
 			errors <- imports.LineImportWorkerError{Error: err, LineNumber: errLine}
 		}
 	}
-	wg.Done()
 }
 
 func (a *App) BulkImport(c *request.Context, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun bool, workers int) (*model.AppError, int) {
@@ -194,15 +207,17 @@ func (a *App) bulkImport(c request.CTX, jsonlReader io.Reader, attachmentsReader
 	}
 
 	for scanner.Scan() {
-		decoder := json.NewDecoder(bytes.NewReader(scanner.Bytes()))
 		lineNumber++
+		if lineNumber%statusUpdateAfterLines == 0 {
+			c.Logger().Info("Reader progress", mlog.Int("processed_lines", lineNumber))
+		}
 
 		var line imports.LineImportData
-		if err := decoder.Decode(&line); err != nil {
+		if err := json.Unmarshal(scanner.Bytes(), &line); err != nil {
 			return model.NewAppError("BulkImport", "app.import.bulk_import.json_decode.error", nil, "", http.StatusBadRequest).Wrap(err), lineNumber
 		}
 
-		if err := processAttachments(&line, importPath, attachedFiles); err != nil {
+		if err := processAttachments(c, &line, importPath, attachedFiles); err != nil {
 			c.Logger().Warn("Error while processing import attachments. Objects might be broken.", mlog.Err(err))
 		}
 
@@ -222,6 +237,12 @@ func (a *App) bulkImport(c request.CTX, jsonlReader io.Reader, attachmentsReader
 		if line.Type != lastLineType {
 			// Only clear the worker queue if is not the first data entry
 			if lineNumber != 2 {
+				c.Logger().Info(
+					"Finished parsing segment, waiting for workers to finish",
+					mlog.String("old_segment", lastLineType),
+					mlog.String("new_segment", line.Type),
+				)
+
 				// Changing type. Clear out the worker queue before continuing.
 				close(linesChan)
 				wg.Wait()
@@ -235,6 +256,13 @@ func (a *App) bulkImport(c request.CTX, jsonlReader io.Reader, attachmentsReader
 				}
 			}
 
+			c.Logger().Info(
+				"Starting workers for new segment",
+				mlog.String("old_segment", lastLineType),
+				mlog.String("new_segment", line.Type),
+				mlog.Int("workers", workers),
+			)
+
 			// Set up the workers and channel for this type.
 			lastLineType = line.Type
 			linesChan = make(chan imports.LineImportWorkerData, workers)
@@ -290,7 +318,7 @@ func (a *App) importLine(c request.CTX, line imports.LineImportData, dryRun bool
 		if line.Scheme == nil {
 			return model.NewAppError("BulkImport", "app.import.import_line.null_scheme.error", nil, "", http.StatusBadRequest)
 		}
-		return a.importScheme(line.Scheme, dryRun)
+		return a.importScheme(c, line.Scheme, dryRun)
 	case line.Type == "team":
 		if line.Team == nil {
 			return model.NewAppError("BulkImport", "app.import.import_line.null_team.error", nil, "", http.StatusBadRequest)
@@ -315,7 +343,7 @@ func (a *App) importLine(c request.CTX, line imports.LineImportData, dryRun bool
 		if line.Emoji == nil {
 			return model.NewAppError("BulkImport", "app.import.import_line.null_emoji.error", nil, "", http.StatusBadRequest)
 		}
-		return a.importEmoji(line.Emoji, dryRun)
+		return a.importEmoji(c, line.Emoji, dryRun)
 	default:
 		return model.NewAppError("BulkImport", "app.import.import_line.unknown_line_type.error", map[string]any{"Type": line.Type}, "", http.StatusBadRequest)
 	}

app/import_functions.go

@@ -15,6 +15,7 @@ import (
 	"path"
 	"strings"
 
+	"github.com/mattermost/logr/v2"
 	"github.com/mattermost/mattermost-server/v6/app/imports"
 	"github.com/mattermost/mattermost-server/v6/app/request"
 	"github.com/mattermost/mattermost-server/v6/app/teams"
@@ -25,13 +26,16 @@ import (
 	"github.com/mattermost/mattermost-server/v6/utils"
 )
 
-//
 // -- Bulk Import Functions --
 // These functions import data directly into the database. Security and permission checks are bypassed but validity is
 // still enforced.
-//
+func (a *App) importScheme(c request.CTX, data *imports.SchemeImportData, dryRun bool) *model.AppError {
+	var fields []logr.Field
+	if data != nil && data.Name != nil {
+		fields = append(fields, mlog.String("schema_name", *data.Name))
+	}
+	c.Logger().Info("Validating schema", fields...)
 
-func (a *App) importScheme(data *imports.SchemeImportData, dryRun bool) *model.AppError {
 	if err := imports.ValidateSchemeImportData(data); err != nil {
 		return err
 	}
@@ -41,6 +45,8 @@ func (a *App) importScheme(data *imports.SchemeImportData, dryRun bool) *model.A
 		return nil
 	}
 
+	c.Logger().Info("Importing schema", fields...)
+
 	scheme, err := a.GetSchemeByName(*data.Name)
 	if err != nil {
 		scheme = new(model.Scheme)
@@ -68,12 +74,12 @@ func (a *App) importScheme(data *imports.SchemeImportData, dryRun bool) *model.A
 
 	if scheme.Scope == model.SchemeScopeTeam {
 		data.DefaultTeamAdminRole.Name = &scheme.DefaultTeamAdminRole
-		if err := a.importRole(data.DefaultTeamAdminRole, dryRun, true); err != nil {
+		if err := a.importRole(c, data.DefaultTeamAdminRole, dryRun, true); err != nil {
 			return err
 		}
 
 		data.DefaultTeamUserRole.Name = &scheme.DefaultTeamUserRole
-		if err := a.importRole(data.DefaultTeamUserRole, dryRun, true); err != nil {
+		if err := a.importRole(c, data.DefaultTeamUserRole, dryRun, true); err != nil {
 			return err
 		}
 
@@ -83,19 +89,19 @@ func (a *App) importScheme(data *imports.SchemeImportData, dryRun bool) *model.A
 			}
 		}
 		data.DefaultTeamGuestRole.Name = &scheme.DefaultTeamGuestRole
-		if err := a.importRole(data.DefaultTeamGuestRole, dryRun, true); err != nil {
+		if err := a.importRole(c, data.DefaultTeamGuestRole, dryRun, true); err != nil {
 			return err
 		}
 	}
 
 	if scheme.Scope == model.SchemeScopeTeam || scheme.Scope == model.SchemeScopeChannel {
 		data.DefaultChannelAdminRole.Name = &scheme.DefaultChannelAdminRole
-		if err := a.importRole(data.DefaultChannelAdminRole, dryRun, true); err != nil {
+		if err := a.importRole(c, data.DefaultChannelAdminRole, dryRun, true); err != nil {
 			return err
 		}
 
 		data.DefaultChannelUserRole.Name = &scheme.DefaultChannelUserRole
-		if err := a.importRole(data.DefaultChannelUserRole, dryRun, true); err != nil {
+		if err := a.importRole(c, data.DefaultChannelUserRole, dryRun, true); err != nil {
 			return err
 		}
 
@@ -105,7 +111,7 @@ func (a *App) importScheme(data *imports.SchemeImportData, dryRun bool) *model.A
 			}
 		}
 		data.DefaultChannelGuestRole.Name = &scheme.DefaultChannelGuestRole
-		if err := a.importRole(data.DefaultChannelGuestRole, dryRun, true); err != nil {
+		if err := a.importRole(c, data.DefaultChannelGuestRole, dryRun, true); err != nil {
 			return err
 		}
 	}
@@ -113,8 +119,15 @@ func (a *App) importScheme(data *imports.SchemeImportData, dryRun bool) *model.A
 	return nil
 }
 
-func (a *App) importRole(data *imports.RoleImportData, dryRun bool, isSchemeRole bool) *model.AppError {
+func (a *App) importRole(c request.CTX, data *imports.RoleImportData, dryRun bool, isSchemeRole bool) *model.AppError {
+	var fields []logr.Field
+	if data != nil && data.Name != nil {
+		fields = append(fields, mlog.String("role_name", *data.Name))
+	}
+
 	if !isSchemeRole {
+		c.Logger().Info("Validating role", fields...)
+
 		if err := imports.ValidateRoleImportData(data); err != nil {
 			return err
 		}
@@ -125,6 +138,8 @@ func (a *App) importRole(data *imports.RoleImportData, dryRun bool, isSchemeRole
 		return nil
 	}
 
+	c.Logger().Info("Importing role", fields...)
+
 	role, err := a.GetRoleByName(context.Background(), *data.Name)
 	if err != nil {
 		role = new(model.Role)
@@ -160,6 +175,12 @@ func (a *App) importRole(data *imports.RoleImportData, dryRun bool, isSchemeRole
 }
 
 func (a *App) importTeam(c request.CTX, data *imports.TeamImportData, dryRun bool) *model.AppError {
+	var fields []logr.Field
+	if data != nil && data.Name != nil {
+		fields = append(fields, mlog.String("team_name", *data.Name))
+	}
+	c.Logger().Info("Validating team", fields...)
+
 	if err := imports.ValidateTeamImportData(data); err != nil {
 		return err
 	}
@@ -169,6 +190,8 @@ func (a *App) importTeam(c request.CTX, data *imports.TeamImportData, dryRun boo
 		return nil
 	}
 
+	c.Logger().Info("Importing team", fields...)
+
 	var team *model.Team
 	team, err := a.Srv().Store().Team().GetByName(*data.Name)
 
@@ -228,6 +251,12 @@ func (a *App) importTeam(c request.CTX, data *imports.TeamImportData, dryRun boo
 }
 
 func (a *App) importChannel(c request.CTX, data *imports.ChannelImportData, dryRun bool) *model.AppError {
+	var fields []logr.Field
+	if data != nil && data.Name != nil {
+		fields = append(fields, mlog.String("channel_name", *data.Name))
+	}
+	c.Logger().Info("Validating channel", fields...)
+
 	if err := imports.ValidateChannelImportData(data); err != nil {
 		return err
 	}
@@ -237,6 +266,8 @@ func (a *App) importChannel(c request.CTX, data *imports.ChannelImportData, dryR
 		return nil
 	}
 
+	c.Logger().Info("Importing channel", fields...)
+
 	team, err := a.Srv().Store().Team().GetByName(*data.Team)
 	if err != nil {
 		return model.NewAppError("BulkImport", "app.import.import_channel.team_not_found.error", map[string]any{"TeamName": *data.Team}, "", http.StatusBadRequest).Wrap(err)
@@ -293,6 +324,12 @@ func (a *App) importChannel(c request.CTX, data *imports.ChannelImportData, dryR
 }
 
 func (a *App) importUser(c request.CTX, data *imports.UserImportData, dryRun bool) *model.AppError {
+	var fields []logr.Field
+	if data != nil && data.Username != nil {
+		fields = append(fields, mlog.String("user_name", *data.Username))
+	}
+	c.Logger().Info("Validating user", fields...)
+
 	if err := imports.ValidateUserImportData(data); err != nil {
 		return err
 	}
@@ -302,6 +339,8 @@ func (a *App) importUser(c request.CTX, data *imports.UserImportData, dryRun boo
 		return nil
 	}
 
+	c.Logger().Info("Importing user", fields...)
+
 	// We want to avoid database writes if nothing has changed.
 	hasUserChanged := false
 	hasNotifyPropsChanged := false
@@ -1214,6 +1253,8 @@ func (a *App) importAttachment(c request.CTX, data *imports.AttachmentImportData
 		defer zipFile.Close()
 		name = data.Data.Name
 		file = zipFile.(io.Reader)
+
+		c.Logger().Info("Preparing file upload from ZIP", mlog.String("file_name", name), mlog.Uint64("file_size", data.Data.UncompressedSize64))
 	} else {
 		realFile, err := os.Open(*data.Path)
 		if err != nil {
@@ -1222,6 +1263,12 @@ func (a *App) importAttachment(c request.CTX, data *imports.AttachmentImportData
 		defer realFile.Close()
 		name = realFile.Name()
 		file = realFile
+
+		fields := []logr.Field{mlog.String("file_name", name)}
+		if info, err := realFile.Stat(); err != nil {
+			fields = append(fields, mlog.Int64("file_size", info.Size()))
+		}
+		c.Logger().Info("Preparing file upload from file system", fields...)
 	}
 
 	timestamp := utils.TimeFromMillis(post.CreateAt)
@@ -1241,7 +1288,8 @@ func (a *App) importAttachment(c request.CTX, data *imports.AttachmentImportData
 			if oldFile.Name != path.Base(name) || oldFile.Size != int64(len(fileData)) {
 				continue
 			}
-			// check md5
+
+			// check sha1
 			newHash := sha1.Sum(fileData)
 			oldFileData, err := a.getFileIgnoreCloudLimit(oldFile.Id)
 			if err != nil {
@@ -1260,7 +1308,7 @@ func (a *App) importAttachment(c request.CTX, data *imports.AttachmentImportData
 
 	fileInfo, appErr := a.DoUploadFile(c, timestamp, teamID, post.ChannelId, post.UserId, name, fileData)
 	if appErr != nil {
-		mlog.Error("Failed to upload file:", mlog.Err(appErr))
+		mlog.Error("Failed to upload file", mlog.Err(appErr), mlog.String("file_name", name))
 		return nil, appErr
 	}
 
@@ -1358,6 +1406,8 @@ func (a *App) importMultiplePostLines(c request.CTX, lines []imports.LineImportW
 		return 0, nil
 	}
 
+	c.Logger().Info("Validating post lines", mlog.Int("count", len(lines)), mlog.Int("first_line", lines[0].LineNumber))
+
 	for _, line := range lines {
 		if err := imports.ValidatePostImportData(line.Post, a.MaxPostSize()); err != nil {
 			return line.LineNumber, err
@@ -1369,6 +1419,8 @@ func (a *App) importMultiplePostLines(c request.CTX, lines []imports.LineImportW
 		return 0, nil
 	}
 
+	c.Logger().Info("Importing post lines", mlog.Int("count", len(lines)), mlog.Int("first_line", lines[0].LineNumber))
+
 	usernames := []string{}
 	teamNames := make([]string, len(lines))
 	postsData := make([]*imports.PostImportData, len(lines))
@@ -1855,7 +1907,13 @@ func (a *App) importMultipleDirectPostLines(c request.CTX, lines []imports.LineI
 	return 0, nil
 }
 
-func (a *App) importEmoji(data *imports.EmojiImportData, dryRun bool) *model.AppError {
+func (a *App) importEmoji(c request.CTX, data *imports.EmojiImportData, dryRun bool) *model.AppError {
+	var fields []logr.Field
+	if data != nil && data.Name != nil {
+		fields = append(fields, mlog.String("emoji_name", *data.Name))
+	}
+	c.Logger().Info("Validating emoji", fields...)
+
 	aerr := imports.ValidateEmojiImportData(data)
 	if aerr != nil {
 		if aerr.Id == "model.emoji.system_emoji_name.app_error" {
@@ -1870,6 +1928,8 @@ func (a *App) importEmoji(data *imports.EmojiImportData, dryRun bool) *model.App
 		return nil
 	}
 
+	c.Logger().Info("Importing emoji", fields...)
+
 	var emoji *model.Emoji
 
 	emoji, err := a.Srv().Store().Emoji().GetByName(context.Background(), *data.Name, true)

app/import_functions_test.go

@@ -65,7 +65,7 @@ func TestImportImportScheme(t *testing.T) {
 		Description: ptrStr("description"),
 	}
 
-	err := th.App.importScheme(&data, true)
+	err := th.App.importScheme(th.Context, &data, true)
 	require.NotNil(t, err, "Should have failed to import.")
 
 	_, nErr := th.App.Srv().Store().Scheme().GetByName(*data.Name)
@@ -74,7 +74,7 @@ func TestImportImportScheme(t *testing.T) {
 	// Try importing a valid scheme in dryRun mode.
 	data.DisplayName = ptrStr("display name")
 
-	err = th.App.importScheme(&data, true)
+	err = th.App.importScheme(th.Context, &data, true)
 	require.Nil(t, err, "Should have succeeded.")
 
 	_, nErr = th.App.Srv().Store().Scheme().GetByName(*data.Name)
@@ -83,7 +83,7 @@ func TestImportImportScheme(t *testing.T) {
 	// Try importing an invalid scheme.
 	data.DisplayName = nil
 
-	err = th.App.importScheme(&data, false)
+	err = th.App.importScheme(th.Context, &data, false)
 	require.NotNil(t, err, "Should have failed to import.")
 
 	_, nErr = th.App.Srv().Store().Scheme().GetByName(*data.Name)
@@ -92,7 +92,7 @@ func TestImportImportScheme(t *testing.T) {
 	// Try importing a valid scheme with all params set.
 	data.DisplayName = ptrStr("display name")
 
-	err = th.App.importScheme(&data, false)
+	err = th.App.importScheme(th.Context, &data, false)
 	require.Nil(t, err, "Should have succeeded.")
 
 	scheme, nErr := th.App.Srv().Store().Scheme().GetByName(*data.Name)
@@ -149,7 +149,7 @@ func TestImportImportScheme(t *testing.T) {
 	data.DisplayName = ptrStr("new display name")
 	data.Description = ptrStr("new description")
 
-	err = th.App.importScheme(&data, false)
+	err = th.App.importScheme(th.Context, &data, false)
 	require.Nil(t, err, "Should have succeeded: %v", err)
 
 	scheme, nErr = th.App.Srv().Store().Scheme().GetByName(*data.Name)
@@ -205,7 +205,7 @@ func TestImportImportScheme(t *testing.T) {
 	// Try changing the scope of the scheme and reimporting.
 	data.Scope = ptrStr("channel")
 
-	err = th.App.importScheme(&data, false)
+	err = th.App.importScheme(th.Context, &data, false)
 	require.NotNil(t, err, "Should have failed to import.")
 
 	scheme, nErr = th.App.Srv().Store().Scheme().GetByName(*data.Name)
@@ -252,7 +252,7 @@ func TestImportImportSchemeWithoutGuestRoles(t *testing.T) {
 		Description: ptrStr("description"),
 	}
 
-	err := th.App.importScheme(&data, true)
+	err := th.App.importScheme(th.Context, &data, true)
 	require.NotNil(t, err, "Should have failed to import.")
 
 	_, nErr := th.App.Srv().Store().Scheme().GetByName(*data.Name)
@@ -261,7 +261,7 @@ func TestImportImportSchemeWithoutGuestRoles(t *testing.T) {
 	// Try importing a valid scheme in dryRun mode.
 	data.DisplayName = ptrStr("display name")
 
-	err = th.App.importScheme(&data, true)
+	err = th.App.importScheme(th.Context, &data, true)
 	require.Nil(t, err, "Should have succeeded.")
 
 	_, nErr = th.App.Srv().Store().Scheme().GetByName(*data.Name)
@@ -270,7 +270,7 @@ func TestImportImportSchemeWithoutGuestRoles(t *testing.T) {
 	// Try importing an invalid scheme.
 	data.DisplayName = nil
 
-	err = th.App.importScheme(&data, false)
+	err = th.App.importScheme(th.Context, &data, false)
 	require.NotNil(t, err, "Should have failed to import.")
 
 	_, nErr = th.App.Srv().Store().Scheme().GetByName(*data.Name)
@@ -279,7 +279,7 @@ func TestImportImportSchemeWithoutGuestRoles(t *testing.T) {
 	// Try importing a valid scheme with all params set.
 	data.DisplayName = ptrStr("display name")
 
-	err = th.App.importScheme(&data, false)
+	err = th.App.importScheme(th.Context, &data, false)
 	require.Nil(t, err, "Should have succeeded.")
 
 	scheme, nErr := th.App.Srv().Store().Scheme().GetByName(*data.Name)
@@ -336,7 +336,7 @@ func TestImportImportSchemeWithoutGuestRoles(t *testing.T) {
 	data.DisplayName = ptrStr("new display name")
 	data.Description = ptrStr("new description")
 
-	err = th.App.importScheme(&data, false)
+	err = th.App.importScheme(th.Context, &data, false)
 	require.Nil(t, err, "Should have succeeded: %v", err)
 
 	scheme, nErr = th.App.Srv().Store().Scheme().GetByName(*data.Name)
@@ -392,7 +392,7 @@ func TestImportImportSchemeWithoutGuestRoles(t *testing.T) {
 	// Try changing the scope of the scheme and reimporting.
 	data.Scope = ptrStr("channel")
 
-	err = th.App.importScheme(&data, false)
+	err = th.App.importScheme(th.Context, &data, false)
 	require.NotNil(t, err, "Should have failed to import.")
 
 	scheme, nErr = th.App.Srv().Store().Scheme().GetByName(*data.Name)
@@ -414,7 +414,7 @@ func TestImportImportRole(t *testing.T) {
 		Name: &rid1,
 	}
 
-	err := th.App.importRole(&data, true, false)
+	err := th.App.importRole(th.Context, &data, true, false)
 	require.NotNil(t, err, "Should have failed to import.")
 
 	_, nErr := th.App.Srv().Store().Role().GetByName(context.Background(), rid1)
@@ -423,7 +423,7 @@ func TestImportImportRole(t *testing.T) {
 	// Try importing the valid role in dryRun mode.
 	data.DisplayName = ptrStr("display name")
 
-	err = th.App.importRole(&data, true, false)
+	err = th.App.importRole(th.Context, &data, true, false)
 	require.Nil(t, err, "Should have succeeded.")
 
 	_, nErr = th.App.Srv().Store().Role().GetByName(context.Background(), rid1)
@@ -432,7 +432,7 @@ func TestImportImportRole(t *testing.T) {
 	// Try importing an invalid role.
 	data.DisplayName = nil
 
-	err = th.App.importRole(&data, false, false)
+	err = th.App.importRole(th.Context, &data, false, false)
 	require.NotNil(t, err, "Should have failed to import.")
 
 	_, nErr = th.App.Srv().Store().Role().GetByName(context.Background(), rid1)
@@ -443,7 +443,7 @@ func TestImportImportRole(t *testing.T) {
 	data.Description = ptrStr("description")
 	data.Permissions = &[]string{"invite_user", "add_user_to_team"}
 
-	err = th.App.importRole(&data, false, false)
+	err = th.App.importRole(th.Context, &data, false, false)
 	require.Nil(t, err, "Should have succeeded.")
 
 	role, nErr := th.App.Srv().Store().Role().GetByName(context.Background(), rid1)
@@ -461,7 +461,7 @@ func TestImportImportRole(t *testing.T) {
 	data.Description = ptrStr("description")
 	data.Permissions = &[]string{"use_slash_commands"}
 
-	err = th.App.importRole(&data, false, true)
+	err = th.App.importRole(th.Context, &data, false, true)
 	require.Nil(t, err, "Should have succeeded. %v", err)
 
 	role, nErr = th.App.Srv().Store().Role().GetByName(context.Background(), rid1)
@@ -480,7 +480,7 @@ func TestImportImportRole(t *testing.T) {
 		DisplayName: ptrStr("new display name again"),
 	}
 
-	err = th.App.importRole(&data2, false, false)
+	err = th.App.importRole(th.Context, &data2, false, false)
 	require.Nil(t, err, "Should have succeeded.")
 
 	role, nErr = th.App.Srv().Store().Role().GetByName(context.Background(), rid1)
@@ -1384,7 +1384,7 @@ func TestImportImportUser(t *testing.T) {
 		Description: ptrStr("description"),
 	}
 
-	appErr = th.App.importScheme(teamSchemeData, false)
+	appErr = th.App.importScheme(th.Context, teamSchemeData, false)
 	assert.Nil(t, appErr)
 
 	teamScheme, nErr := th.App.Srv().Store().Scheme().GetByName(*teamSchemeData.Name)
@@ -4151,7 +4151,7 @@ func TestImportImportEmoji(t *testing.T) {
 	testImage := filepath.Join(testsDir, "test.png")
 
 	data := imports.EmojiImportData{Name: ptrStr(model.NewId())}
-	appErr := th.App.importEmoji(&data, true)
+	appErr := th.App.importEmoji(th.Context, &data, true)
 	assert.NotNil(t, appErr, "Invalid emoji should have failed dry run")
 
 	emoji, nErr := th.App.Srv().Store().Emoji().GetByName(context.Background(), *data.Name, true)
@@ -4159,35 +4159,35 @@ func TestImportImportEmoji(t *testing.T) {
 	assert.Error(t, nErr)
 
 	data.Image = ptrStr(testImage)
-	appErr = th.App.importEmoji(&data, true)
+	appErr = th.App.importEmoji(th.Context, &data, true)
 	assert.Nil(t, appErr, "Valid emoji should have passed dry run")
 
 	data = imports.EmojiImportData{Name: ptrStr(model.NewId())}
-	appErr = th.App.importEmoji(&data, false)
+	appErr = th.App.importEmoji(th.Context, &data, false)
 	assert.NotNil(t, appErr, "Invalid emoji should have failed apply mode")
 
 	data.Image = ptrStr("non-existent-file")
-	appErr = th.App.importEmoji(&data, false)
+	appErr = th.App.importEmoji(th.Context, &data, false)
 	assert.NotNil(t, appErr, "Emoji with bad image file should have failed apply mode")
 
 	data.Image = ptrStr(testImage)
-	appErr = th.App.importEmoji(&data, false)
+	appErr = th.App.importEmoji(th.Context, &data, false)
 	assert.Nil(t, appErr, "Valid emoji should have succeeded apply mode")
 
 	emoji, nErr = th.App.Srv().Store().Emoji().GetByName(context.Background(), *data.Name, true)
 	assert.NotNil(t, emoji, "Emoji should have been imported")
 	assert.NoError(t, nErr, "Emoji should have been imported without any error")
 
-	appErr = th.App.importEmoji(&data, false)
+	appErr = th.App.importEmoji(th.Context, &data, false)
 	assert.Nil(t, appErr, "Second run should have succeeded apply mode")
 
 	data = imports.EmojiImportData{Name: ptrStr("smiley"), Image: ptrStr(testImage)}
-	appErr = th.App.importEmoji(&data, false)
+	appErr = th.App.importEmoji(th.Context, &data, false)
 	assert.Nil(t, appErr, "System emoji should not fail")
 
 	largeImage := filepath.Join(testsDir, "large_image_file.jpg")
 	data = imports.EmojiImportData{Name: ptrStr(model.NewId()), Image: ptrStr(largeImage)}
-	appErr = th.App.importEmoji(&data, false)
+	appErr = th.App.importEmoji(th.Context, &data, false)
 	require.NotNil(t, appErr)
 	require.ErrorIs(t, appErr.Unwrap(), utils.SizeLimitExceeded)
 }

app/import_test.go

@@ -17,7 +17,9 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/mattermost/mattermost-server/v6/app/imports"
+	"github.com/mattermost/mattermost-server/v6/app/request"
 	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/shared/mlog"
 	"github.com/mattermost/mattermost-server/v6/utils"
 	"github.com/mattermost/mattermost-server/v6/utils/fileutils"
 )
@@ -238,7 +240,7 @@ func TestImportBulkImport(t *testing.T) {
 {"type": "user", "user": {"username": "` + username + `", "email": "` + username + `@example.com", "teams": [{"name": "` + teamName + `","theme": "` + teamTheme1 + `", "channels": [{"name": "` + channelName + `"}]}]}}
 {"type": "post", "post": {"team": "` + teamName + `", "channel": "` + channelName + `", "user": "` + username + `", "message": "Hello World", "create_at": 123456789012, "attachments":[{"path": "` + testImage + `"}], "props":{"attachments":[{"id":0,"fallback":"[February 4th, 2020 2:46 PM] author: fallback","color":"D0D0D0","pretext":"","author_name":"author","author_link":"","title":"","title_link":"","text":"this post has props","fields":null,"image_url":"","thumb_url":"","footer":"Posted in #general","footer_icon":"","ts":"1580823992.000100"}]}}}
 {"type": "direct_channel", "direct_channel": {"members": ["` + username + `", "` + username + `"]}}
-{"type": "direct_post", "direct_post": {"channel_members": ["` + username + `", "` + username + `"], "user": "` + username + `", "message": "Hello Direct Channel to myself", "create_at": 123456789014, "props":{"attachments":[{"id":0,"fallback":"[February 4th, 2020 2:46 PM] author: fallback","color":"D0D0D0","pretext":"","author_name":"author","author_link":"","title":"","title_link":"","text":"this post has props","fields":null,"image_url":"","thumb_url":"","footer":"Posted in #general","footer_icon":"","ts":"1580823992.000100"}]}}}}`
+{"type": "direct_post", "direct_post": {"channel_members": ["` + username + `", "` + username + `"], "user": "` + username + `", "message": "Hello Direct Channel to myself", "create_at": 123456789014, "props":{"attachments":[{"id":0,"fallback":"[February 4th, 2020 2:46 PM] author: fallback","color":"D0D0D0","pretext":"","author_name":"author","author_link":"","title":"","title_link":"","text":"this post has props","fields":null,"image_url":"","thumb_url":"","footer":"Posted in #general","footer_icon":"","ts":"1580823992.000100"}]}}}`
 
 		err, line := th.App.BulkImport(th.Context, strings.NewReader(data6), nil, false, 2)
 		require.Nil(t, err, "BulkImport should have succeeded")
@@ -285,6 +287,9 @@ func AssertFileIdsInPost(files []*model.FileInfo, th *TestHelper, t *testing.T)
 }
 
 func TestProcessAttachments(t *testing.T) {
+	logger, _ := mlog.NewLogger()
+	c := request.EmptyContext(logger)
+
 	genAttachments := func() *[]imports.AttachmentImportData {
 		return &[]imports.AttachmentImportData{
 			{
@@ -333,10 +338,11 @@ func TestProcessAttachments(t *testing.T) {
 				Path: model.NewString("somedir/file.jpg"),
 			},
 		}
-		err := processAttachments(&line, "", nil)
+
+		err := processAttachments(c, &line, "", nil)
 		require.NoError(t, err)
 		require.Equal(t, expected, line.Post.Attachments)
-		err = processAttachments(&line2, "", nil)
+		err = processAttachments(c, &line2, "", nil)
 		require.NoError(t, err)
 		require.Equal(t, expected, line2.DirectPost.Attachments)
 	})
@@ -352,27 +358,27 @@ func TestProcessAttachments(t *testing.T) {
 		}
 
 		t.Run("post attachments", func(t *testing.T) {
-			err := processAttachments(&line, "/tmp", nil)
+			err := processAttachments(c, &line, "/tmp", nil)
 			require.NoError(t, err)
 			require.Equal(t, expected, line.Post.Attachments)
 		})
 
 		t.Run("direct post attachments", func(t *testing.T) {
-			err := processAttachments(&line2, "/tmp", nil)
+			err := processAttachments(c, &line2, "/tmp", nil)
 			require.NoError(t, err)
 			require.Equal(t, expected, line2.DirectPost.Attachments)
 		})
 
 		t.Run("profile image", func(t *testing.T) {
 			expected := "/tmp/profile.jpg"
-			err := processAttachments(&userLine, "/tmp", nil)
+			err := processAttachments(c, &userLine, "/tmp", nil)
 			require.NoError(t, err)
 			require.Equal(t, expected, *userLine.User.ProfileImage)
 		})
 
 		t.Run("emoji", func(t *testing.T) {
 			expected := "/tmp/emoji.png"
-			err := processAttachments(&emojiLine, "/tmp", nil)
+			err := processAttachments(c, &emojiLine, "/tmp", nil)
 			require.NoError(t, err)
 			require.Equal(t, expected, *emojiLine.Emoji.Image)
 		})
@@ -383,11 +389,11 @@ func TestProcessAttachments(t *testing.T) {
 			filesMap := map[string]*zip.File{
 				"/tmp/file.jpg": nil,
 			}
-			err := processAttachments(&line, "", filesMap)
+			err := processAttachments(c, &line, "", filesMap)
 			require.Error(t, err)
 
 			filesMap["/tmp/somedir/file.jpg"] = nil
-			err = processAttachments(&line, "", filesMap)
+			err = processAttachments(c, &line, "", filesMap)
 			require.NoError(t, err)
 		})
 
@@ -395,11 +401,11 @@ func TestProcessAttachments(t *testing.T) {
 			filesMap := map[string]*zip.File{
 				"/tmp/file.jpg": nil,
 			}
-			err := processAttachments(&line2, "", filesMap)
+			err := processAttachments(c, &line2, "", filesMap)
 			require.Error(t, err)
 
 			filesMap["/tmp/somedir/file.jpg"] = nil
-			err = processAttachments(&line2, "", filesMap)
+			err = processAttachments(c, &line2, "", filesMap)
 			require.NoError(t, err)
 		})
 
@@ -407,11 +413,11 @@ func TestProcessAttachments(t *testing.T) {
 			filesMap := map[string]*zip.File{
 				"/tmp/file.jpg": nil,
 			}
-			err := processAttachments(&userLine, "", filesMap)
+			err := processAttachments(c, &userLine, "", filesMap)
 			require.Error(t, err)
 
 			filesMap["/tmp/profile.jpg"] = nil
-			err = processAttachments(&userLine, "", filesMap)
+			err = processAttachments(c, &userLine, "", filesMap)
 			require.NoError(t, err)
 		})
 
@@ -419,11 +425,11 @@ func TestProcessAttachments(t *testing.T) {
 			filesMap := map[string]*zip.File{
 				"/tmp/file.jpg": nil,
 			}
-			err := processAttachments(&emojiLine, "", filesMap)
+			err := processAttachments(c, &emojiLine, "", filesMap)
 			require.Error(t, err)
 
 			filesMap["/tmp/emoji.png"] = nil
-			err = processAttachments(&emojiLine, "", filesMap)
+			err = processAttachments(c, &emojiLine, "", filesMap)
 			require.NoError(t, err)
 		})
 	})