From 6f1bbcd8ec21a46411503170a4bf916856852527 Mon Sep 17 00:00:00 2001 From: Julien Tant <785518+JulienTant@users.noreply.github.com> Date: Mon, 15 Jan 2024 14:29:45 -0700 Subject: [PATCH] [MM-54770] Add ability to export profile pictures and fix importing them (#25042) * add ability to export pp and fix import * remove unused nopSeeker * remove debug log * fix shadow vars * generate a warning instead of an error when unable to export profile picture * fix merge conflicts --------- Co-authored-by: Mattermost Build --- server/channels/app/app_iface.go | 2 + server/channels/app/export.go | 37 +++++++++++++++---- server/channels/app/imaging/decode.go | 2 +- server/channels/app/import_functions.go | 28 +++++++++++--- .../channels/app/imports/import_validators.go | 4 +- .../app/opentracing/opentracing_layer.go | 22 +++++++++++ server/channels/app/server.go | 2 +- server/channels/app/user.go | 19 ++++++++++ server/channels/jobs/export_process/worker.go | 5 +++ server/cmd/mattermost/commands/export.go | 7 ++++ server/cmd/mmctl/commands/export.go | 6 +++ server/cmd/mmctl/docs/mmctl_export_create.rst | 1 + server/i18n/en.json | 4 ++ server/public/model/bulk_export.go | 1 + 14 files changed, 123 insertions(+), 17 deletions(-) diff --git a/server/channels/app/app_iface.go b/server/channels/app/app_iface.go index 3d4e66aa75..4b09fe5f17 100644 --- a/server/channels/app/app_iface.go +++ b/server/channels/app/app_iface.go @@ -220,6 +220,8 @@ type AppIface interface { GetPostsUsage() (int64, *model.AppError) // GetProductNotices is called from the frontend to fetch the product notices that are relevant to the caller GetProductNotices(c request.CTX, userID, teamID string, client model.NoticeClientType, clientVersion string, locale string) (model.NoticeMessages, *model.AppError) + // GetProfileImagePaths returns the paths to the profile images for the given user IDs if such a profile image exists. + GetProfileImagePath(user *model.User) (string, *model.AppError) // GetPublicKey will return the actual public key saved in the `name` file. GetPublicKey(name string) ([]byte, *model.AppError) // GetSanitizedConfig gets the configuration for a system admin without any secrets. diff --git a/server/channels/app/export.go b/server/channels/app/export.go index e216a76ae7..dfb5cafea1 100644 --- a/server/channels/app/export.go +++ b/server/channels/app/export.go @@ -102,7 +102,8 @@ func (a *App) BulkExport(ctx request.CTX, writer io.Writer, outPath string, job } ctx.Logger().Info("Bulk export: exporting users") - if err = a.exportAllUsers(ctx, job, writer, opts.IncludeArchivedChannels); err != nil { + profilePictures, err := a.exportAllUsers(ctx, job, writer, opts.IncludeArchivedChannels, opts.IncludeProfilePictures) + if err != nil { return err } @@ -150,6 +151,15 @@ func (a *App) BulkExport(ctx request.CTX, writer io.Writer, outPath string, job updateJobProgress(ctx.Logger(), a.Srv().Store(), job, "attachments_exported", len(attachments)+len(directAttachments)+len(emojiPaths)) } + if opts.IncludeProfilePictures { + for _, profilePicture := range profilePictures { + if err := a.exportFile(outPath, profilePicture, zipWr); err != nil { + ctx.Logger().Warn("Unable to export profile picture", mlog.String("profile_picture", profilePicture), mlog.Err(err)) + } + } + updateJobProgress(ctx.Logger(), a.Srv().Store(), job, "profile_pictures_exported", len(profilePictures)) + } + return nil } @@ -257,14 +267,15 @@ func (a *App) exportAllChannels(ctx request.CTX, job *model.Job, writer io.Write return nil } -func (a *App) exportAllUsers(ctx request.CTX, job *model.Job, writer io.Writer, includeArchivedChannels bool) *model.AppError { +func (a *App) exportAllUsers(ctx request.CTX, job *model.Job, writer io.Writer, includeArchivedChannels, includeProfilePictures bool) ([]string, *model.AppError) { afterId := strings.Repeat("0", 26) cnt := 0 + profilePictures := []string{} for { users, err := a.Srv().Store().User().GetAllAfter(1000, afterId) if err != nil { - return model.NewAppError("exportAllUsers", "app.user.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + return profilePictures, model.NewAppError("exportAllUsers", "app.user.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err) } if len(users) == 0 { @@ -280,7 +291,7 @@ func (a *App) exportAllUsers(ctx request.CTX, job *model.Job, writer io.Writer, exportedPrefs := make(map[string]*string) allPrefs, err := a.GetPreferencesForUser(ctx, user.Id) if err != nil { - return err + return profilePictures, err } for _, pref := range allPrefs { // We need to manage the special cases @@ -316,23 +327,35 @@ func (a *App) exportAllUsers(ctx request.CTX, job *model.Job, writer io.Writer, userLine := ImportLineFromUser(user, exportedPrefs) + if includeProfilePictures { + var pp string + pp, err = a.GetProfileImagePath(user) + if err != nil { + return profilePictures, err + } + if pp != "" { + userLine.User.ProfileImage = &pp + profilePictures = append(profilePictures, pp) + } + } + userLine.User.NotifyProps = a.buildUserNotifyProps(user.NotifyProps) // Do the Team Memberships. members, err := a.buildUserTeamAndChannelMemberships(ctx, user.Id, includeArchivedChannels) if err != nil { - return err + return profilePictures, err } userLine.User.Teams = members if err := a.exportWriteLine(writer, userLine); err != nil { - return err + return profilePictures, err } } } - return nil + return profilePictures, nil } func (a *App) buildUserTeamAndChannelMemberships(c request.CTX, userID string, includeArchivedChannels bool) (*[]imports.UserTeamImportData, *model.AppError) { diff --git a/server/channels/app/imaging/decode.go b/server/channels/app/imaging/decode.go index 7ee27895d7..59e4155e51 100644 --- a/server/channels/app/imaging/decode.go +++ b/server/channels/app/imaging/decode.go @@ -113,7 +113,7 @@ func (d *Decoder) DecodeConfig(rd io.Reader) (image.Config, string, error) { // GetDimensions returns the dimensions for the given encoded image data. func GetDimensions(imageData io.Reader) (int, int, error) { cfg, _, err := image.DecodeConfig(imageData) - if seeker, ok := imageData.(io.ReadSeeker); ok { + if seeker, ok := imageData.(io.Seeker); ok { defer seeker.Seek(0, 0) } return cfg.Width, cfg.Height, err diff --git a/server/channels/app/import_functions.go b/server/channels/app/import_functions.go index 42b4f5a7dd..c2713b95b3 100644 --- a/server/channels/app/import_functions.go +++ b/server/channels/app/import_functions.go @@ -619,18 +619,34 @@ func (a *App) importUser(rctx request.CTX, data *imports.UserImportData, dryRun } if data.ProfileImage != nil { - var file io.ReadCloser + var file io.ReadSeeker var err error if data.ProfileImageData != nil { - file, err = data.ProfileImageData.Open() + // *zip.File does not support Seek, and we need a seeker to reset the cursor position after checking the picture dimension + var f io.ReadCloser + f, err = data.ProfileImageData.Open() + if err != nil { + rctx.Logger().Warn("Unable to open the profile image data.", mlog.Err(err)) + } else { + limitedReader := io.LimitReader(f, *a.Config().FileSettings.MaxFileSize) + var b []byte + b, err = io.ReadAll(limitedReader) + if err != nil { + rctx.Logger().Warn("Unable to read all bytes from profile picture.", mlog.Err(err)) + } else { + file = bytes.NewReader(b) + } + } } else { file, err = os.Open(*data.ProfileImage) + if err != nil { + rctx.Logger().Warn("Unable to open the profile image.", mlog.Err(err)) + } else { + defer file.(*os.File).Close() + } } - if err != nil { - rctx.Logger().Warn("Unable to open the profile image.", mlog.Err(err)) - } else { - defer file.Close() + if file != nil { if limitErr := checkImageLimits(file, *a.Config().FileSettings.MaxImageResolution); limitErr != nil { return model.NewAppError("SetProfileImage", "api.user.upload_profile_user.check_image_limits.app_error", nil, "", http.StatusBadRequest) } diff --git a/server/channels/app/imports/import_validators.go b/server/channels/app/imports/import_validators.go index 040deafe92..3e98dc0d54 100644 --- a/server/channels/app/imports/import_validators.go +++ b/server/channels/app/imports/import_validators.go @@ -190,9 +190,9 @@ func ValidateChannelImportData(data *ChannelImportData) *model.AppError { } func ValidateUserImportData(data *UserImportData) *model.AppError { - if data.ProfileImage != nil { + if data.ProfileImage != nil && data.ProfileImageData == nil { if _, err := os.Stat(*data.ProfileImage); os.IsNotExist(err) { - return model.NewAppError("BulkImport", "app.import.validate_user_import_data.profile_image.error", nil, "", http.StatusBadRequest) + return model.NewAppError("BulkImport", "app.import.validate_user_import_data.profile_image.error", nil, "", http.StatusBadRequest).Wrap(err) } } diff --git a/server/channels/app/opentracing/opentracing_layer.go b/server/channels/app/opentracing/opentracing_layer.go index e2b784c6a8..736aac36fb 100644 --- a/server/channels/app/opentracing/opentracing_layer.go +++ b/server/channels/app/opentracing/opentracing_layer.go @@ -8603,6 +8603,28 @@ func (a *OpenTracingAppLayer) GetProfileImage(user *model.User) ([]byte, bool, * return resultVar0, resultVar1, resultVar2 } +func (a *OpenTracingAppLayer) GetProfileImagePath(user *model.User) (string, *model.AppError) { + origCtx := a.ctx + span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetProfileImagePath") + + a.ctx = newCtx + a.app.Srv().Store().SetContext(newCtx) + defer func() { + a.app.Srv().Store().SetContext(origCtx) + a.ctx = origCtx + }() + + defer span.Finish() + resultVar0, resultVar1 := a.app.GetProfileImagePath(user) + + if resultVar1 != nil { + span.LogFields(spanlog.Error(resultVar1)) + ext.Error.Set(span, true) + } + + return resultVar0, resultVar1 +} + func (a *OpenTracingAppLayer) GetPublicChannelsByIdsForTeam(c request.CTX, teamID string, channelIDs []string) (model.ChannelList, *model.AppError) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetPublicChannelsByIdsForTeam") diff --git a/server/channels/app/server.go b/server/channels/app/server.go index 1394b783cc..81aa04ad2b 100644 --- a/server/channels/app/server.go +++ b/server/channels/app/server.go @@ -1748,7 +1748,7 @@ func (s *Server) GetProfileImage(user *model.User) ([]byte, bool, *model.AppErro return img, false, nil } - path := "users/" + user.Id + "/profile.png" + path := getProfileImagePath(user.Id) data, err := s.ReadFile(path) if err != nil { diff --git a/server/channels/app/user.go b/server/channels/app/user.go index 986de0bcbd..3607e2c50e 100644 --- a/server/channels/app/user.go +++ b/server/channels/app/user.go @@ -789,6 +789,25 @@ func (a *App) DeactivateMfa(userID string) *model.AppError { return nil } +// GetProfileImagePaths returns the paths to the profile images for the given user IDs if such a profile image exists. +func (a *App) GetProfileImagePath(user *model.User) (string, *model.AppError) { + path := getProfileImagePath(user.Id) + exist, err := a.ch.srv.FileBackend().FileExists(path) + if err != nil { + return "", model.NewAppError( + "GetProfileImagePath", + "api.user.get_profile_image_path.app_error", + nil, + "", + http.StatusInternalServerError, + ).Wrap(err) + } + if !exist { + return "", nil + } + return path, nil +} + func (a *App) GetProfileImage(user *model.User) ([]byte, bool, *model.AppError) { return a.ch.srv.GetProfileImage(user) } diff --git a/server/channels/jobs/export_process/worker.go b/server/channels/jobs/export_process/worker.go index eb8a534be0..607f44dd40 100644 --- a/server/channels/jobs/export_process/worker.go +++ b/server/channels/jobs/export_process/worker.go @@ -43,6 +43,11 @@ func MakeWorker(jobServer *jobs.JobServer, app AppIface) *jobs.SimpleWorker { opts.IncludeArchivedChannels = true } + includeProfilePictures, ok := job.Data["include_profile_pictures"] + if ok && includeProfilePictures == "true" { + opts.IncludeProfilePictures = true + } + outPath := *app.Config().ExportSettings.Directory exportFilename := job.Id + "_export.zip" diff --git a/server/cmd/mattermost/commands/export.go b/server/cmd/mattermost/commands/export.go index 8ebd23495b..5e2894ada4 100644 --- a/server/cmd/mattermost/commands/export.go +++ b/server/cmd/mattermost/commands/export.go @@ -82,6 +82,7 @@ func init() { BulkExportCmd.Flags().Bool("all-teams", true, "Export all teams from the server.") BulkExportCmd.Flags().Bool("with-archived-channels", false, "Also exports archived channels.") + BulkExportCmd.Flags().Bool("with-profile-pictures", false, "Also exports profile pictures.") BulkExportCmd.Flags().Bool("attachments", false, "Also export file attachments.") BulkExportCmd.Flags().Bool("archive", false, "Outputs a single archive file.") @@ -240,6 +241,11 @@ func bulkExportCmdF(command *cobra.Command, args []string) error { return errors.Wrap(err, "with-archived-channels flag error") } + includeProfilePictures, err := command.Flags().GetBool("with-profile-pictures") + if err != nil { + return errors.Wrap(err, "with-profile-pictures flag error") + } + fileWriter, err := os.Create(args[0]) if err != nil { return err @@ -255,6 +261,7 @@ func bulkExportCmdF(command *cobra.Command, args []string) error { opts.IncludeAttachments = attachments opts.CreateArchive = archive opts.IncludeArchivedChannels = withArchivedChannels + opts.IncludeProfilePictures = includeProfilePictures if err := a.BulkExport(rctx, fileWriter, filepath.Dir(outPath), nil /* nil job since it's spawned from CLI */, opts); err != nil { CommandPrintErrorln(err.Error()) return err diff --git a/server/cmd/mmctl/commands/export.go b/server/cmd/mmctl/commands/export.go index 3cccb77e45..869bee1259 100644 --- a/server/cmd/mmctl/commands/export.go +++ b/server/cmd/mmctl/commands/export.go @@ -101,6 +101,7 @@ func init() { ExportCreateCmd.Flags().Bool("no-attachments", false, "Exclude file attachments from the export file.") ExportCreateCmd.Flags().Bool("include-archived-channels", false, "Include archived channels in the export file.") + ExportCreateCmd.Flags().Bool("include-profile-pictures", false, "Include profile pictures in the export file.") ExportDownloadCmd.Flags().Bool("resume", false, "Set to true to resume an export download.") _ = ExportDownloadCmd.Flags().MarkHidden("resume") @@ -142,6 +143,11 @@ func exportCreateCmdF(c client.Client, command *cobra.Command, args []string) er data["include_archived_channels"] = "true" } + includeProfilePictures, _ := command.Flags().GetBool("include-profile-pictures") + if includeProfilePictures { + data["include_profile_pictures"] = "true" + } + job, _, err := c.CreateJob(context.TODO(), &model.Job{ Type: model.JobTypeExportProcess, Data: data, diff --git a/server/cmd/mmctl/docs/mmctl_export_create.rst b/server/cmd/mmctl/docs/mmctl_export_create.rst index a52088e733..c70a3f8bb3 100644 --- a/server/cmd/mmctl/docs/mmctl_export_create.rst +++ b/server/cmd/mmctl/docs/mmctl_export_create.rst @@ -22,6 +22,7 @@ Options -h, --help help for create --include-archived-channels Include archived channels in the export file. + --include-profile-pictures Include profile pictures in the export file. --no-attachments Exclude file attachments from the export file. Options inherited from parent commands diff --git a/server/i18n/en.json b/server/i18n/en.json index 2fd8456d43..d87f4beeb5 100644 --- a/server/i18n/en.json +++ b/server/i18n/en.json @@ -4370,6 +4370,10 @@ "id": "api.user.get_authorization_code.endpoint.app_error", "translation": "Error retrieving endpoint from Discovery Document." }, + { + "id": "api.user.get_profile_image_path.app_error", + "translation": "Error while checking if a user has a custom profile image." + }, { "id": "api.user.get_uploads_for_user.forbidden.app_error", "translation": "Failed to get uploads." diff --git a/server/public/model/bulk_export.go b/server/public/model/bulk_export.go index 3a32d3468f..b5bf5a4b9c 100644 --- a/server/public/model/bulk_export.go +++ b/server/public/model/bulk_export.go @@ -9,6 +9,7 @@ const ExportDataDir = "data" type BulkExportOpts struct { IncludeAttachments bool + IncludeProfilePictures bool IncludeArchivedChannels bool CreateArchive bool }