diff --git a/server/channels/app/export.go b/server/channels/app/export.go index 47a4d2d90b..f418fd8780 100644 --- a/server/channels/app/export.go +++ b/server/channels/app/export.go @@ -153,6 +153,13 @@ func (a *App) BulkExport(ctx request.CTX, writer io.Writer, outPath string, job return err } + ctx.Logger().Info("Bulk export: exporting bots") + botPPs, err := a.exportAllBots(ctx, job, writer, opts.IncludeProfilePictures) + if err != nil { + return err + } + profilePictures = append(profilePictures, botPPs...) + ctx.Logger().Info("Bulk export: exporting posts") attachments, err := a.exportAllPosts(ctx, job, writer, opts.IncludeAttachments, opts.IncludeArchivedChannels) if err != nil { @@ -455,6 +462,11 @@ func (a *App) exportAllUsers(ctx request.CTX, job *model.Job, writer io.Writer, for _, user := range users { afterId = user.Id + // Skip bots as they are exported separately. + if user.IsBot { + continue + } + // Gathering here the exportable preferences to pass them on to ImportLineFromUser exportedPrefs := make(map[string]*string) allPrefs, err := a.GetPreferencesForUser(ctx, user.Id) @@ -531,6 +543,64 @@ func (a *App) exportAllUsers(ctx request.CTX, job *model.Job, writer io.Writer, return profilePictures, nil } +func (a *App) exportAllBots(ctx request.CTX, job *model.Job, writer io.Writer, includeProfilePictures bool) ([]string, *model.AppError) { + afterId := "" + cnt := 0 + profilePictures := []string{} + + const pageSize = 1000 + + for { + bots, err := a.Srv().Store().Bot().GetAllAfter(pageSize, afterId) + if err != nil { + return profilePictures, model.NewAppError("exportAllBots", "app.user.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + cnt += len(bots) + updateJobProgress(ctx.Logger(), a.Srv().Store(), job, "bots_exported", cnt) + + for _, bot := range bots { + afterId = bot.UserId + + var ownerUsername string + owner, err := a.Srv().Store().User().Get(ctx.Context(), bot.OwnerId) + if err != nil { + var nfErr *store.ErrNotFound + if errors.As(err, &nfErr) { + ownerUsername = bot.OwnerId + } else { + return profilePictures, model.NewAppError("exportAllBots", "app.user.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + } else { + ownerUsername = owner.Username + } + + botLine := ImportLineFromBot(bot, ownerUsername) + + if includeProfilePictures { + pp, err := a.GetProfileImagePath(model.UserFromBot(bot)) + if err != nil { + return profilePictures, err + } + if pp != "" { + botLine.Bot.ProfileImage = &pp + profilePictures = append(profilePictures, pp) + } + } + + if err := a.exportWriteLine(writer, botLine); err != nil { + return profilePictures, err + } + } + + if len(bots) < pageSize { + break + } + } + + return profilePictures, nil +} + func (a *App) buildUserTeamAndChannelMemberships(c request.CTX, userID string, includeArchivedChannels bool) (*[]imports.UserTeamImportData, *model.AppError) { var memberships []imports.UserTeamImportData diff --git a/server/channels/app/export_converters.go b/server/channels/app/export_converters.go index aa2a839129..9c28594ac2 100644 --- a/server/channels/app/export_converters.go +++ b/server/channels/app/export_converters.go @@ -177,6 +177,19 @@ func ImportLineFromUser(user *model.User, exportedPrefs map[string]*string) *imp } } +func ImportLineFromBot(bot *model.Bot, ownerUsername string) *imports.LineImportData { + return &imports.LineImportData{ + Type: "bot", + Bot: &imports.BotImportData{ + Username: &bot.Username, + Owner: &ownerUsername, + DisplayName: &bot.DisplayName, + Description: &bot.Description, + DeleteAt: &bot.DeleteAt, + }, + } +} + func ImportUserTeamDataFromTeamMember(member *model.TeamMemberForExport) *imports.UserTeamImportData { rolesList := strings.Fields(member.Roles) if member.SchemeAdmin { diff --git a/server/channels/app/export_test.go b/server/channels/app/export_test.go index bfceb36282..1acc8ffe10 100644 --- a/server/channels/app/export_test.go +++ b/server/channels/app/export_test.go @@ -232,6 +232,41 @@ func TestExportAllUsers(t *testing.T) { assert.ElementsMatch(t, deletedUsers1, deletedUsers2) } +func TestExportAllBots(t *testing.T) { + th1 := Setup(t) + defer th1.TearDown() + + u := th1.CreateUser() + bot, err := th1.App.CreateBot(th1.Context, &model.Bot{ + Username: "bot_1", + DisplayName: model.NewId(), + OwnerId: u.Id, + }) + require.Nil(t, err) + + var b bytes.Buffer + err = th1.App.BulkExport(th1.Context, &b, "somePath", nil, model.BulkExportOpts{}) + require.Nil(t, err) + + th2 := Setup(t) + defer th2.TearDown() + err, i := th2.App.BulkImport(th2.Context, &b, nil, false, 5) + require.Nil(t, err) + assert.EqualValues(t, 0, i) + + u, err = th2.App.GetUserByUsername(u.Username) + require.Nil(t, err) + + bots, err := th2.App.GetBots(th2.Context, &model.BotGetOptions{ + OwnerId: u.Id, + Page: 0, + PerPage: 10, + }) + require.Nil(t, err) + require.Len(t, bots, 1) + assert.Equal(t, bot.Username, bots[0].Username) +} + func TestExportDMChannel(t *testing.T) { t.Run("Export a DM channel to another server", func(t *testing.T) { th1 := Setup(t).InitBasic() diff --git a/server/channels/app/import.go b/server/channels/app/import.go index 82b9c9af78..57e5e8213d 100644 --- a/server/channels/app/import.go +++ b/server/channels/app/import.go @@ -96,6 +96,16 @@ func processAttachments(c request.CTX, line *imports.LineImportData, basePath st } } } + case "bot": + if line.Bot.ProfileImage != nil { + path := filepath.Join(basePath, *line.Bot.ProfileImage) + *line.Bot.ProfileImage = path + if len(filesMap) > 0 { + if line.Bot.ProfileImageData, ok = filesMap[path]; !ok { + return fmt.Errorf("attachment %q not found in map", path) + } + } + } case "emoji": if line.Emoji.Image != nil { path := filepath.Join(basePath, *line.Emoji.Image) @@ -339,6 +349,11 @@ func (a *App) importLine(c request.CTX, line imports.LineImportData, dryRun bool return model.NewAppError("BulkImport", "app.import.import_line.null_user.error", nil, "", http.StatusBadRequest) } return a.importUser(c, line.User, dryRun) + case line.Type == "bot": + if line.Bot == nil { + return model.NewAppError("BulkImport", "app.import.import_line.null_bot.error", nil, "", http.StatusBadRequest) + } + return a.importBot(c, line.Bot, dryRun) case line.Type == "direct_channel": if line.DirectChannel == nil { return model.NewAppError("BulkImport", "app.import.import_line.null_direct_channel.error", nil, "", http.StatusBadRequest) diff --git a/server/channels/app/import_functions.go b/server/channels/app/import_functions.go index dc9b333267..9f6960d5da 100644 --- a/server/channels/app/import_functions.go +++ b/server/channels/app/import_functions.go @@ -629,41 +629,10 @@ func (a *App) importUser(rctx request.CTX, data *imports.UserImportData, dryRun savedUser = user } - if data.ProfileImage != nil { - var file io.ReadSeeker - var err error - if data.ProfileImageData != nil { - // *zip.File does not support Seek, and we need a seeker to reset the cursor position after checking the picture dimension - var f io.ReadCloser - f, err = data.ProfileImageData.Open() - if err != nil { - rctx.Logger().Warn("Unable to open the profile image data.", mlog.Err(err)) - } else { - limitedReader := io.LimitReader(f, *a.Config().FileSettings.MaxFileSize) - var b []byte - b, err = io.ReadAll(limitedReader) - if err != nil { - rctx.Logger().Warn("Unable to read all bytes from profile picture.", mlog.Err(err)) - } else { - file = bytes.NewReader(b) - } - } - } else { - file, err = os.Open(*data.ProfileImage) - if err != nil { - rctx.Logger().Warn("Unable to open the profile image.", mlog.Err(err)) - } else { - defer file.(*os.File).Close() - } - } - - if file != nil { - if limitErr := checkImageLimits(file, *a.Config().FileSettings.MaxImageResolution); limitErr != nil { - return model.NewAppError("SetProfileImage", "api.user.upload_profile_user.check_image_limits.app_error", nil, "", http.StatusBadRequest) - } - if err := a.SetProfileImageFromFile(rctx, savedUser.Id, file); err != nil { - rctx.Logger().Warn("Unable to set the profile image from a file.", mlog.Err(err)) - } + if data.Avatar.ProfileImage != nil { + appErr := a.importProfileImage(rctx, savedUser.Id, &data.Avatar) + if appErr != nil { + return appErr } } @@ -865,6 +834,143 @@ func (a *App) importUser(rctx request.CTX, data *imports.UserImportData, dryRun return a.importUserTeams(rctx, savedUser, data.Teams) } +func (a *App) importBot(rctx request.CTX, data *imports.BotImportData, dryRun bool) *model.AppError { + var fields []mlog.Field + if data != nil && data.Username != nil { + fields = append(fields, mlog.String("user_name", *data.Username)) + } + rctx.Logger().Info("Validating bot", fields...) + + if err := imports.ValidateBotImportData(data); err != nil { + return err + } + + // If this is a Dry Run, do not continue any further. + if dryRun { + return nil + } + + rctx.Logger().Info("Importing bot", fields...) + + // We want to avoid database writes if nothing has changed. + hasBotChanged := false + + var bot *model.Bot + var nErr error + bot, nErr = a.Srv().Store().Bot().GetByUsername(*data.Username) + if nErr != nil { + bot = &model.Bot{} + hasBotChanged = true + } + + bot.Username = *data.Username + + if data.Description != nil && bot.Description != *data.Description { + bot.Description = *data.Description + hasBotChanged = true + } + + if data.DisplayName != nil && bot.DisplayName != *data.DisplayName { + bot.DisplayName = *data.DisplayName + hasBotChanged = true + } + + var owner *model.User + if data.Owner != nil { + owner, nErr = a.Srv().Store().User().GetByUsername(*data.Owner) + if nErr != nil { + var nfErr *store.ErrNotFound + switch { + case errors.As(nErr, &nfErr): + // If the owner does not exist, we assume the owner is a plugin hence keeping the owner username as is. + bot.OwnerId = *data.Owner + default: + return model.NewAppError("importBot", "app.import.import_bot.owner_could_not_found.error", map[string]any{"Owner": *data.Owner}, "", http.StatusInternalServerError).Wrap(nErr) + } + } else { + bot.OwnerId = owner.Id + } + } + + var savedBot *model.Bot + if bot.UserId == "" { + var appErr *model.AppError + if savedBot, appErr = a.CreateBot(rctx, bot); appErr != nil { + var appErr *model.AppError + var invErr *store.ErrInvalidInput + switch { + case errors.As(appErr, &invErr): + switch invErr.Field { + case "username": + return model.NewAppError("importUser", "app.user.save.username_exists.app_error", nil, "", http.StatusBadRequest).Wrap(appErr) + default: + return model.NewAppError("importUser", "app.user.save.existing.app_error", nil, "", http.StatusBadRequest).Wrap(appErr) + } + default: + return appErr + } + } + } else if hasBotChanged { + var err error + if savedBot, err = a.Srv().Store().Bot().Update(bot); err != nil { + return model.NewAppError("importBot", "app.bot.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + } + + if savedBot == nil { + savedBot = bot + } + + if data.Avatar.ProfileImage != nil { + appErr := a.importProfileImage(rctx, savedBot.UserId, &data.Avatar) + if appErr != nil { + return appErr + } + } + + return nil +} + +func (a *App) importProfileImage(rctx request.CTX, userID string, data *imports.Avatar) *model.AppError { + var file io.ReadSeeker + var err error + if data.ProfileImageData != nil { + // *zip.File does not support Seek, and we need a seeker to reset the cursor position after checking the picture dimension + var f io.ReadCloser + f, err = data.ProfileImageData.Open() + if err != nil { + rctx.Logger().Warn("Unable to open the profile image data.", mlog.Err(err)) + } else { + limitedReader := io.LimitReader(f, *a.Config().FileSettings.MaxFileSize) + var b []byte + b, err = io.ReadAll(limitedReader) + if err != nil { + rctx.Logger().Warn("Unable to read all bytes from profile picture.", mlog.Err(err)) + } else { + file = bytes.NewReader(b) + } + } + } else { + file, err = os.Open(*data.ProfileImage) + if err != nil { + rctx.Logger().Warn("Unable to open the profile image.", mlog.Err(err)) + } else { + defer file.(*os.File).Close() + } + } + + if file != nil { + if limitErr := checkImageLimits(file, *a.Config().FileSettings.MaxImageResolution); limitErr != nil { + return model.NewAppError("SetProfileImage", "api.user.upload_profile_user.check_image_limits.app_error", nil, "", http.StatusBadRequest) + } + if err := a.SetProfileImageFromFile(rctx, userID, file); err != nil { + rctx.Logger().Warn("Unable to set the profile image from a file.", mlog.Err(err)) + } + } + + return nil +} + func (a *App) importUserTeams(rctx request.CTX, user *model.User, data *[]imports.UserTeamImportData) *model.AppError { if data == nil { return nil diff --git a/server/channels/app/import_functions_test.go b/server/channels/app/import_functions_test.go index 08aa496cf9..0c936ec2f2 100644 --- a/server/channels/app/import_functions_test.go +++ b/server/channels/app/import_functions_test.go @@ -795,13 +795,15 @@ func TestImportImportUser(t *testing.T) { username := model.NewUsername() testsDir, _ := fileutils.FindDir("tests") data = imports.UserImportData{ - ProfileImage: model.NewPointer(filepath.Join(testsDir, "test.png")), - Username: &username, - Email: model.NewPointer(model.NewId() + "@example.com"), - Nickname: model.NewPointer(model.NewId()), - FirstName: model.NewPointer(model.NewId()), - LastName: model.NewPointer(model.NewId()), - Position: model.NewPointer(model.NewId()), + Avatar: imports.Avatar{ + ProfileImage: model.NewPointer(filepath.Join(testsDir, "test.png")), + }, + Username: &username, + Email: model.NewPointer(model.NewId() + "@example.com"), + Nickname: model.NewPointer(model.NewId()), + FirstName: model.NewPointer(model.NewId()), + LastName: model.NewPointer(model.NewId()), + Position: model.NewPointer(model.NewId()), } appErr = th.App.importUser(th.Context, &data, false) require.Nil(t, appErr, "Should have succeeded to import valid user.") diff --git a/server/channels/app/import_test.go b/server/channels/app/import_test.go index 2e0a5a291a..02e66d461e 100644 --- a/server/channels/app/import_test.go +++ b/server/channels/app/import_test.go @@ -304,7 +304,9 @@ func TestProcessAttachments(t *testing.T) { userLine := imports.LineImportData{ Type: "user", User: &imports.UserImportData{ - ProfileImage: model.NewPointer("profile.jpg"), + Avatar: imports.Avatar{ + ProfileImage: model.NewPointer("profile.jpg"), + }, }, } diff --git a/server/channels/app/imports/import_types.go b/server/channels/app/imports/import_types.go index b4ff056b25..9e48f04eb3 100644 --- a/server/channels/app/imports/import_types.go +++ b/server/channels/app/imports/import_types.go @@ -19,6 +19,7 @@ type LineImportData struct { Team *TeamImportData `json:"team,omitempty"` Channel *ChannelImportData `json:"channel,omitempty"` User *UserImportData `json:"user,omitempty"` + Bot *BotImportData `json:"bot,omitempty"` Post *PostImportData `json:"post,omitempty"` DirectChannel *DirectChannelImportData `json:"direct_channel,omitempty"` DirectPost *DirectPostImportData `json:"direct_post,omitempty"` @@ -54,24 +55,28 @@ type ChannelImportData struct { DeletedAt *int64 `json:"deleted_at,omitempty"` } +type Avatar struct { + ProfileImage *string `json:"profile_image,omitempty"` + ProfileImageData *zip.File `json:"-"` +} + type UserImportData struct { - ProfileImage *string `json:"profile_image,omitempty"` - ProfileImageData *zip.File `json:"-"` - Username *string `json:"username"` - Email *string `json:"email"` - AuthService *string `json:"auth_service"` - AuthData *string `json:"auth_data,omitempty"` - Password *string `json:"password,omitempty"` - Nickname *string `json:"nickname"` - FirstName *string `json:"first_name"` - LastName *string `json:"last_name"` - Position *string `json:"position"` - Roles *string `json:"roles"` - Locale *string `json:"locale"` - UseMarkdownPreview *string `json:"feature_enabled_markdown_preview,omitempty"` - UseFormatting *string `json:"formatting,omitempty"` - ShowUnreadSection *string `json:"show_unread_section,omitempty"` - DeleteAt *int64 `json:"delete_at,omitempty"` + Avatar + Username *string `json:"username"` + Email *string `json:"email"` + AuthService *string `json:"auth_service"` + AuthData *string `json:"auth_data,omitempty"` + Password *string `json:"password,omitempty"` + Nickname *string `json:"nickname"` + FirstName *string `json:"first_name"` + LastName *string `json:"last_name"` + Position *string `json:"position"` + Roles *string `json:"roles"` + Locale *string `json:"locale"` + UseMarkdownPreview *string `json:"feature_enabled_markdown_preview,omitempty"` + UseFormatting *string `json:"formatting,omitempty"` + ShowUnreadSection *string `json:"show_unread_section,omitempty"` + DeleteAt *int64 `json:"delete_at,omitempty"` SendOnCtrlEnter *string `json:"send_on_ctrl_enter,omitempty"` CodeBlockCtrlEnter *string `json:"code_block_ctrl_enter,omitempty"` @@ -97,6 +102,15 @@ type UserImportData struct { CustomStatus *model.CustomStatus `json:"custom_status,omitempty"` } +type BotImportData struct { + Avatar + Username *string `json:"username"` + Owner *string `json:"owner"` + DisplayName *string `json:"display_name"` + Description *string `json:"description,omitempty"` + DeleteAt *int64 `json:"delete_at,omitempty"` +} + type UserNotifyPropsImportData struct { Desktop *string `json:"desktop"` DesktopSound *string `json:"desktop_sound"` diff --git a/server/channels/app/imports/import_validators.go b/server/channels/app/imports/import_validators.go index ee58f35d4e..faff856c19 100644 --- a/server/channels/app/imports/import_validators.go +++ b/server/channels/app/imports/import_validators.go @@ -192,6 +192,8 @@ func ValidateChannelImportData(data *ChannelImportData) *model.AppError { func ValidateUserImportData(data *UserImportData) *model.AppError { if data.ProfileImage != nil && data.ProfileImageData == nil { if _, err := os.Stat(*data.ProfileImage); os.IsNotExist(err) { + return model.NewAppError("BulkImport", "app.import.validate_user_import_data.profile_image.error", nil, "", http.StatusNotFound).Wrap(err) + } else if err != nil { return model.NewAppError("BulkImport", "app.import.validate_user_import_data.profile_image.error", nil, "", http.StatusBadRequest).Wrap(err) } } @@ -312,6 +314,34 @@ func ValidateUserImportData(data *UserImportData) *model.AppError { return nil } +func ValidateBotImportData(data *BotImportData) *model.AppError { + if data.ProfileImage != nil && data.ProfileImageData == nil { + if _, err := os.Stat(*data.ProfileImage); os.IsNotExist(err) { + return model.NewAppError("BulkImport", "app.import.validate_user_import_data.profile_image.error", nil, "", http.StatusNotFound).Wrap(err) + } else if err != nil { + return model.NewAppError("BulkImport", "app.import.validate_user_import_data.profile_image.error", nil, "", http.StatusBadRequest).Wrap(err) + } + } + + if data.Username == nil { + return model.NewAppError("BulkImport", "app.import.validate_user_import_data.username_missing.error", nil, "", http.StatusBadRequest) + } else if !model.IsValidUsername(*data.Username) { + return model.NewAppError("BulkImport", "app.import.validate_user_import_data.username_invalid.error", nil, "", http.StatusBadRequest) + } + + if data.DisplayName != nil && utf8.RuneCountInString(*data.DisplayName) > model.UserFirstNameMaxRunes { + return model.NewAppError("BulkImport", "app.import.validate_user_import_data.first_name_length.error", nil, "", http.StatusBadRequest) + } + + if data.Owner == nil { + return model.NewAppError("BulkImport", "app.import.validate_bot_import_data.owner_missing.error", nil, "", http.StatusBadRequest) + } else if !model.IsValidUsername(*data.Owner) { + return model.NewAppError("BulkImport", "app.import.validate_user_import_data.username_invalid.error", nil, "", http.StatusBadRequest) + } + + return nil +} + var validAuthServices = []string{ "", model.UserAuthServiceEmail, diff --git a/server/channels/app/imports/import_validators_test.go b/server/channels/app/imports/import_validators_test.go index 453261cf51..4d6495250a 100644 --- a/server/channels/app/imports/import_validators_test.go +++ b/server/channels/app/imports/import_validators_test.go @@ -532,17 +532,19 @@ func TestImportValidateUserImportData(t *testing.T) { // Test a valid User with all fields populated. testsDir, _ := fileutils.FindDir("tests") data = UserImportData{ - ProfileImage: model.NewPointer(filepath.Join(testsDir, "test.png")), - Username: model.NewPointer("bob"), - Email: model.NewPointer("bob@example.com"), - AuthService: model.NewPointer("ldap"), - AuthData: model.NewPointer("bob"), - Nickname: model.NewPointer("BobNick"), - FirstName: model.NewPointer("Bob"), - LastName: model.NewPointer("Blob"), - Position: model.NewPointer("The Boss"), - Roles: model.NewPointer("system_user"), - Locale: model.NewPointer("en"), + Avatar: Avatar{ + ProfileImage: model.NewPointer(filepath.Join(testsDir, "test.png")), + }, + Username: model.NewPointer("bob"), + Email: model.NewPointer("bob@example.com"), + AuthService: model.NewPointer("ldap"), + AuthData: model.NewPointer("bob"), + Nickname: model.NewPointer("BobNick"), + FirstName: model.NewPointer("Bob"), + LastName: model.NewPointer("Blob"), + Position: model.NewPointer("The Boss"), + Roles: model.NewPointer("system_user"), + Locale: model.NewPointer("en"), } err = ValidateUserImportData(&data) require.Nil(t, err, "Validation failed but should have been valid.") @@ -671,6 +673,56 @@ func TestImportValidateUserAuth(t *testing.T) { } } +func TestImportValidateBotImportData(t *testing.T) { + // Test with minimum required valid properties. + data := BotImportData{ + Username: model.NewPointer("bob"), + DisplayName: model.NewPointer("Display Name"), + Owner: model.NewPointer("owner"), + } + err := ValidateBotImportData(&data) + require.Nil(t, err, "Validation failed but should have been valid.") + + // Test with various invalid names. + data.Username = nil + err = ValidateBotImportData(&data) + require.NotNil(t, err, "Should have failed due to nil Username.") + + data.Username = model.NewPointer("") + err = ValidateBotImportData(&data) + require.NotNil(t, err, "Should have failed due to 0 length Username.") + + data.Username = model.NewPointer(strings.Repeat("abcdefghij", 7)) + err = ValidateBotImportData(&data) + require.NotNil(t, err, "Should have failed due to too long Username.") + + data.Username = model.NewPointer("i am a username with spaces and !!!") + err = ValidateBotImportData(&data) + require.NotNil(t, err, "Should have failed due to invalid characters in Username.") + + data.Username = model.NewPointer("bob") + + // Invalid Display Name. + data.DisplayName = model.NewPointer(strings.Repeat("abcdefghij", 7)) + err = ValidateBotImportData(&data) + require.NotNil(t, err, "Should have failed due to too long DisplayName.") + + data.DisplayName = model.NewPointer("Display Name") + + // Invalid Owner Name. + data.Owner = nil + err = ValidateBotImportData(&data) + require.NotNil(t, err, "Should have failed due to too long DisplayName.") + + data.Owner = model.NewPointer("") + err = ValidateBotImportData(&data) + require.NotNil(t, err, "Should have failed due to too long DisplayName.") + + data.Owner = model.NewPointer(strings.Repeat("abcdefghij", 7)) + err = ValidateBotImportData(&data) + require.NotNil(t, err, "Should have failed due to too long OwnerID.") +} + func TestImportValidateUserTeamsImportData(t *testing.T) { // Invalid Name. data := []UserTeamImportData{ diff --git a/server/channels/store/opentracinglayer/opentracinglayer.go b/server/channels/store/opentracinglayer/opentracinglayer.go index b696267812..0dde285eba 100644 --- a/server/channels/store/opentracinglayer/opentracinglayer.go +++ b/server/channels/store/opentracinglayer/opentracinglayer.go @@ -561,6 +561,42 @@ func (s *OpenTracingLayerBotStore) GetAll(options *model.BotGetOptions) ([]*mode return result, err } +func (s *OpenTracingLayerBotStore) GetAllAfter(limit int, afterId string) ([]*model.Bot, error) { + origCtx := s.Root.Store.Context() + span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "BotStore.GetAllAfter") + s.Root.Store.SetContext(newCtx) + defer func() { + s.Root.Store.SetContext(origCtx) + }() + + defer span.Finish() + result, err := s.BotStore.GetAllAfter(limit, afterId) + if err != nil { + span.LogFields(spanlog.Error(err)) + ext.Error.Set(span, true) + } + + return result, err +} + +func (s *OpenTracingLayerBotStore) GetByUsername(username string) (*model.Bot, error) { + origCtx := s.Root.Store.Context() + span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "BotStore.GetByUsername") + s.Root.Store.SetContext(newCtx) + defer func() { + s.Root.Store.SetContext(origCtx) + }() + + defer span.Finish() + result, err := s.BotStore.GetByUsername(username) + if err != nil { + span.LogFields(spanlog.Error(err)) + ext.Error.Set(span, true) + } + + return result, err +} + func (s *OpenTracingLayerBotStore) PermanentDelete(userID string) error { origCtx := s.Root.Store.Context() span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "BotStore.PermanentDelete") diff --git a/server/channels/store/retrylayer/retrylayer.go b/server/channels/store/retrylayer/retrylayer.go index 74016b404b..2c7741e96c 100644 --- a/server/channels/store/retrylayer/retrylayer.go +++ b/server/channels/store/retrylayer/retrylayer.go @@ -596,6 +596,48 @@ func (s *RetryLayerBotStore) GetAll(options *model.BotGetOptions) ([]*model.Bot, } +func (s *RetryLayerBotStore) GetAllAfter(limit int, afterId string) ([]*model.Bot, error) { + + tries := 0 + for { + result, err := s.BotStore.GetAllAfter(limit, afterId) + if err == nil { + return result, nil + } + if !isRepeatableError(err) { + return result, err + } + tries++ + if tries >= 3 { + err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures") + return result, err + } + timepkg.Sleep(100 * timepkg.Millisecond) + } + +} + +func (s *RetryLayerBotStore) GetByUsername(username string) (*model.Bot, error) { + + tries := 0 + for { + result, err := s.BotStore.GetByUsername(username) + if err == nil { + return result, nil + } + if !isRepeatableError(err) { + return result, err + } + tries++ + if tries >= 3 { + err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures") + return result, err + } + timepkg.Sleep(100 * timepkg.Millisecond) + } + +} + func (s *RetryLayerBotStore) PermanentDelete(userID string) error { tries := 0 diff --git a/server/channels/store/sqlstore/bot_store.go b/server/channels/store/sqlstore/bot_store.go index 4564977f70..3cfed416bf 100644 --- a/server/channels/store/sqlstore/bot_store.go +++ b/server/channels/store/sqlstore/bot_store.go @@ -13,6 +13,7 @@ import ( "github.com/mattermost/mattermost/server/public/model" "github.com/mattermost/mattermost/server/v8/channels/store" "github.com/mattermost/mattermost/server/v8/einterfaces" + sq "github.com/mattermost/squirrel" ) // bot is a subset of the model.Bot type, omitting the model.User fields. @@ -44,14 +45,25 @@ func botFromModel(b *model.Bot) *bot { type SqlBotStore struct { *SqlStore metrics einterfaces.MetricsInterface + + // botsQuery is a starting point for all queries that return one or more Bots. + botsQuery sq.SelectBuilder } // newSqlBotStore creates an instance of SqlBotStore, registering the table schema in question. func newSqlBotStore(sqlStore *SqlStore, metrics einterfaces.MetricsInterface) store.BotStore { - return &SqlBotStore{ + bs := &SqlBotStore{ SqlStore: sqlStore, metrics: metrics, } + + // note: we are providing field names explicitly here to maintain order of columns (needed when using raw queries) + bs.botsQuery = bs.getQueryBuilder(). + Select("b.UserId", "u.Username", "u.FirstName AS DisplayName", "b.Description", "b.OwnerId", "COALESCE(b.LastIconUpdate, 0) AS LastIconUpdate", "b.CreateAt", "b.UpdateAt", "b.DeleteAt"). + From("Bots b"). + Join("Users u ON ( u.Id = b.UserId )") + + return bs } // Get fetches the given bot in the database. @@ -219,3 +231,40 @@ func (us SqlBotStore) PermanentDelete(botUserId string) error { } return nil } + +func (us SqlBotStore) GetAllAfter(limit int, afterId string) ([]*model.Bot, error) { + query := us.botsQuery.Where("b.UserId > ?", afterId).OrderBy("b.UserId ASC").Limit(uint64(limit)) + + queryString, args, err := query.ToSql() + if err != nil { + return nil, errors.Wrap(err, "get_all_after_tosql") + } + + bots := []*model.Bot{} + if err := us.GetReplicaX().Select(&bots, queryString, args...); err != nil { + return nil, errors.Wrap(err, "failed to find Bots") + } + + return bots, nil +} + +// Get fetches the given bot in the database. +func (us SqlBotStore) GetByUsername(username string) (*model.Bot, error) { + query := us.botsQuery.Where("u.Username = lower(?)", username) + + queryString, args, err := query.ToSql() + if err != nil { + return nil, errors.Wrap(err, "get_by_username_tosql") + } + + bot := model.Bot{} + if err := us.GetReplicaX().Get(&bot, queryString, args...); err != nil { + if err == sql.ErrNoRows { + return nil, errors.Wrap(store.NewErrNotFound("Bot", fmt.Sprintf("username=%s", username)), "failed to find Bot") + } + + return nil, errors.Wrapf(err, "failed to find Bot with username=%s", username) + } + + return &bot, nil +} diff --git a/server/channels/store/store.go b/server/channels/store/store.go index 74250e20a7..9fc32bc231 100644 --- a/server/channels/store/store.go +++ b/server/channels/store/store.go @@ -497,7 +497,9 @@ type UserStore interface { type BotStore interface { Get(userID string, includeDeleted bool) (*model.Bot, error) + GetByUsername(username string) (*model.Bot, error) GetAll(options *model.BotGetOptions) ([]*model.Bot, error) + GetAllAfter(limit int, afterId string) ([]*model.Bot, error) Save(bot *model.Bot) (*model.Bot, error) Update(bot *model.Bot) (*model.Bot, error) PermanentDelete(userID string) error diff --git a/server/channels/store/storetest/bot_store.go b/server/channels/store/storetest/bot_store.go index b225326b90..a1a892c052 100644 --- a/server/channels/store/storetest/bot_store.go +++ b/server/channels/store/storetest/bot_store.go @@ -5,8 +5,10 @@ package storetest import ( "errors" + "strings" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/mattermost/mattermost/server/public/model" @@ -27,7 +29,9 @@ func makeBotWithUser(t *testing.T, rctx request.CTX, ss store.Store, bot *model. func TestBotStore(t *testing.T, rctx request.CTX, ss store.Store, s SqlStore) { t.Run("Get", func(t *testing.T) { testBotStoreGet(t, rctx, ss, s) }) + t.Run("GetByUsername", func(t *testing.T) { testBotStoreGetByUsername(t, rctx, ss) }) t.Run("GetAll", func(t *testing.T) { testBotStoreGetAll(t, rctx, ss, s) }) + t.Run("GetAllAfter", func(t *testing.T) { testBotStoreGetAllAfter(t, rctx, ss) }) t.Run("Save", func(t *testing.T) { testBotStoreSave(t, rctx, ss) }) t.Run("Update", func(t *testing.T) { testBotStoreUpdate(t, rctx, ss) }) t.Run("PermanentDelete", func(t *testing.T) { testBotStorePermanentDelete(t, rctx, ss) }) @@ -208,8 +212,8 @@ func testBotStoreGetAll(t *testing.T, rctx request.CTX, ss store.Store, s SqlSto Description: "Orphaned bot 5", OwnerId: deletedUser.Id, }) - defer func() { require.NoError(t, ss.Bot().PermanentDelete(b4.UserId)) }() - defer func() { require.NoError(t, ss.User().PermanentDelete(rctx, b4.UserId)) }() + defer func() { require.NoError(t, ss.Bot().PermanentDelete(ob5.UserId)) }() + defer func() { require.NoError(t, ss.User().PermanentDelete(rctx, ob5.UserId)) }() t.Run("get newly created bot stoo", func(t *testing.T) { bots, err := ss.Bot().GetAll(&model.BotGetOptions{Page: 0, PerPage: 10}) @@ -471,3 +475,119 @@ func testBotStorePermanentDelete(t *testing.T, rctx request.CTX, ss store.Store) require.True(t, errors.As(err, &nfErr)) }) } + +func testBotStoreGetAllAfter(t *testing.T, rctx request.CTX, ss store.Store) { + bot1 := &model.Bot{ + Username: "bot_1", + Description: "description", + OwnerId: model.NewId(), + } + + user1, err := ss.User().Save(rctx, model.UserFromBot(bot1)) + require.NoError(t, err) + defer func() { require.NoError(t, ss.User().PermanentDelete(rctx, user1.Id)) }() + bot1.UserId = user1.Id + + returnedNewBot1, nErr := ss.Bot().Save(bot1) + require.NoError(t, nErr) + defer func() { require.NoError(t, ss.Bot().PermanentDelete(bot1.UserId)) }() + + bot2 := &model.Bot{ + Username: "bot_2", + Description: "description", + OwnerId: model.NewId(), + } + + user2, err := ss.User().Save(rctx, model.UserFromBot(bot2)) + require.NoError(t, err) + defer func() { require.NoError(t, ss.User().PermanentDelete(rctx, user2.Id)) }() + bot2.UserId = user2.Id + + returnedNewBot2, nErr := ss.Bot().Save(bot2) + require.NoError(t, nErr) + defer func() { require.NoError(t, ss.Bot().PermanentDelete(bot2.UserId)) }() + + expected := []*model.Bot{returnedNewBot1, returnedNewBot2} + if strings.Compare(returnedNewBot2.UserId, returnedNewBot1.UserId) < 0 { + expected = []*model.Bot{returnedNewBot2, returnedNewBot1} + } + + t.Run("get after lowest possible id", func(t *testing.T) { + actual, err := ss.Bot().GetAllAfter(10000, strings.Repeat("0", 26)) + require.NoError(t, err) + + assert.Equal(t, expected, actual) + }) + + t.Run("get after first user", func(t *testing.T) { + actual, err := ss.Bot().GetAllAfter(10000, expected[0].UserId) + require.NoError(t, err) + + assert.Equal(t, []*model.Bot{expected[1]}, actual) + }) + + t.Run("get after second user", func(t *testing.T) { + actual, err := ss.Bot().GetAllAfter(10000, expected[1].UserId) + require.NoError(t, err) + + assert.Equal(t, []*model.Bot{}, actual) + }) +} + +func testBotStoreGetByUsername(t *testing.T, rctx request.CTX, ss store.Store) { + bot1 := &model.Bot{ + Username: "bot_1", + Description: "description", + OwnerId: model.NewId(), + } + + user1, err := ss.User().Save(rctx, model.UserFromBot(bot1)) + require.NoError(t, err) + defer func() { require.NoError(t, ss.User().PermanentDelete(rctx, user1.Id)) }() + bot1.UserId = user1.Id + + returnedNewBot1, nErr := ss.Bot().Save(bot1) + require.NoError(t, nErr) + defer func() { require.NoError(t, ss.Bot().PermanentDelete(bot1.UserId)) }() + + bot2 := &model.Bot{ + Username: "bot_2", + Description: "description", + OwnerId: model.NewId(), + } + + user2, err := ss.User().Save(rctx, model.UserFromBot(bot2)) + require.NoError(t, err) + defer func() { require.NoError(t, ss.User().PermanentDelete(rctx, user2.Id)) }() + bot2.UserId = user2.Id + + returnedNewBot2, nErr := ss.Bot().Save(bot2) + require.NoError(t, nErr) + defer func() { require.NoError(t, ss.Bot().PermanentDelete(bot2.UserId)) }() + + t.Run("get bot1 by username", func(t *testing.T) { + result, err := ss.Bot().GetByUsername(returnedNewBot1.Username) + require.NoError(t, err) + assert.Equal(t, returnedNewBot1, result) + }) + + t.Run("get bot2 by username", func(t *testing.T) { + result, err := ss.Bot().GetByUsername(returnedNewBot2.Username) + require.NoError(t, err) + assert.Equal(t, returnedNewBot2, result) + }) + + t.Run("get by empty username", func(t *testing.T) { + _, err := ss.Bot().GetByUsername("") + require.Error(t, err) + var nfErr *store.ErrNotFound + require.True(t, errors.As(err, &nfErr)) + }) + + t.Run("get by unknown", func(t *testing.T) { + _, err := ss.Bot().GetByUsername("unknown") + require.Error(t, err) + var nfErr *store.ErrNotFound + require.True(t, errors.As(err, &nfErr)) + }) +} diff --git a/server/channels/store/storetest/mocks/BotStore.go b/server/channels/store/storetest/mocks/BotStore.go index 83d3a82fd7..c2dfbadda1 100644 --- a/server/channels/store/storetest/mocks/BotStore.go +++ b/server/channels/store/storetest/mocks/BotStore.go @@ -74,6 +74,66 @@ func (_m *BotStore) GetAll(options *model.BotGetOptions) ([]*model.Bot, error) { return r0, r1 } +// GetAllAfter provides a mock function with given fields: limit, afterId +func (_m *BotStore) GetAllAfter(limit int, afterId string) ([]*model.Bot, error) { + ret := _m.Called(limit, afterId) + + if len(ret) == 0 { + panic("no return value specified for GetAllAfter") + } + + var r0 []*model.Bot + var r1 error + if rf, ok := ret.Get(0).(func(int, string) ([]*model.Bot, error)); ok { + return rf(limit, afterId) + } + if rf, ok := ret.Get(0).(func(int, string) []*model.Bot); ok { + r0 = rf(limit, afterId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*model.Bot) + } + } + + if rf, ok := ret.Get(1).(func(int, string) error); ok { + r1 = rf(limit, afterId) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetByUsername provides a mock function with given fields: username +func (_m *BotStore) GetByUsername(username string) (*model.Bot, error) { + ret := _m.Called(username) + + if len(ret) == 0 { + panic("no return value specified for GetByUsername") + } + + var r0 *model.Bot + var r1 error + if rf, ok := ret.Get(0).(func(string) (*model.Bot, error)); ok { + return rf(username) + } + if rf, ok := ret.Get(0).(func(string) *model.Bot); ok { + r0 = rf(username) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.Bot) + } + } + + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(username) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // PermanentDelete provides a mock function with given fields: userID func (_m *BotStore) PermanentDelete(userID string) error { ret := _m.Called(userID) diff --git a/server/channels/store/timerlayer/timerlayer.go b/server/channels/store/timerlayer/timerlayer.go index ebd94945ff..ae65b7b1f2 100644 --- a/server/channels/store/timerlayer/timerlayer.go +++ b/server/channels/store/timerlayer/timerlayer.go @@ -551,6 +551,38 @@ func (s *TimerLayerBotStore) GetAll(options *model.BotGetOptions) ([]*model.Bot, return result, err } +func (s *TimerLayerBotStore) GetAllAfter(limit int, afterId string) ([]*model.Bot, error) { + start := time.Now() + + result, err := s.BotStore.GetAllAfter(limit, afterId) + + elapsed := float64(time.Since(start)) / float64(time.Second) + if s.Root.Metrics != nil { + success := "false" + if err == nil { + success = "true" + } + s.Root.Metrics.ObserveStoreMethodDuration("BotStore.GetAllAfter", success, elapsed) + } + return result, err +} + +func (s *TimerLayerBotStore) GetByUsername(username string) (*model.Bot, error) { + start := time.Now() + + result, err := s.BotStore.GetByUsername(username) + + elapsed := float64(time.Since(start)) / float64(time.Second) + if s.Root.Metrics != nil { + success := "false" + if err == nil { + success = "true" + } + s.Root.Metrics.ObserveStoreMethodDuration("BotStore.GetByUsername", success, elapsed) + } + return result, err +} + func (s *TimerLayerBotStore) PermanentDelete(userID string) error { start := time.Now() diff --git a/server/cmd/mmctl/commands/importer/validate.go b/server/cmd/mmctl/commands/importer/validate.go index f9bac13d56..147c1210c3 100644 --- a/server/cmd/mmctl/commands/importer/validate.go +++ b/server/cmd/mmctl/commands/importer/validate.go @@ -83,6 +83,7 @@ const ( LineTypeTeam = "team" LineTypeChannel = "channel" LineTypeUser = "user" + LineTypeBot = "bot" LineTypePost = "post" LineTypeDirectChannel = "direct_channel" LineTypeDirectPost = "direct_post" @@ -410,6 +411,8 @@ func (v *Validator) validateLine(info ImportFileInfo, line imports.LineImportDat err = v.validateChannel(info, line) case LineTypeUser: err = v.validateUser(info, line) + case LineTypeBot: + err = v.validateBot(info, line) case LineTypePost: err = v.validatePost(info, line) case LineTypeDirectChannel: @@ -725,6 +728,36 @@ func (v *Validator) validateUser(info ImportFileInfo, line imports.LineImportDat return nil } +func (v *Validator) validateBot(info ImportFileInfo, line imports.LineImportData) (err error) { + ivErr := validateNotNil(info, "bot", line.Bot, func(data imports.BotImportData) *ImportValidationError { + appErr := imports.ValidateBotImportData(&data) + if appErr != nil { + return &ImportValidationError{ + ImportFileInfo: info, + FieldName: "bot", + Err: appErr, + } + } + + if data.Username != nil { + // e-mails are for bots are converted to the the username@localhost format + // see model.BotFromUser + botMail := model.NormalizeEmail(fmt.Sprintf("%s@localhost", *data.Username)) + if ive := v.checkDuplicateUser(info, *data.Username, botMail); ive != nil { + return ive + } + v.users[*data.Username] = info + } + + return nil + }) + if ivErr != nil { + return v.onError(ivErr) + } + + return nil +} + func (v *Validator) validatePost(info ImportFileInfo, line imports.LineImportData) (err error) { ivErr := validateNotNil(info, "post", line.Post, func(data imports.PostImportData) *ImportValidationError { appErr := imports.ValidatePostImportData(&data, v.maxPostSize) diff --git a/server/cmd/mmctl/commands/sampledata_util.go b/server/cmd/mmctl/commands/sampledata_util.go index 8d3714e31e..a5d9bec440 100644 --- a/server/cmd/mmctl/commands/sampledata_util.go +++ b/server/cmd/mmctl/commands/sampledata_util.go @@ -215,7 +215,9 @@ func createUser(idx int, teamMemberships int, channelMemberships int, teamsAndCh } user := imports.UserImportData{ - ProfileImage: profileImage, + Avatar: imports.Avatar{ + ProfileImage: profileImage, + }, Username: &username, Email: &email, Password: &password, diff --git a/server/i18n/en.json b/server/i18n/en.json index 756deaeef8..193c47cded 100644 --- a/server/i18n/en.json +++ b/server/i18n/en.json @@ -4478,6 +4478,10 @@ "id": "app.bot.permenent_delete.bad_id", "translation": "Unable to delete the bot." }, + { + "id": "app.bot.update.app_error", + "translation": "Unable to update the bot." + }, { "id": "app.channel.add_member.deleted_user.app_error", "translation": "Unable to add the user as a member of the channel." @@ -5210,6 +5214,10 @@ "id": "app.import.get_users_by_username.some_users_not_found.error", "translation": "Some users not found" }, + { + "id": "app.import.import_bot.owner_could_not_found.error", + "translation": "Unable to find owner of the bot" + }, { "id": "app.import.import_channel.deleting.app_error", "translation": "Unable to archive imported channel." @@ -5254,6 +5262,10 @@ "id": "app.import.import_direct_post.create_group_channel.error", "translation": "Failed to get group channel" }, + { + "id": "app.import.import_line.null_bot.error", + "translation": "Import data line has type \"bot\" but the bot object is null" + }, { "id": "app.import.import_line.null_channel.error", "translation": "Import data line has type \"channel\" but the channel object is null." @@ -5350,6 +5362,10 @@ "id": "app.import.process_import_data_file_version_line.invalid_version.error", "translation": "Unable to read the version of the data import file." }, + { + "id": "app.import.validate_bot_import_data.owner_missing.error", + "translation": "Bot owner is missing" + }, { "id": "app.import.validate_channel_import_data.display_name_length.error", "translation": "Channel display_name is not within permitted length constraints."