diff --git a/api4/file.go b/api4/file.go index c1cf36d4a5..f81da72bf8 100644 --- a/api4/file.go +++ b/api4/file.go @@ -482,6 +482,7 @@ func getFile(c *Context, w http.ResponseWriter, r *http.Request) { info, err := c.App.GetFileInfo(c.Params.FileId) if err != nil { c.Err = err + setInaccessibleFileHeader(w, err) return } auditRec.AddMeta("file", info) @@ -514,6 +515,7 @@ func getFileThumbnail(c *Context, w http.ResponseWriter, r *http.Request) { info, err := c.App.GetFileInfo(c.Params.FileId) if err != nil { c.Err = err + setInaccessibleFileHeader(w, err) return } @@ -555,6 +557,7 @@ func getFileLink(c *Context, w http.ResponseWriter, r *http.Request) { info, err := c.App.GetFileInfo(c.Params.FileId) if err != nil { c.Err = err + setInaccessibleFileHeader(w, err) return } auditRec.AddMeta("file", info) @@ -589,6 +592,7 @@ func getFilePreview(c *Context, w http.ResponseWriter, r *http.Request) { info, err := c.App.GetFileInfo(c.Params.FileId) if err != nil { c.Err = err + setInaccessibleFileHeader(w, err) return } @@ -622,6 +626,7 @@ func getFileInfo(c *Context, w http.ResponseWriter, r *http.Request) { info, err := c.App.GetFileInfo(c.Params.FileId) if err != nil { c.Err = err + setInaccessibleFileHeader(w, err) return } @@ -650,6 +655,7 @@ func getPublicFile(c *Context, w http.ResponseWriter, r *http.Request) { info, err := c.App.GetFileInfo(c.Params.FileId) if err != nil { c.Err = err + setInaccessibleFileHeader(w, err) return } @@ -824,3 +830,10 @@ func searchFiles(c *Context, w http.ResponseWriter, r *http.Request, teamID stri c.Logger.Warn("Error while writing response", mlog.Err(err)) } } + +func setInaccessibleFileHeader(w http.ResponseWriter, appErr *model.AppError) { + // File is inaccessible due to cloud plan's limit. + if appErr.Id == "app.file.cloud.get.app_error" { + w.Header().Set(model.HeaderFirstInaccessibleFileTime, "1") + } +} diff --git a/app/app_iface.go b/app/app_iface.go index 952362d63e..25fbb1694c 100644 --- a/app/app_iface.go +++ b/app/app_iface.go @@ -76,6 +76,9 @@ type AppIface interface { CheckProviderAttributes(user *model.User, patch *model.UserPatch) string // ClientConfigWithComputed gets the configuration in a format suitable for sending to the client. ClientConfigWithComputed() map[string]string + // ComputeLastAccessibleFileTime updates cache with CreateAt time of the last accessible file as per the cloud plan's limit. + // Use GetLastAccessibleFileTime() to access the result. + ComputeLastAccessibleFileTime() error // ComputeLastAccessiblePostTime updates cache with CreateAt time of the last accessible post as per the cloud plan's limit. // Use GetLastAccessiblePostTime() to access the result. ComputeLastAccessiblePostTime() error @@ -176,6 +179,8 @@ type AppIface interface { // GetEnvironmentConfig returns a map of configuration keys whose values have been overridden by an environment variable. // If filter is not nil and returns false for a struct field, that field will be omitted. GetEnvironmentConfig(filter func(reflect.StructField) bool) map[string]any + // GetFileInfosForPost also returns firstInaccessibleFileTime based on cloud plan's limit. + GetFileInfosForPost(postID string, fromMaster bool, includeDeleted bool) ([]*model.FileInfo, int64, *model.AppError) // GetFilteredUsersStats is used to get a count of users based on the set of filters supported by UserCountOptions. GetFilteredUsersStats(options *model.UserCountOptions) (*model.UsersStats, *model.AppError) // GetGroupsByTeam returns the paged list and the total count of group associated to the given team. @@ -186,6 +191,8 @@ type AppIface interface { // relationship with a user. That means any user sharing any channel, including // direct and group channels. GetKnownUsers(userID string) ([]string, *model.AppError) + // GetLastAccessibleFileTime returns CreateAt time(from cache) of the last accessible post as per the cloud limit + GetLastAccessibleFileTime() (int64, *model.AppError) // GetLastAccessiblePostTime returns CreateAt time(from cache) of the last accessible post as per the cloud limit GetLastAccessiblePostTime() (int64, *model.AppError) // GetLdapGroup retrieves a single LDAP group by the given LDAP group id. @@ -634,7 +641,6 @@ type AppIface interface { GetFile(fileID string) ([]byte, *model.AppError) GetFileInfo(fileID string) (*model.FileInfo, *model.AppError) GetFileInfos(page, perPage int, opt *model.GetFileInfosOptions) ([]*model.FileInfo, *model.AppError) - GetFileInfosForPost(postID string, fromMaster bool, includeDeleted bool) ([]*model.FileInfo, *model.AppError) GetFileInfosForPostWithMigration(postID string, includeDeleted bool) ([]*model.FileInfo, *model.AppError) GetFlaggedPosts(userID string, offset int, limit int) (*model.PostList, *model.AppError) GetFlaggedPostsForChannel(userID, channelID string, offset int, limit int) (*model.PostList, *model.AppError) diff --git a/app/file.go b/app/file.go index c461d1765b..d2520c5f62 100644 --- a/app/file.go +++ b/app/file.go @@ -12,12 +12,14 @@ import ( "fmt" "image" "io" + "math" "net/http" "net/url" "os" "path" "path/filepath" "regexp" + "strconv" "strings" "sync" "time" @@ -1082,11 +1084,30 @@ func (s *Server) getFileInfo(fileID string) (*model.FileInfo, *model.AppError) { } func (a *App) GetFileInfo(fileID string) (*model.FileInfo, *model.AppError) { - fileInfo, err := a.Srv().getFileInfo(fileID) - if err == nil { + fileInfo, appErr := a.Srv().getFileInfo(fileID) + if appErr != nil { + return nil, appErr + } + + firstInaccessibleFileTime, appErr := a.isInaccessibleFile(fileInfo) + if appErr != nil { + return nil, appErr + } + if firstInaccessibleFileTime > 0 { + return nil, model.NewAppError("GetFileInfo", "app.file.cloud.get.app_error", nil, "", http.StatusForbidden) + } + + a.generateMiniPreview(fileInfo) + return fileInfo, appErr +} + +func (a *App) getFileInfoIgnoreCloudLimit(fileID string) (*model.FileInfo, *model.AppError) { + fileInfo, appErr := a.Srv().getFileInfo(fileID) + if appErr == nil { a.generateMiniPreview(fileInfo) } - return fileInfo, err + + return fileInfo, appErr } func (a *App) GetFileInfos(page, perPage int, opt *model.GetFileInfosOptions) ([]*model.FileInfo, *model.AppError) { @@ -1104,6 +1125,16 @@ func (a *App) GetFileInfos(page, perPage int, opt *model.GetFileInfosOptions) ([ } } + filterOptions := filterFileOptions{} + if opt != nil && (opt.SortBy == "" || opt.SortBy == model.FileinfoSortByCreated) { + filterOptions.assumeSortedCreatedAt = true + } + + fileInfos, _, appErr := a.getFilteredAccessibleFiles(fileInfos, filterOptions) + if appErr != nil { + return nil, appErr + } + a.generateMiniPreviewForInfos(fileInfos) return fileInfos, nil @@ -1123,6 +1154,20 @@ func (a *App) GetFile(fileID string) ([]byte, *model.AppError) { return data, nil } +func (a *App) getFileIgnoreCloudLimit(fileID string) ([]byte, *model.AppError) { + info, err := a.getFileInfoIgnoreCloudLimit(fileID) + if err != nil { + return nil, err + } + + data, err := a.ReadFile(info.Path) + if err != nil { + return nil, err + } + + return data, nil +} + func (a *App) CopyFileInfos(userID string, fileIDs []string) ([]string, *model.AppError) { var newFileIds []string @@ -1252,7 +1297,7 @@ func (a *App) SearchFilesInTeamForUser(c *request.Context, terms string, userId } } - return fileInfoSearchResults, nil + return fileInfoSearchResults, a.filterInaccessibleFiles(fileInfoSearchResults, filterFileOptions{assumeSortedCreatedAt: true}) } func (a *App) ExtractContentFromFileInfo(fileInfo *model.FileInfo) error { @@ -1288,3 +1333,79 @@ func (a *App) ExtractContentFromFileInfo(fileInfo *model.FileInfo) error { } return nil } + +// GetLastAccessibleFileTime returns CreateAt time(from cache) of the last accessible post as per the cloud limit +func (a *App) GetLastAccessibleFileTime() (int64, *model.AppError) { + license := a.Srv().License() + if license == nil || !*license.Features.Cloud { + return 0, nil + } + + system, err := a.Srv().Store.System().GetByName(model.SystemLastAccessibleFileTime) + if err != nil { + var nfErr *store.ErrNotFound + switch { + case errors.As(err, &nfErr): + // All files are accessible + return 0, nil + default: + return 0, model.NewAppError("GetLastAccessibleFileTime", "app.system.get_by_name.app_error", nil, err.Error(), http.StatusInternalServerError) + } + } + + lastAccessibleFileTime, err := strconv.ParseInt(system.Value, 10, 64) + if err != nil { + return 0, model.NewAppError("GetLastAccessibleFileTime", "common.parse_error_int64", map[string]interface{}{"Value": system.Value}, err.Error(), http.StatusInternalServerError) + } + + return lastAccessibleFileTime, nil +} + +// ComputeLastAccessibleFileTime updates cache with CreateAt time of the last accessible file as per the cloud plan's limit. +// Use GetLastAccessibleFileTime() to access the result. +func (a *App) ComputeLastAccessibleFileTime() error { + limit, appErr := a.getCloudFilesSizeLimit() + if appErr != nil { + return appErr + } + + createdAt, err := a.Srv().GetStore().FileInfo().GetUptoNSizeFileTime(limit) + if err != nil { + var nfErr *store.ErrNotFound + if !errors.As(err, &nfErr) { + return model.NewAppError("ComputeLastAccessibleFileTime", "app.last_accessible_file.app_error", nil, err.Error(), http.StatusInternalServerError) + } + } + + // Update Cache + err = a.Srv().Store.System().SaveOrUpdate(&model.System{ + Name: model.SystemLastAccessibleFileTime, + Value: strconv.FormatInt(createdAt, 10), + }) + if err != nil { + return model.NewAppError("ComputeLastAccessibleFileTime", "app.system.save.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + return nil +} + +// getCloudFilesSizeLimit returns size in bytes +func (a *App) getCloudFilesSizeLimit() (int64, *model.AppError) { + license := a.Srv().License() + if license == nil || !*license.Features.Cloud { + return 0, nil + } + + // limits is in bits + limits, err := a.Cloud().GetCloudLimits("") + if err != nil { + return 0, model.NewAppError("getCloudFilesSizeLimit", "api.cloud.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + if limits == nil || limits.Files == nil || limits.Files.TotalStorage == nil { + // Cloud limit is not applicable + return 0, nil + } + + return int64(math.Ceil(float64(*limits.Files.TotalStorage) / 8)), nil +} diff --git a/app/file_helper.go b/app/file_helper.go new file mode 100644 index 0000000000..a3059ff30b --- /dev/null +++ b/app/file_helper.go @@ -0,0 +1,210 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package app + +import ( + "net/http" + + "github.com/mattermost/mattermost-server/v6/model" +) + +// removeInaccessibleContentFromFilesSlice removes content from the files beyond the cloud plan's limit +// and also returns the firstInaccessibleFileTime +func (a *App) removeInaccessibleContentFromFilesSlice(files []*model.FileInfo) (int64, *model.AppError) { + if len(files) == 0 { + return 0, nil + } + + lastAccessibleFileTime, appErr := a.GetLastAccessibleFileTime() + if appErr != nil { + return 0, model.NewAppError("removeInaccessibleFileListContent", "app.last_accessible_file.app_error", nil, appErr.Error(), http.StatusInternalServerError) + } + if lastAccessibleFileTime == 0 { + // No need to remove content, all files are accessible + return 0, nil + } + + var firstInaccessibleFileTime int64 = 0 + for _, file := range files { + if createAt := file.CreateAt; createAt < lastAccessibleFileTime { + file.MakeContentInaccessible() + if createAt > firstInaccessibleFileTime { + firstInaccessibleFileTime = createAt + } + } + } + + return firstInaccessibleFileTime, nil +} + +// filterInaccessibleFiles filters out the files, past the cloud limit +func (a *App) filterInaccessibleFiles(fileList *model.FileInfoList, options filterFileOptions) *model.AppError { + if fileList == nil || fileList.FileInfos == nil || len(fileList.FileInfos) == 0 { + return nil + } + + lastAccessibleFileTime, appErr := a.GetLastAccessibleFileTime() + if appErr != nil { + return model.NewAppError("filterInaccessibleFiles", "app.last_accessible_file.app_error", nil, appErr.Error(), http.StatusInternalServerError) + } + if lastAccessibleFileTime == 0 { + // No need to filter, all files are accessible + return nil + } + + if len(fileList.FileInfos) == len(fileList.Order) && options.assumeSortedCreatedAt { + lenFiles := len(fileList.FileInfos) + getCreateAt := func(i int) int64 { return fileList.FileInfos[fileList.Order[i]].CreateAt } + + bounds := getTimeSortedPostAccessibleBounds(lastAccessibleFileTime, lenFiles, getCreateAt) + + if bounds.allAccessible(lenFiles) { + return nil + } + if bounds.noAccessible() { + if lenFiles > 0 { + firstFileCreatedAt := fileList.FileInfos[fileList.Order[0]].CreateAt + lastFileCreatedAt := fileList.FileInfos[fileList.Order[lenFiles-1]].CreateAt + fileList.FirstInaccessibleFileTime = max(firstFileCreatedAt, lastFileCreatedAt) + } + fileList.FileInfos = map[string]*model.FileInfo{} + fileList.Order = []string{} + return nil + } + startInaccessibleIndex, endInaccessibleIndex := bounds.getInaccessibleRange(len(fileList.Order)) + startInaccessibleCreatedAt := fileList.FileInfos[fileList.Order[startInaccessibleIndex]].CreateAt + endInaccessibleCreatedAt := fileList.FileInfos[fileList.Order[endInaccessibleIndex]].CreateAt + fileList.FirstInaccessibleFileTime = max(startInaccessibleCreatedAt, endInaccessibleCreatedAt) + + files := fileList.FileInfos + order := fileList.Order + accessibleCount := bounds.end - bounds.start + 1 + inaccessibleCount := lenFiles - accessibleCount + // Linearly cover shorter route to traverse files map + if inaccessibleCount < accessibleCount { + for i := 0; i < bounds.start; i++ { + delete(files, order[i]) + } + for i := bounds.end + 1; i < lenFiles; i++ { + delete(files, order[i]) + } + } else { + accessibleFiles := make(map[string]*model.FileInfo, accessibleCount) + for i := bounds.start; i <= bounds.end; i++ { + accessibleFiles[order[i]] = files[order[i]] + } + fileList.FileInfos = accessibleFiles + } + + fileList.Order = fileList.Order[bounds.start : bounds.end+1] + } else { + linearFilterFileList(fileList, lastAccessibleFileTime) + } + + return nil +} + +// isInaccessibleFile indicates if the file is past the cloud plan's limit. +func (a *App) isInaccessibleFile(file *model.FileInfo) (int64, *model.AppError) { + if file == nil { + return 0, nil + } + + fl := &model.FileInfoList{ + Order: []string{file.Id}, + FileInfos: map[string]*model.FileInfo{file.Id: file}, + } + + appErr := a.filterInaccessibleFiles(fl, filterFileOptions{assumeSortedCreatedAt: true}) + return fl.FirstInaccessibleFileTime, appErr +} + +// getFilteredAccessibleFiles returns accessible files filtered as per the cloud plan's limit and also indicates if there were any inaccessible files +func (a *App) getFilteredAccessibleFiles(files []*model.FileInfo, options filterFileOptions) ([]*model.FileInfo, int64, *model.AppError) { + if len(files) == 0 { + return files, 0, nil + } + + filteredFiles := []*model.FileInfo{} + lastAccessibleFileTime, appErr := a.GetLastAccessibleFileTime() + if appErr != nil { + return filteredFiles, 0, model.NewAppError("getFilteredAccessibleFiles", "app.last_accessible_file.app_error", nil, appErr.Error(), http.StatusInternalServerError) + } else if lastAccessibleFileTime == 0 { + // No need to filter, all files are accessible + return files, 0, nil + } + + if options.assumeSortedCreatedAt { + lenFiles := len(files) + getCreateAt := func(i int) int64 { return files[i].CreateAt } + bounds := getTimeSortedPostAccessibleBounds(lastAccessibleFileTime, lenFiles, getCreateAt) + if bounds.allAccessible(lenFiles) { + return files, 0, nil + } + if bounds.noAccessible() { + var firstInaccessibleFileTime int64 = 0 + if lenFiles > 0 { + firstFileCreatedAt := files[0].CreateAt + lastFileCreatedAt := files[len(files)-1].CreateAt + firstInaccessibleFileTime = max(firstFileCreatedAt, lastFileCreatedAt) + } + return filteredFiles, firstInaccessibleFileTime, nil + } + + startInaccessibleIndex, endInaccessibleIndex := bounds.getInaccessibleRange(len(files)) + firstFileCreatedAt := files[startInaccessibleIndex].CreateAt + lastFileCreatedAt := files[endInaccessibleIndex].CreateAt + firstInaccessibleFileTime := max(firstFileCreatedAt, lastFileCreatedAt) + filteredFiles = files[bounds.start : bounds.end+1] + return filteredFiles, firstInaccessibleFileTime, nil + } + + filteredFiles, firstInaccessibleFileTime := linearFilterFilesSlice(files, lastAccessibleFileTime) + return filteredFiles, firstInaccessibleFileTime, nil +} + +type filterFileOptions struct { + assumeSortedCreatedAt bool +} + +// linearFilterFileList make no assumptions about ordering, go through files one by one +// this is the slower fallback that is still safe +// if we can not assume files are ordered by CreatedAt +func linearFilterFileList(fileList *model.FileInfoList, earliestAccessibleTime int64) { + files := fileList.FileInfos + order := fileList.Order + + n := 0 + for i, fileID := range order { + if createAt := files[fileID].CreateAt; createAt >= earliestAccessibleTime { + order[n] = order[i] + n++ + } else { + if createAt > fileList.FirstInaccessibleFileTime { + fileList.FirstInaccessibleFileTime = createAt + } + delete(files, fileID) + } + } + fileList.Order = order[:n] +} + +// linearFilterFilesSlice make no assumptions about ordering, go through files one by one +// this is the slower fallback that is still safe +// if we can not assume files are ordered by CreatedAt +func linearFilterFilesSlice(files []*model.FileInfo, earliestAccessibleTime int64) ([]*model.FileInfo, int64) { + var firstInaccessibleFileTime int64 = 0 + n := 0 + for i := range files { + if createAt := files[i].CreateAt; createAt >= earliestAccessibleTime { + files[n] = files[i] + n++ + } else { + if createAt > firstInaccessibleFileTime { + firstInaccessibleFileTime = createAt + } + } + } + return files[:n], firstInaccessibleFileTime +} diff --git a/app/file_helper_test.go b/app/file_helper_test.go new file mode 100644 index 0000000000..57842ccdbb --- /dev/null +++ b/app/file_helper_test.go @@ -0,0 +1,206 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package app + +import ( + "testing" + + "github.com/mattermost/mattermost-server/v6/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFilterInaccessibleFiles(t *testing.T) { + th := Setup(t) + th.App.Srv().SetLicense(model.NewTestLicense("cloud")) + th.App.Srv().Store.System().Save(&model.System{ + Name: model.SystemLastAccessibleFileTime, + Value: "2", + }) + + defer th.TearDown() + + var getFileWithCreateAt = func(at int64) *model.FileInfo { + return &model.FileInfo{CreateAt: at} + } + + t.Run("ascending order returns correct files", func(t *testing.T) { + fileList := &model.FileInfoList{ + FileInfos: map[string]*model.FileInfo{ + "file_a": getFileWithCreateAt(0), + "file_b": getFileWithCreateAt(1), + "file_c": getFileWithCreateAt(2), + "file_d": getFileWithCreateAt(3), + "file_e": getFileWithCreateAt(4), + }, + Order: []string{"file_a", "file_b", "file_c", "file_d", "file_e"}, + } + appErr := th.App.filterInaccessibleFiles(fileList, filterFileOptions{assumeSortedCreatedAt: true}) + + require.Nil(t, appErr) + + assert.Equal(t, map[string]*model.FileInfo{ + "file_c": getFileWithCreateAt(2), + "file_d": getFileWithCreateAt(3), + "file_e": getFileWithCreateAt(4), + }, fileList.FileInfos) + + assert.Equal(t, []string{ + "file_c", + "file_d", + "file_e", + }, fileList.Order) + assert.Equal(t, int64(1), fileList.FirstInaccessibleFileTime) + }) + + t.Run("descending order returns correct files", func(t *testing.T) { + fileList := &model.FileInfoList{ + FileInfos: map[string]*model.FileInfo{ + "file_a": getFileWithCreateAt(0), + "file_b": getFileWithCreateAt(1), + "file_c": getFileWithCreateAt(2), + "file_d": getFileWithCreateAt(3), + "file_e": getFileWithCreateAt(4), + }, + Order: []string{"file_e", "file_d", "file_c", "file_b", "file_a"}, + } + appErr := th.App.filterInaccessibleFiles(fileList, filterFileOptions{assumeSortedCreatedAt: true}) + + require.Nil(t, appErr) + + assert.Equal(t, map[string]*model.FileInfo{ + "file_c": getFileWithCreateAt(2), + "file_d": getFileWithCreateAt(3), + "file_e": getFileWithCreateAt(4), + }, fileList.FileInfos) + + assert.Equal(t, []string{ + "file_e", + "file_d", + "file_c", + }, fileList.Order) + + assert.Equal(t, int64(1), fileList.FirstInaccessibleFileTime) + }) + + t.Run("handles mixed create at ordering correctly if correct options given", func(t *testing.T) { + fileList := &model.FileInfoList{ + FileInfos: map[string]*model.FileInfo{ + "file_a": getFileWithCreateAt(0), + "file_b": getFileWithCreateAt(1), + "file_c": getFileWithCreateAt(2), + "file_d": getFileWithCreateAt(3), + "file_e": getFileWithCreateAt(4), + }, + Order: []string{"file_e", "file_b", "file_a", "file_d", "file_c"}, + } + appErr := th.App.filterInaccessibleFiles(fileList, filterFileOptions{assumeSortedCreatedAt: false}) + + require.Nil(t, appErr) + + assert.Equal(t, map[string]*model.FileInfo{ + "file_c": getFileWithCreateAt(2), + "file_d": getFileWithCreateAt(3), + "file_e": getFileWithCreateAt(4), + }, fileList.FileInfos) + + assert.Equal(t, []string{ + "file_e", + "file_d", + "file_c", + }, fileList.Order) + }) +} + +func TestGetFilteredAccessibleFiles(t *testing.T) { + th := Setup(t) + th.App.Srv().SetLicense(model.NewTestLicense("cloud")) + th.App.Srv().Store.System().Save(&model.System{ + Name: model.SystemLastAccessibleFileTime, + Value: "2", + }) + + defer th.TearDown() + + var getFileWithCreateAt = func(at int64) *model.FileInfo { + return &model.FileInfo{CreateAt: at} + } + + t.Run("ascending order returns correct files", func(t *testing.T) { + files := []*model.FileInfo{getFileWithCreateAt(0), getFileWithCreateAt(1), getFileWithCreateAt(2), getFileWithCreateAt(3), getFileWithCreateAt(4)} + filteredFiles, firstInaccessibleFileTime, appErr := th.App.getFilteredAccessibleFiles(files, filterFileOptions{assumeSortedCreatedAt: true}) + + require.Nil(t, appErr) + assert.Equal(t, []*model.FileInfo{getFileWithCreateAt(2), getFileWithCreateAt(3), getFileWithCreateAt(4)}, filteredFiles) + assert.Equal(t, int64(1), firstInaccessibleFileTime) + }) + + t.Run("descending order returns correct files", func(t *testing.T) { + files := []*model.FileInfo{getFileWithCreateAt(4), getFileWithCreateAt(3), getFileWithCreateAt(2), getFileWithCreateAt(1), getFileWithCreateAt(0)} + filteredFiles, firstInaccessibleFileTime, appErr := th.App.getFilteredAccessibleFiles(files, filterFileOptions{assumeSortedCreatedAt: true}) + + require.Nil(t, appErr) + assert.Equal(t, []*model.FileInfo{getFileWithCreateAt(4), getFileWithCreateAt(3), getFileWithCreateAt(2)}, filteredFiles) + assert.Equal(t, int64(1), firstInaccessibleFileTime) + }) + + t.Run("handles mixed create at ordering correctly if correct options given", func(t *testing.T) { + files := []*model.FileInfo{getFileWithCreateAt(4), getFileWithCreateAt(1), getFileWithCreateAt(0), getFileWithCreateAt(3), getFileWithCreateAt(2)} + filteredFiles, _, appErr := th.App.getFilteredAccessibleFiles(files, filterFileOptions{assumeSortedCreatedAt: false}) + + require.Nil(t, appErr) + assert.Equal(t, []*model.FileInfo{getFileWithCreateAt(4), getFileWithCreateAt(3), getFileWithCreateAt(2)}, filteredFiles) + }) +} + +func TestIsInaccessibleFile(t *testing.T) { + th := Setup(t) + th.App.Srv().SetLicense(model.NewTestLicense("cloud")) + th.App.Srv().Store.System().Save(&model.System{ + Name: model.SystemLastAccessibleFileTime, + Value: "2", + }) + + defer th.TearDown() + + file := &model.FileInfo{CreateAt: 3} + firstInaccessibleFileTime, appErr := th.App.isInaccessibleFile(file) + require.Nil(t, appErr) + assert.Equal(t, int64(0), firstInaccessibleFileTime) + + file = &model.FileInfo{CreateAt: 1} + firstInaccessibleFileTime, appErr = th.App.isInaccessibleFile(file) + require.Nil(t, appErr) + assert.Equal(t, int64(1), firstInaccessibleFileTime) +} + +func TestRemoveInaccessibleContentFromFilesSlice(t *testing.T) { + th := Setup(t) + th.App.Srv().SetLicense(model.NewTestLicense("cloud")) + th.App.Srv().Store.System().Save(&model.System{ + Name: model.SystemLastAccessibleFileTime, + Value: "2", + }) + + defer th.TearDown() + + var getFileWithCreateAt = func(at int64) *model.FileInfo { + return &model.FileInfo{CreateAt: at} + } + + files := []*model.FileInfo{getFileWithCreateAt(4), getFileWithCreateAt(1), getFileWithCreateAt(0), getFileWithCreateAt(3), getFileWithCreateAt(2)} + + _, appErr := th.App.removeInaccessibleContentFromFilesSlice(files) + + require.Nil(t, appErr) + assert.Len(t, files, len(files)) + for _, file := range files { + // Inaccessible files are archived + if file.CreateAt < 2 { + assert.True(t, file.Archived) + } else { + assert.False(t, file.Archived) + } + } +} diff --git a/app/file_test.go b/app/file_test.go index 6f84894b8f..c30a2df471 100644 --- a/app/file_test.go +++ b/app/file_test.go @@ -16,9 +16,12 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + eMocks "github.com/mattermost/mattermost-server/v6/einterfaces/mocks" "github.com/mattermost/mattermost-server/v6/model" "github.com/mattermost/mattermost-server/v6/services/searchengine/mocks" filesStoreMocks "github.com/mattermost/mattermost-server/v6/shared/filestore/mocks" + "github.com/mattermost/mattermost-server/v6/store" + storemocks "github.com/mattermost/mattermost-server/v6/store/storetest/mocks" "github.com/mattermost/mattermost-server/v6/utils/fileutils" ) @@ -553,3 +556,65 @@ func TestExtractContentFromFileInfo(t *testing.T) { // Test that we don't process images. require.NoError(t, app.ExtractContentFromFileInfo(fi)) } + +func TestGetLastAccessibleFileTime(t *testing.T) { + th := SetupWithStoreMock(t) + defer th.TearDown() + + r, err := th.App.GetLastAccessibleFileTime() + require.Nil(t, err) + assert.Equal(t, int64(0), r) + + th.App.Srv().SetLicense(model.NewTestLicense("cloud")) + + mockStore := th.App.Srv().Store.(*storemocks.Store) + + mockSystemStore := storemocks.SystemStore{} + mockStore.On("System").Return(&mockSystemStore) + mockSystemStore.On("GetByName", mock.Anything).Return(nil, store.NewErrNotFound("", "")) + r, err = th.App.GetLastAccessibleFileTime() + require.Nil(t, err) + assert.Equal(t, int64(0), r) + + mockSystemStore = storemocks.SystemStore{} + mockStore.On("System").Return(&mockSystemStore) + mockSystemStore.On("GetByName", mock.Anything).Return(nil, errors.New("test")) + _, err = th.App.GetLastAccessibleFileTime() + require.NotNil(t, err) + + mockSystemStore = storemocks.SystemStore{} + mockStore.On("System").Return(&mockSystemStore) + mockSystemStore.On("GetByName", mock.Anything).Return(&model.System{Name: model.SystemLastAccessibleFileTime, Value: "10"}, nil) + r, err = th.App.GetLastAccessibleFileTime() + require.Nil(t, err) + assert.Equal(t, int64(10), r) +} + +func TestComputeLastAccessibleFileTime(t *testing.T) { + th := SetupWithStoreMock(t) + defer th.TearDown() + + th.App.Srv().SetLicense(model.NewTestLicense("cloud")) + + cloud := &eMocks.CloudInterface{} + th.App.Srv().Cloud = cloud + + cloud.Mock.On("GetCloudLimits", mock.Anything).Return(&model.ProductLimits{ + Files: &model.FilesLimits{ + TotalStorage: model.NewInt64(1), + }, + }, nil) + + mockStore := th.App.Srv().Store.(*storemocks.Store) + mockFileStore := storemocks.FileInfoStore{} + mockFileStore.On("GetUptoNSizeFileTime", mock.Anything).Return(int64(1), nil) + mockSystemStore := storemocks.SystemStore{} + mockSystemStore.On("SaveOrUpdate", mock.Anything).Return(nil) + mockStore.On("FileInfo").Return(&mockFileStore) + mockStore.On("System").Return(&mockSystemStore) + + err := th.App.ComputeLastAccessibleFileTime() + require.NoError(t, err) + + mockSystemStore.AssertCalled(t, "SaveOrUpdate", mock.Anything) +} diff --git a/app/import_functions.go b/app/import_functions.go index 837a4b875c..240c01ba34 100644 --- a/app/import_functions.go +++ b/app/import_functions.go @@ -1225,7 +1225,7 @@ func (a *App) importAttachment(c request.CTX, data *imports.AttachmentImportData // Go over existing files in the post and see if there already exists a file with the same name, size and hash. If so - skip it if post.Id != "" { - oldFiles, err := a.GetFileInfosForPost(post.Id, true, false) + oldFiles, err := a.getFileInfosForPostIgnoreCloudLimit(post.Id, true, false) if err != nil { return nil, model.NewAppError("BulkImport", "app.import.attachment.file_upload.error", map[string]any{"FilePath": *data.Path}, "", http.StatusBadRequest) } @@ -1235,7 +1235,7 @@ func (a *App) importAttachment(c request.CTX, data *imports.AttachmentImportData } // check md5 newHash := sha1.Sum(fileData) - oldFileData, err := a.GetFile(oldFile.Id) + oldFileData, err := a.getFileIgnoreCloudLimit(oldFile.Id) if err != nil { return nil, model.NewAppError("BulkImport", "app.import.attachment.file_upload.error", map[string]any{"FilePath": *data.Path}, "", http.StatusBadRequest) } diff --git a/app/opentracing/opentracing_layer.go b/app/opentracing/opentracing_layer.go index 1869099fb7..30379abfe2 100644 --- a/app/opentracing/opentracing_layer.go +++ b/app/opentracing/opentracing_layer.go @@ -1783,6 +1783,28 @@ func (a *OpenTracingAppLayer) CompleteSwitchWithOAuth(service string, userData i return resultVar0, resultVar1 } +func (a *OpenTracingAppLayer) ComputeLastAccessibleFileTime() error { + origCtx := a.ctx + span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ComputeLastAccessibleFileTime") + + a.ctx = newCtx + a.app.Srv().Store.SetContext(newCtx) + defer func() { + a.app.Srv().Store.SetContext(origCtx) + a.ctx = origCtx + }() + + defer span.Finish() + resultVar0 := a.app.ComputeLastAccessibleFileTime() + + if resultVar0 != nil { + span.LogFields(spanlog.Error(resultVar0)) + ext.Error.Set(span, true) + } + + return resultVar0 +} + func (a *OpenTracingAppLayer) ComputeLastAccessiblePostTime() error { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ComputeLastAccessiblePostTime") @@ -6053,7 +6075,7 @@ func (a *OpenTracingAppLayer) GetFileInfos(page int, perPage int, opt *model.Get return resultVar0, resultVar1 } -func (a *OpenTracingAppLayer) GetFileInfosForPost(postID string, fromMaster bool, includeDeleted bool) ([]*model.FileInfo, *model.AppError) { +func (a *OpenTracingAppLayer) GetFileInfosForPost(postID string, fromMaster bool, includeDeleted bool) ([]*model.FileInfo, int64, *model.AppError) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetFileInfosForPost") @@ -6065,14 +6087,14 @@ func (a *OpenTracingAppLayer) GetFileInfosForPost(postID string, fromMaster bool }() defer span.Finish() - resultVar0, resultVar1 := a.app.GetFileInfosForPost(postID, fromMaster, includeDeleted) + resultVar0, resultVar1, resultVar2 := a.app.GetFileInfosForPost(postID, fromMaster, includeDeleted) - if resultVar1 != nil { - span.LogFields(spanlog.Error(resultVar1)) + if resultVar2 != nil { + span.LogFields(spanlog.Error(resultVar2)) ext.Error.Set(span, true) } - return resultVar0, resultVar1 + return resultVar0, resultVar1, resultVar2 } func (a *OpenTracingAppLayer) GetFileInfosForPostWithMigration(postID string, includeDeleted bool) ([]*model.FileInfo, *model.AppError) { @@ -6884,6 +6906,28 @@ func (a *OpenTracingAppLayer) GetKnownUsers(userID string) ([]string, *model.App return resultVar0, resultVar1 } +func (a *OpenTracingAppLayer) GetLastAccessibleFileTime() (int64, *model.AppError) { + origCtx := a.ctx + span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetLastAccessibleFileTime") + + a.ctx = newCtx + a.app.Srv().Store.SetContext(newCtx) + defer func() { + a.app.Srv().Store.SetContext(origCtx) + a.ctx = origCtx + }() + + defer span.Finish() + resultVar0, resultVar1 := a.app.GetLastAccessibleFileTime() + + if resultVar1 != nil { + span.LogFields(spanlog.Error(resultVar1)) + ext.Error.Set(span, true) + } + + return resultVar0, resultVar1 +} + func (a *OpenTracingAppLayer) GetLastAccessiblePostTime() (int64, *model.AppError) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetLastAccessiblePostTime") diff --git a/app/post.go b/app/post.go index 0a6c300e5c..ea5249ef09 100644 --- a/app/post.go +++ b/app/post.go @@ -1558,12 +1558,12 @@ func (a *App) GetFileInfosForPostWithMigration(postID string, includeDeleted boo close(pchan) }() - infos, err := a.GetFileInfosForPost(postID, false, includeDeleted) + infos, firstInaccessibleFileTime, err := a.GetFileInfosForPost(postID, false, includeDeleted) if err != nil { return nil, err } - if len(infos) == 0 { + if len(infos) == 0 && firstInaccessibleFileTime == 0 { // No FileInfos were returned so check if they need to be created for this post result := <-pchan if result.NErr != nil { @@ -1588,10 +1588,27 @@ func (a *App) GetFileInfosForPostWithMigration(postID string, includeDeleted boo return infos, nil } -func (a *App) GetFileInfosForPost(postID string, fromMaster bool, includeDeleted bool) ([]*model.FileInfo, *model.AppError) { +// GetFileInfosForPost also returns firstInaccessibleFileTime based on cloud plan's limit. +func (a *App) GetFileInfosForPost(postID string, fromMaster bool, includeDeleted bool) ([]*model.FileInfo, int64, *model.AppError) { fileInfos, err := a.Srv().Store.FileInfo().GetForPost(postID, fromMaster, includeDeleted, true) if err != nil { - return nil, model.NewAppError("GetFileInfosForPost", "app.file_info.get_for_post.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + return nil, 0, model.NewAppError("GetFileInfosForPost", "app.file_info.get_for_post.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + firstInaccessibleFileTime, appErr := a.removeInaccessibleContentFromFilesSlice(fileInfos) + if appErr != nil { + return nil, 0, appErr + } + + a.generateMiniPreviewForInfos(fileInfos) + + return fileInfos, firstInaccessibleFileTime, nil +} + +func (a *App) getFileInfosForPostIgnoreCloudLimit(postID string, fromMaster bool, includeDeleted bool) ([]*model.FileInfo, *model.AppError) { + fileInfos, err := a.Srv().Store.FileInfo().GetForPost(postID, fromMaster, includeDeleted, true) + if err != nil { + return nil, model.NewAppError("getFileInfosForPostIgnoreCloudLimit", "app.file_info.get_for_post.app_error", nil, "", http.StatusInternalServerError).Wrap(err) } a.generateMiniPreviewForInfos(fileInfos) diff --git a/app/post_metadata.go b/app/post_metadata.go index f7155e0a7c..b863ac21f4 100644 --- a/app/post_metadata.go +++ b/app/post_metadata.go @@ -123,7 +123,7 @@ func (a *App) PreparePostForClient(originalPost *model.Post, isNewPost, isEditPo } // Files - if fileInfos, err := a.getFileMetadataForPost(post, isNewPost || isEditPost); err != nil { + if fileInfos, _, err := a.getFileMetadataForPost(post, isNewPost || isEditPost); err != nil { mlog.Warn("Failed to get files for a post", mlog.String("post_id", post.Id), mlog.Err(err)) } else { post.Metadata.Files = fileInfos @@ -210,9 +210,9 @@ func (a *App) SanitizePostListMetadataForUser(c request.CTX, postList *model.Pos return clonedPostList, nil } -func (a *App) getFileMetadataForPost(post *model.Post, fromMaster bool) ([]*model.FileInfo, *model.AppError) { +func (a *App) getFileMetadataForPost(post *model.Post, fromMaster bool) ([]*model.FileInfo, int64, *model.AppError) { if len(post.FileIds) == 0 { - return nil, nil + return nil, 0, nil } return a.GetFileInfosForPost(post.Id, fromMaster, false) diff --git a/app/post_test.go b/app/post_test.go index 800511abf9..c51eabb5db 100644 --- a/app/post_test.go +++ b/app/post_test.go @@ -219,7 +219,7 @@ func TestAttachFilesToPost(t *testing.T) { appErr := th.App.attachFilesToPost(post) assert.Nil(t, appErr) - infos, appErr := th.App.GetFileInfosForPost(post.Id, false, false) + infos, _, appErr := th.App.GetFileInfosForPost(post.Id, false, false) assert.Nil(t, appErr) assert.Len(t, infos, 2) }) @@ -247,7 +247,7 @@ func TestAttachFilesToPost(t *testing.T) { appErr := th.App.attachFilesToPost(post) assert.Nil(t, appErr) - infos, appErr := th.App.GetFileInfosForPost(post.Id, false, false) + infos, _, appErr := th.App.GetFileInfosForPost(post.Id, false, false) assert.Nil(t, appErr) assert.Len(t, infos, 1) assert.Equal(t, info2.Id, infos[0].Id) diff --git a/app/server.go b/app/server.go index 3e3fd0a18b..997c5a3f64 100644 --- a/app/server.go +++ b/app/server.go @@ -46,6 +46,7 @@ import ( "github.com/mattermost/mattermost-server/v6/jobs/extract_content" "github.com/mattermost/mattermost-server/v6/jobs/import_delete" "github.com/mattermost/mattermost-server/v6/jobs/import_process" + "github.com/mattermost/mattermost-server/v6/jobs/last_accessible_file" "github.com/mattermost/mattermost-server/v6/jobs/last_accessible_post" "github.com/mattermost/mattermost-server/v6/jobs/migrations" "github.com/mattermost/mattermost-server/v6/jobs/notify_admin" @@ -1870,6 +1871,12 @@ func (s *Server) initJobs() { last_accessible_post.MakeScheduler(s.Jobs, s.License()), ) + s.Jobs.RegisterJobType( + model.JobTypeLastAccessibleFile, + last_accessible_file.MakeWorker(s.Jobs, s.License(), New(ServerConnector(s.Channels()))), + last_accessible_file.MakeScheduler(s.Jobs, s.License()), + ) + s.Jobs.RegisterJobType( model.JobTypeUpgradeNotifyAdmin, notify_admin.MakeUpgradeNotifyWorker(s.Jobs, s.License(), New(ServerConnector(s.Channels()))), diff --git a/i18n/en.json b/i18n/en.json index eb49526f56..0b4e92e034 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -4987,6 +4987,10 @@ "id": "app.export.zip_create.error", "translation": "Failed to add file to zip archive during export." }, + { + "id": "app.file.cloud.get.app_error", + "translation": "Can not fetch the file as it is past the cloud plan's limit." + }, { "id": "app.file_info.get.app_error", "translation": "Unable to get the file info." @@ -5639,6 +5643,10 @@ "id": "app.job.update.app_error", "translation": "Unable to update the job." }, + { + "id": "app.last_accessible_file.app_error", + "translation": "Error fetching last accessible file" + }, { "id": "app.last_accessible_post.app_error", "translation": "Error fetching last accessible post" diff --git a/jobs/last_accessible_file/scheduler.go b/jobs/last_accessible_file/scheduler.go new file mode 100644 index 0000000000..d351278c20 --- /dev/null +++ b/jobs/last_accessible_file/scheduler.go @@ -0,0 +1,24 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package last_accessible_file + +import ( + "strconv" + "time" + + "github.com/mattermost/mattermost-server/v6/jobs" + "github.com/mattermost/mattermost-server/v6/model" + "github.com/mattermost/mattermost-server/v6/shared/mlog" +) + +const schedFreq = 2 * time.Hour + +func MakeScheduler(jobServer *jobs.JobServer, license *model.License) model.Scheduler { + isEnabled := func(cfg *model.Config) bool { + enabled := license != nil && *license.Features.Cloud + mlog.Debug("Scheduler: isEnabled: "+strconv.FormatBool(enabled), mlog.String("scheduler", model.JobTypeLastAccessibleFile)) + return enabled + } + return jobs.NewPeriodicScheduler(jobServer, model.JobTypeLastAccessibleFile, schedFreq, isEnabled) +} diff --git a/jobs/last_accessible_file/worker.go b/jobs/last_accessible_file/worker.go new file mode 100644 index 0000000000..177c1016b0 --- /dev/null +++ b/jobs/last_accessible_file/worker.go @@ -0,0 +1,28 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package last_accessible_file + +import ( + "github.com/mattermost/mattermost-server/v6/jobs" + "github.com/mattermost/mattermost-server/v6/model" +) + +const ( + JobName = "LastAccessibleFile" +) + +type AppIface interface { + ComputeLastAccessibleFileTime() error +} + +func MakeWorker(jobServer *jobs.JobServer, license *model.License, app AppIface) model.Worker { + isEnabled := func(_ *model.Config) bool { + return license != nil && *license.Features.Cloud + } + execute := func(_ *model.Job) error { + return app.ComputeLastAccessibleFileTime() + } + worker := jobs.NewSimpleWorker(JobName, jobServer, execute, isEnabled) + return worker +} diff --git a/model/client4.go b/model/client4.go index f00852a77e..cfe0f9244e 100644 --- a/model/client4.go +++ b/model/client4.go @@ -35,6 +35,7 @@ const ( HeaderRequestedWith = "X-Requested-With" HeaderRequestedWithXML = "XMLHttpRequest" HeaderFirstInaccessiblePostTime = "First-Inaccessible-Post-Time" + HeaderFirstInaccessibleFileTime = "First-Inaccessible-File-Time" HeaderRange = "Range" STATUS = "status" StatusOk = "OK" diff --git a/model/file_info.go b/model/file_info.go index f6ae41ab36..4aa0a84fa2 100644 --- a/model/file_info.go +++ b/model/file_info.go @@ -188,3 +188,17 @@ func GetEtagForFileInfos(infos []*FileInfo) string { return Etag(infos[0].PostId, maxUpdateAt) } + +func (fi *FileInfo) MakeContentInaccessible() { + if fi == nil { + return + } + + fi.Archived = true + fi.Content = "" + fi.HasPreviewImage = false + fi.MiniPreview = nil + fi.Path = "" + fi.PreviewPath = "" + fi.ThumbnailPath = "" +} diff --git a/model/file_info_list.go b/model/file_info_list.go index 219b24f1dc..e5de2d076b 100644 --- a/model/file_info_list.go +++ b/model/file_info_list.go @@ -12,6 +12,8 @@ type FileInfoList struct { FileInfos map[string]*FileInfo `json:"file_infos"` NextFileInfoId string `json:"next_file_info_id"` PrevFileInfoId string `json:"prev_file_info_id"` + // If there are inaccessible files, FirstInaccessibleFileTime is the time of the latest inaccessible file + FirstInaccessibleFileTime int64 `json:"first_inaccessible_file_time"` } func NewFileInfoList() *FileInfoList { diff --git a/model/job.go b/model/job.go index 0868643557..9b03430718 100644 --- a/model/job.go +++ b/model/job.go @@ -28,6 +28,7 @@ const ( JobTypeResendInvitationEmail = "resend_invitation_email" JobTypeExtractContent = "extract_content" JobTypeLastAccessiblePost = "last_accessible_post" + JobTypeLastAccessibleFile = "last_accessible_file" JobTypeUpgradeNotifyAdmin = "upgrade_notify_admin" JobTypeTrialNotifyAdmin = "trial_notify_admin" @@ -59,6 +60,7 @@ var AllJobTypes = [...]string{ JobTypeCloud, JobTypeExtractContent, JobTypeLastAccessiblePost, + JobTypeLastAccessibleFile, } type Job struct { diff --git a/model/system.go b/model/system.go index 10370d0e08..0ae731f983 100644 --- a/model/system.go +++ b/model/system.go @@ -32,6 +32,7 @@ const ( SystemFirstAdminVisitMarketplace = "FirstAdminVisitMarketplace" SystemFirstAdminSetupComplete = "FirstAdminSetupComplete" SystemLastAccessiblePostTime = "LastAccessiblePostTime" + SystemLastAccessibleFileTime = "LastAccessibleFileTime" AwsMeteringReportInterval = 1 AwsMeteringDimensionUsageHrs = "UsageHrs" ) diff --git a/store/opentracinglayer/opentracinglayer.go b/store/opentracinglayer/opentracinglayer.go index 45b6277728..eb65b1f798 100644 --- a/store/opentracinglayer/opentracinglayer.go +++ b/store/opentracinglayer/opentracinglayer.go @@ -3527,6 +3527,24 @@ func (s *OpenTracingLayerFileInfoStore) GetStorageUsage(allowFromCache bool, inc return result, err } +func (s *OpenTracingLayerFileInfoStore) GetUptoNSizeFileTime(n int64) (int64, error) { + origCtx := s.Root.Store.Context() + span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.GetUptoNSizeFileTime") + s.Root.Store.SetContext(newCtx) + defer func() { + s.Root.Store.SetContext(origCtx) + }() + + defer span.Finish() + result, err := s.FileInfoStore.GetUptoNSizeFileTime(n) + if err != nil { + span.LogFields(spanlog.Error(err)) + ext.Error.Set(span, true) + } + + return result, err +} + func (s *OpenTracingLayerFileInfoStore) GetWithOptions(page int, perPage int, opt *model.GetFileInfosOptions) ([]*model.FileInfo, error) { origCtx := s.Root.Store.Context() span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.GetWithOptions") diff --git a/store/retrylayer/retrylayer.go b/store/retrylayer/retrylayer.go index 6e544138a2..4efa7172ba 100644 --- a/store/retrylayer/retrylayer.go +++ b/store/retrylayer/retrylayer.go @@ -3957,6 +3957,27 @@ func (s *RetryLayerFileInfoStore) GetStorageUsage(allowFromCache bool, includeDe } +func (s *RetryLayerFileInfoStore) GetUptoNSizeFileTime(n int64) (int64, error) { + + tries := 0 + for { + result, err := s.FileInfoStore.GetUptoNSizeFileTime(n) + if err == nil { + return result, nil + } + if !isRepeatableError(err) { + return result, err + } + tries++ + if tries >= 3 { + err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures") + return result, err + } + timepkg.Sleep(100 * timepkg.Millisecond) + } + +} + func (s *RetryLayerFileInfoStore) GetWithOptions(page int, perPage int, opt *model.GetFileInfosOptions) ([]*model.FileInfo, error) { tries := 0 diff --git a/store/sqlstore/file_info_store.go b/store/sqlstore/file_info_store.go index 4a6eb7e655..dbf47020e2 100644 --- a/store/sqlstore/file_info_store.go +++ b/store/sqlstore/file_info_store.go @@ -753,3 +753,49 @@ func (fs SqlFileInfoStore) GetStorageUsage(allowFromCache, includeDeleted bool) } return size, nil } + +// GetUptoNSizeFileTime returns the CreateAt time of the last accessible file with a running-total size upto n bytes. +func (fs *SqlFileInfoStore) GetUptoNSizeFileTime(n int64) (int64, error) { + if n <= 0 { + return 0, errors.New("n can't be less than 1") + } + + var sizeSubQuery sq.SelectBuilder + // Separate query for MySql, as current min-version 5.x doesn't support window-functions + if fs.DriverName() == model.DatabaseDriverMysql { + sizeSubQuery = sq. + Select("(@runningSum := @runningSum + fi.Size) RunningTotal", "fi.CreateAt"). + From("FileInfo fi"). + Join("(SELECT @runningSum := 0) as tmp"). + Where(sq.Eq{"fi.DeleteAt": 0}). + OrderBy("fi.CreateAt DESC, fi.Id") + } else { + sizeSubQuery = sq. + Select("SUM(fi.Size) OVER(ORDER BY CreateAt DESC, fi.Id) RunningTotal", "fi.CreateAt"). + From("FileInfo fi"). + Where(sq.Eq{"fi.DeleteAt": 0}) + } + + builder := fs.getQueryBuilder(). + Select("fi2.CreateAt"). + FromSelect(sizeSubQuery, "fi2"). + Where(sq.LtOrEq{"fi2.RunningTotal": n}). + OrderBy("fi2.CreateAt"). + Limit(1) + + query, queryArgs, err := builder.ToSql() + if err != nil { + return 0, errors.Wrap(err, "GetUptoNSizeFileTime_tosql") + } + + var createAt int64 + if err := fs.GetReplicaX().Get(&createAt, query, queryArgs...); err != nil { + if err == sql.ErrNoRows { + return 0, store.NewErrNotFound("File", "none") + } + + return 0, errors.Wrapf(err, "failed to get the File for size upto=%d", n) + } + + return createAt, nil +} diff --git a/store/store.go b/store/store.go index df2473b701..4984612a0b 100644 --- a/store/store.go +++ b/store/store.go @@ -698,6 +698,8 @@ type FileInfoStore interface { GetFilesBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.FileForIndexing, error) ClearCaches() GetStorageUsage(allowFromCache, includeDeleted bool) (int64, error) + // GetUptoNSizeFileTime returns the CreateAt time of the last accessible file with a running-total size upto n bytes. + GetUptoNSizeFileTime(n int64) (int64, error) } type UploadSessionStore interface { diff --git a/store/storetest/file_info_store.go b/store/storetest/file_info_store.go index 34cd2da114..1dc72b0536 100644 --- a/store/storetest/file_info_store.go +++ b/store/storetest/file_info_store.go @@ -11,6 +11,7 @@ import ( "github.com/mattermost/mattermost-server/v6/model" "github.com/mattermost/mattermost-server/v6/store" + "github.com/mattermost/mattermost-server/v6/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -30,6 +31,7 @@ func TestFileInfoStore(t *testing.T, ss store.Store) { t.Run("GetFilesBatchForIndexing", func(t *testing.T) { testFileInfoStoreGetFilesBatchForIndexing(t, ss) }) t.Run("CountAll", func(t *testing.T) { testFileInfoStoreCountAll(t, ss) }) t.Run("GetStorageUsage", func(t *testing.T) { testFileInfoGetStorageUsage(t, ss) }) + t.Run("GetUptoNSizeFileTime", func(t *testing.T) { testGetUptoNSizeFileTime(t, ss) }) } func testFileInfoSaveGet(t *testing.T, ss store.Store) { @@ -772,3 +774,71 @@ func testFileInfoGetStorageUsage(t *testing.T, ss store.Store) { require.NoError(t, err) require.Equal(t, int64(30), usage) } + +func testGetUptoNSizeFileTime(t *testing.T, ss store.Store) { + _, err := ss.FileInfo().GetUptoNSizeFileTime(0) + assert.Error(t, err) + _, err = ss.FileInfo().GetUptoNSizeFileTime(-1) + assert.Error(t, err) + + _, err = ss.FileInfo().PermanentDeleteBatch(model.GetMillis(), 100000) + require.NoError(t, err) + + diff := int64(10000) + now := utils.MillisFromTime(time.Now()) + diff + + f1, err := ss.FileInfo().Save(&model.FileInfo{ + PostId: model.NewId(), + CreatorId: model.NewId(), + Size: 10, + Path: "file1.txt", + CreateAt: now, + }) + require.NoError(t, err) + now = now + diff + f2, err := ss.FileInfo().Save(&model.FileInfo{ + PostId: model.NewId(), + CreatorId: model.NewId(), + Size: 10, + Path: "file2.txt", + CreateAt: now, + }) + require.NoError(t, err) + now = now + diff + f3, err := ss.FileInfo().Save(&model.FileInfo{ + PostId: model.NewId(), + CreatorId: model.NewId(), + Size: 10, + Path: "file3.txt", + CreateAt: now, + }) + require.NoError(t, err) + now = now + diff + _, err = ss.FileInfo().Save(&model.FileInfo{ + PostId: model.NewId(), + CreatorId: model.NewId(), + Size: 10, + Path: "file4.txt", + CreateAt: now, + }) + require.NoError(t, err) + + createAt, err := ss.FileInfo().GetUptoNSizeFileTime(20) + require.NoError(t, err) + assert.Equal(t, f3.CreateAt, createAt) + + _, err = ss.FileInfo().GetUptoNSizeFileTime(5) + assert.Error(t, err) + assert.IsType(t, &store.ErrNotFound{}, err) + + createAt, err = ss.FileInfo().GetUptoNSizeFileTime(1000) + require.NoError(t, err) + assert.Equal(t, f1.CreateAt, createAt) + + _, err = ss.FileInfo().DeleteForPost(f3.PostId) + require.NoError(t, err) + + createAt, err = ss.FileInfo().GetUptoNSizeFileTime(20) + require.NoError(t, err) + assert.Equal(t, f2.CreateAt, createAt) +} diff --git a/store/storetest/mocks/FileInfoStore.go b/store/storetest/mocks/FileInfoStore.go index 97432a11f3..c9627a84ad 100644 --- a/store/storetest/mocks/FileInfoStore.go +++ b/store/storetest/mocks/FileInfoStore.go @@ -257,6 +257,27 @@ func (_m *FileInfoStore) GetStorageUsage(allowFromCache bool, includeDeleted boo return r0, r1 } +// GetUptoNSizeFileTime provides a mock function with given fields: n +func (_m *FileInfoStore) GetUptoNSizeFileTime(n int64) (int64, error) { + ret := _m.Called(n) + + var r0 int64 + if rf, ok := ret.Get(0).(func(int64) int64); ok { + r0 = rf(n) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(int64) error); ok { + r1 = rf(n) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetWithOptions provides a mock function with given fields: page, perPage, opt func (_m *FileInfoStore) GetWithOptions(page int, perPage int, opt *model.GetFileInfosOptions) ([]*model.FileInfo, error) { ret := _m.Called(page, perPage, opt) diff --git a/store/timerlayer/timerlayer.go b/store/timerlayer/timerlayer.go index 3a4e40ed39..0d7b338767 100644 --- a/store/timerlayer/timerlayer.go +++ b/store/timerlayer/timerlayer.go @@ -3222,6 +3222,22 @@ func (s *TimerLayerFileInfoStore) GetStorageUsage(allowFromCache bool, includeDe return result, err } +func (s *TimerLayerFileInfoStore) GetUptoNSizeFileTime(n int64) (int64, error) { + start := time.Now() + + result, err := s.FileInfoStore.GetUptoNSizeFileTime(n) + + elapsed := float64(time.Since(start)) / float64(time.Second) + if s.Root.Metrics != nil { + success := "false" + if err == nil { + success = "true" + } + s.Root.Metrics.ObserveStoreMethodDuration("FileInfoStore.GetUptoNSizeFileTime", success, elapsed) + } + return result, err +} + func (s *TimerLayerFileInfoStore) GetWithOptions(page int, perPage int, opt *model.GetFileInfosOptions) ([]*model.FileInfo, error) { start := time.Now()