diff --git a/i18n/en.json b/i18n/en.json index eb2af52a1d..90bc6506f1 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -5158,6 +5158,10 @@ "id": "app.post.get_direct_posts.app_error", "translation": "Unable to get direct posts." }, + { + "id": "app.post.get_files_batch_for_indexing.get.app_error", + "translation": "Unable to get the files batch for indexing." + }, { "id": "app.post.get_flagged_posts.app_error", "translation": "Unable to get the flagged posts." @@ -6078,6 +6082,10 @@ "id": "bleveengine.purge_channel_index.error", "translation": "Failed to purge channel indexes." }, + { + "id": "bleveengine.purge_file_index.error", + "translation": "Failed to purge file indexes." + }, { "id": "bleveengine.purge_post_index.error", "translation": "Failed to purge post indexes." @@ -6434,6 +6442,10 @@ "id": "ent.elasticsearch.not_started.error", "translation": "Elasticsearch is not started" }, + { + "id": "ent.elasticsearch.post.get_files_batch_for_indexing.error", + "translation": "Unable to get the files batch for indexing." + }, { "id": "ent.elasticsearch.post.get_posts_batch_for_indexing.error", "translation": "Unable to get the posts batch for indexing." diff --git a/model/feature_flags.go b/model/feature_flags.go index 7be0905de7..de390e652a 100644 --- a/model/feature_flags.go +++ b/model/feature_flags.go @@ -24,6 +24,8 @@ type FeatureFlags struct { // Feature flags to control plugin versions PluginIncidentManagement string `plugin_id:"com.mattermost.plugin-incident-management"` + // Toggle on and off support for Files search + FilesSearch bool } func (f *FeatureFlags) SetDefaults() { @@ -31,6 +33,7 @@ func (f *FeatureFlags) SetDefaults() { f.TestBoolFeature = false f.CloudDelinquentEmailJobsEnabled = false f.CollapsedThreads = false + f.FilesSearch = false f.CustomUserStatuses = false f.PluginIncidentManagement = "1.4.0" } diff --git a/services/searchengine/bleveengine/bleve.go b/services/searchengine/bleveengine/bleve.go index 04c6a70c8b..1e5dcba056 100644 --- a/services/searchengine/bleveengine/bleve.go +++ b/services/searchengine/bleveengine/bleve.go @@ -257,6 +257,9 @@ func (b *BleveEngine) deleteIndexes() *model.AppError { if err := os.RemoveAll(b.getIndexDir(ChannelIndex)); err != nil { return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_channel_index.error", nil, err.Error(), http.StatusInternalServerError) } + if err := os.RemoveAll(b.getIndexDir(FileIndex)); err != nil { + return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_file_index.error", nil, err.Error(), http.StatusInternalServerError) + } return nil } diff --git a/services/searchengine/bleveengine/common.go b/services/searchengine/bleveengine/common.go index 97bb0624d2..ef4b125d93 100644 --- a/services/searchengine/bleveengine/common.go +++ b/services/searchengine/bleveengine/common.go @@ -125,6 +125,13 @@ func BLVPostFromPostForIndexing(post *model.PostForIndexing) *BLVPost { } } +func splitFilenameWords(name string) string { + result := name + result = strings.ReplaceAll(result, "-", " ") + result = strings.ReplaceAll(result, ".", " ") + return result +} + func BLVFileFromFileInfo(fileInfo *model.FileInfo, channelId string) *BLVFile { return &BLVFile{ Id: fileInfo.Id, @@ -133,7 +140,7 @@ func BLVFileFromFileInfo(fileInfo *model.FileInfo, channelId string) *BLVFile { CreateAt: fileInfo.CreateAt, Content: fileInfo.Content, Extension: fileInfo.Extension, - Name: fileInfo.Name, + Name: fileInfo.Name + " " + splitFilenameWords(fileInfo.Name), } } @@ -145,6 +152,6 @@ func BLVFileFromFileForIndexing(file *model.FileForIndexing) *BLVFile { CreateAt: file.CreateAt, Content: file.Content, Extension: file.Extension, - Name: file.Name, + Name: file.Name + " " + splitFilenameWords(file.Name), } } diff --git a/services/searchengine/bleveengine/indexer/indexing_job.go b/services/searchengine/bleveengine/indexer/indexing_job.go index bfdd835872..f2eaca87da 100644 --- a/services/searchengine/bleveengine/indexer/indexing_job.go +++ b/services/searchengine/bleveengine/indexer/indexing_job.go @@ -21,6 +21,7 @@ const ( BatchSize = 1000 TimeBetweenBatches = 100 EstimatedPostCount = 10000000 + EstimatedFilesCount = 100000 EstimatedChannelCount = 100000 EstimatedUserCount = 10000 ) @@ -68,6 +69,9 @@ type IndexingProgress struct { TotalPostsCount int64 DonePostsCount int64 DonePosts bool + TotalFilesCount int64 + DoneFilesCount int64 + DoneFiles bool TotalChannelsCount int64 DoneChannelsCount int64 DoneChannels bool @@ -77,11 +81,11 @@ type IndexingProgress struct { } func (ip *IndexingProgress) CurrentProgress() int64 { - return (ip.DonePostsCount + ip.DoneChannelsCount + ip.DoneUsersCount) * 100 / (ip.TotalPostsCount + ip.TotalChannelsCount + ip.TotalUsersCount) + return (ip.DonePostsCount + ip.DoneChannelsCount + ip.DoneUsersCount + ip.DoneFilesCount) * 100 / (ip.TotalPostsCount + ip.TotalChannelsCount + ip.TotalUsersCount + ip.TotalFilesCount) } func (ip *IndexingProgress) IsDone() bool { - return ip.DonePosts && ip.DoneChannels && ip.DoneUsers + return ip.DonePosts && ip.DoneChannels && ip.DoneUsers && ip.DoneFiles } func (worker *BleveIndexerWorker) JobChannel() chan<- model.Job { @@ -139,10 +143,15 @@ func (worker *BleveIndexerWorker) DoJob(job *model.Job) { DonePosts: false, DoneChannels: false, DoneUsers: false, + DoneFiles: false, StartAtTime: 0, EndAtTime: model.GetMillis(), } + if !worker.jobServer.Config().FeatureFlags.FilesSearch { + progress.DoneFiles = true + } + // Extract the start and end times, if they are set. if startString, ok := job.Data["start_time"]; ok { startInt, err := strconv.ParseInt(startString, 10, 64) @@ -209,6 +218,15 @@ func (worker *BleveIndexerWorker) DoJob(job *model.Job) { progress.TotalUsersCount = count } + // Counting all files may fail or timeout when the file_info table is large. If this happens, log a warning, but carry + // on with the indexing job anyway. The only issue is that the progress % reporting will be inaccurate. + if count, err := worker.jobServer.Store.FileInfo().CountAll(); err != nil { + mlog.Warn("Worker: Failed to fetch total file info count for job. An estimated value will be used for progress reporting.", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err)) + progress.TotalFilesCount = EstimatedFilesCount + } else { + progress.TotalFilesCount = count + } + cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background()) cancelWatcherChan := make(chan interface{}, 1) go worker.jobServer.CancellationWatcher(cancelCtx, job.Id, cancelWatcherChan) @@ -273,6 +291,9 @@ func (worker *BleveIndexerWorker) IndexBatch(progress IndexingProgress) (Indexin if !progress.DoneUsers { return worker.IndexUsersBatch(progress) } + if !progress.DoneFiles { + return worker.IndexFilesBatch(progress) + } return progress, model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.index_batch.nothing_left_to_index.error", nil, "", http.StatusInternalServerError) } @@ -354,6 +375,60 @@ func (worker *BleveIndexerWorker) BulkIndexPosts(posts []*model.PostForIndexing, return lastCreateAt, nil } +func (worker *BleveIndexerWorker) IndexFilesBatch(progress IndexingProgress) (IndexingProgress, *model.AppError) { + endTime := progress.LastEntityTime + int64(*worker.jobServer.Config().BleveSettings.BulkIndexingTimeWindowSeconds*1000) + + var files []*model.FileForIndexing + + tries := 0 + for files == nil { + var err error + files, err = worker.jobServer.Store.FileInfo().GetFilesBatchForIndexing(progress.LastEntityTime, endTime, BatchSize) + if err != nil { + if tries >= 10 { + return progress, model.NewAppError("IndexFilesBatch", "app.post.get_files_batch_for_indexing.get.app_error", nil, err.Error(), http.StatusInternalServerError) + } + mlog.Warn("Failed to get files batch for indexing. Retrying.", mlog.Err(err)) + + // Wait a bit before trying again. + time.Sleep(15 * time.Second) + } + + tries++ + } + + newLastFileTime, err := worker.BulkIndexFiles(files, progress) + if err != nil { + return progress, err + } + + // Due to the "endTime" parameter in the store query, we might get an incomplete batch before the end. In this + // case, set the "newLastFileTime" to the endTime so we don't get stuck running the same query in a loop. + if len(files) < BatchSize { + newLastFileTime = endTime + } + + // When to Stop: we index either until we pass a batch of messages where the last + // message is created at or after the specified end time when setting up the batch + // index, or until two consecutive full batches have the same end time of their final + // messages. This second case is safe as long as the assumption that the database + // cannot contain more messages with the same CreateAt time than the batch size holds. + if progress.EndAtTime <= newLastFileTime { + progress.DoneFiles = true + progress.LastEntityTime = progress.StartAtTime + } else if progress.LastEntityTime == newLastFileTime && len(files) == BatchSize { + mlog.Warn("More files with the same CreateAt time were detected than the permitted batch size. Aborting indexing job.", mlog.Int64("CreateAt", newLastFileTime), mlog.Int("Batch Size", BatchSize)) + progress.DoneFiles = true + progress.LastEntityTime = progress.StartAtTime + } else { + progress.LastEntityTime = newLastFileTime + } + + progress.DoneFilesCount += int64(len(files)) + + return progress, nil +} + func (worker *BleveIndexerWorker) BulkIndexFiles(files []*model.FileForIndexing, progress IndexingProgress) (int64, *model.AppError) { lastCreateAt := int64(0) batch := worker.engine.FileIndex.NewBatch() diff --git a/store/opentracinglayer/opentracinglayer.go b/store/opentracinglayer/opentracinglayer.go index 0210622aeb..f3ec8d11ed 100644 --- a/store/opentracinglayer/opentracinglayer.go +++ b/store/opentracinglayer/opentracinglayer.go @@ -2946,6 +2946,24 @@ func (s *OpenTracingLayerFileInfoStore) ClearCaches() { } +func (s *OpenTracingLayerFileInfoStore) CountAll() (int64, error) { + origCtx := s.Root.Store.Context() + span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.CountAll") + s.Root.Store.SetContext(newCtx) + defer func() { + s.Root.Store.SetContext(origCtx) + }() + + defer span.Finish() + result, err := s.FileInfoStore.CountAll() + if err != nil { + span.LogFields(spanlog.Error(err)) + ext.Error.Set(span, true) + } + + return result, err +} + func (s *OpenTracingLayerFileInfoStore) DeleteForPost(postID string) (string, error) { origCtx := s.Root.Store.Context() span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.DeleteForPost") @@ -3018,6 +3036,24 @@ func (s *OpenTracingLayerFileInfoStore) GetByPath(path string) (*model.FileInfo, return result, err } +func (s *OpenTracingLayerFileInfoStore) GetFilesBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.FileForIndexing, error) { + origCtx := s.Root.Store.Context() + span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.GetFilesBatchForIndexing") + s.Root.Store.SetContext(newCtx) + defer func() { + s.Root.Store.SetContext(origCtx) + }() + + defer span.Finish() + result, err := s.FileInfoStore.GetFilesBatchForIndexing(startTime, endTime, limit) + if err != nil { + span.LogFields(spanlog.Error(err)) + ext.Error.Set(span, true) + } + + return result, err +} + func (s *OpenTracingLayerFileInfoStore) GetForPost(postID string, readFromMaster bool, includeDeleted bool, allowFromCache bool) ([]*model.FileInfo, error) { origCtx := s.Root.Store.Context() span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.GetForPost") diff --git a/store/retrylayer/retrylayer.go b/store/retrylayer/retrylayer.go index 00e23e9485..6142e9ebaa 100644 --- a/store/retrylayer/retrylayer.go +++ b/store/retrylayer/retrylayer.go @@ -3150,6 +3150,26 @@ func (s *RetryLayerFileInfoStore) ClearCaches() { } +func (s *RetryLayerFileInfoStore) CountAll() (int64, error) { + + tries := 0 + for { + result, err := s.FileInfoStore.CountAll() + if err == nil { + return result, nil + } + if !isRepeatableError(err) { + return result, err + } + tries++ + if tries >= 3 { + err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures") + return result, err + } + } + +} + func (s *RetryLayerFileInfoStore) DeleteForPost(postID string) (string, error) { tries := 0 @@ -3230,6 +3250,26 @@ func (s *RetryLayerFileInfoStore) GetByPath(path string) (*model.FileInfo, error } +func (s *RetryLayerFileInfoStore) GetFilesBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.FileForIndexing, error) { + + tries := 0 + for { + result, err := s.FileInfoStore.GetFilesBatchForIndexing(startTime, endTime, limit) + if err == nil { + return result, nil + } + if !isRepeatableError(err) { + return result, err + } + tries++ + if tries >= 3 { + err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures") + return result, err + } + } + +} + func (s *RetryLayerFileInfoStore) GetForPost(postID string, readFromMaster bool, includeDeleted bool, allowFromCache bool) ([]*model.FileInfo, error) { tries := 0 diff --git a/store/sqlstore/file_info_store.go b/store/sqlstore/file_info_store.go index 56a3d9e4e5..5e676d94fd 100644 --- a/store/sqlstore/file_info_store.go +++ b/store/sqlstore/file_info_store.go @@ -593,3 +593,40 @@ func (fs SqlFileInfoStore) Search(paramsList []*model.SearchParams, userId, team list.MakeNonNil() return list, nil } + +func (fs SqlFileInfoStore) CountAll() (int64, error) { + query := fs.getQueryBuilder(). + Select("COUNT(*)"). + From("FileInfo"). + Where("DeleteAt = 0") + + queryString, args, err := query.ToSql() + if err != nil { + return int64(0), errors.Wrap(err, "count_tosql") + } + + count, err := fs.GetReplica().SelectInt(queryString, args...) + if err != nil { + return int64(0), errors.Wrap(err, "failed to count Files") + } + return count, nil +} + +func (fs SqlFileInfoStore) GetFilesBatchForIndexing(startTime, endTime int64, limit int) ([]*model.FileForIndexing, error) { + var files []*model.FileForIndexing + sql, args, _ := fs.getQueryBuilder(). + Select("fi.*, p.ChannelId"). + From("FileInfo as fi"). + LeftJoin("Posts AS p ON fi.PostId = p.Id"). + Where(sq.GtOrEq{"fi.CreateAt": startTime}). + Where(sq.Lt{"fi.CreateAt": endTime}). + OrderBy("fi.CreateAt"). + Limit(uint64(limit)). + ToSql() + _, err := fs.GetSearchReplica().Select(&files, sql, args...) + if err != nil { + return nil, errors.Wrap(err, "failed to find Files") + } + + return files, nil +} diff --git a/store/store.go b/store/store.go index dd2031f15b..fbe61f3e67 100644 --- a/store/store.go +++ b/store/store.go @@ -580,6 +580,8 @@ type FileInfoStore interface { PermanentDeleteByUser(userId string) (int64, error) SetContent(fileID, content string) error Search(paramsList []*model.SearchParams, userId, teamID string, page, perPage int) (*model.FileInfoList, error) + CountAll() (int64, error) + GetFilesBatchForIndexing(startTime, endTime int64, limit int) ([]*model.FileForIndexing, error) ClearCaches() } diff --git a/store/storetest/file_info_store.go b/store/storetest/file_info_store.go index db68dd7652..00dcb98fde 100644 --- a/store/storetest/file_info_store.go +++ b/store/storetest/file_info_store.go @@ -27,6 +27,8 @@ func TestFileInfoStore(t *testing.T, ss store.Store) { t.Run("FileInfoPermanentDelete", func(t *testing.T) { testFileInfoPermanentDelete(t, ss) }) t.Run("FileInfoPermanentDeleteBatch", func(t *testing.T) { testFileInfoPermanentDeleteBatch(t, ss) }) t.Run("FileInfoPermanentDeleteByUser", func(t *testing.T) { testFileInfoPermanentDeleteByUser(t, ss) }) + t.Run("GetFilesBatchForIndexing", func(t *testing.T) { testFileInfoStoreGetFilesBatchForIndexing(t, ss) }) + t.Run("CountAll", func(t *testing.T) { testFileInfoStoreCountAll(t, ss) }) } func testFileInfoSaveGet(t *testing.T, ss store.Store) { @@ -602,3 +604,144 @@ func testFileInfoPermanentDeleteByUser(t *testing.T, ss store.Store) { _, err = ss.FileInfo().PermanentDeleteByUser(userId) require.NoError(t, err) } + +func testFileInfoStoreGetFilesBatchForIndexing(t *testing.T, ss store.Store) { + c1 := &model.Channel{} + c1.TeamId = model.NewId() + c1.DisplayName = "Channel1" + c1.Name = "zz" + model.NewId() + "b" + c1.Type = model.CHANNEL_OPEN + c1, _ = ss.Channel().Save(c1, -1) + + c2 := &model.Channel{} + c2.TeamId = model.NewId() + c2.DisplayName = "Channel2" + c2.Name = "zz" + model.NewId() + "b" + c2.Type = model.CHANNEL_OPEN + c2, _ = ss.Channel().Save(c2, -1) + + o1 := &model.Post{} + o1.ChannelId = c1.Id + o1.UserId = model.NewId() + o1.Message = "zz" + model.NewId() + "AAAAAAAAAAA" + o1, err := ss.Post().Save(o1) + require.Nil(t, err) + f1, err := ss.FileInfo().Save(&model.FileInfo{ + PostId: o1.Id, + CreatorId: model.NewId(), + Path: "file1.txt", + }) + require.Nil(t, err) + defer func() { + ss.FileInfo().PermanentDelete(f1.Id) + }() + time.Sleep(1 * time.Millisecond) + + o2 := &model.Post{} + o2.ChannelId = c2.Id + o2.UserId = model.NewId() + o2.Message = "zz" + model.NewId() + "CCCCCCCCC" + o2, err = ss.Post().Save(o2) + require.Nil(t, err) + + f2, err := ss.FileInfo().Save(&model.FileInfo{ + PostId: o2.Id, + CreatorId: model.NewId(), + Path: "file2.txt", + }) + require.Nil(t, err) + defer func() { + ss.FileInfo().PermanentDelete(f2.Id) + }() + time.Sleep(1 * time.Millisecond) + + o3 := &model.Post{} + o3.ChannelId = c1.Id + o3.UserId = model.NewId() + o3.ParentId = o1.Id + o3.RootId = o1.Id + o3.Message = "zz" + model.NewId() + "QQQQQQQQQQ" + o3, err = ss.Post().Save(o3) + require.Nil(t, err) + + f3, err := ss.FileInfo().Save(&model.FileInfo{ + PostId: o3.Id, + CreatorId: model.NewId(), + Path: "file3.txt", + }) + require.Nil(t, err) + defer func() { + ss.FileInfo().PermanentDelete(f3.Id) + }() + + t.Run("get all files", func(t *testing.T) { + r, err := ss.FileInfo().GetFilesBatchForIndexing(f1.CreateAt, model.GetMillis()+100000, 100) + require.Nil(t, err) + require.Len(t, r, 3, "Expected 3 posts in results. Got %v", len(r)) + for _, f := range r { + if f.Id == f1.Id { + require.Equal(t, f.ChannelId, o1.ChannelId, "Unexpected channel ID") + require.Equal(t, f.Path, "file1.txt", "Unexpected filename") + } else if f.Id == f2.Id { + require.Equal(t, f.ChannelId, o2.ChannelId, "Unexpected channel ID") + require.Equal(t, f.Path, "file2.txt", "Unexpected filename") + } else if f.Id == f3.Id { + require.Equal(t, f.ChannelId, o3.ChannelId, "Unexpected channel ID") + require.Equal(t, f.Path, "file3.txt", "Unexpected filename") + } else { + require.Fail(t, "unexpected file returned") + } + } + }) + + t.Run("get files after certain date", func(t *testing.T) { + r, err := ss.FileInfo().GetFilesBatchForIndexing(f1.CreateAt+1, model.GetMillis()+100000, 100) + require.Nil(t, err) + require.Len(t, r, 2, "Expected 2 posts in results. Got %v", len(r)) + for _, f := range r { + if f.Id == f2.Id { + require.Equal(t, f.ChannelId, o2.ChannelId, "Unexpected channel ID") + require.Equal(t, f.Path, "file2.txt", "Unexpected filename") + } else if f.Id == f3.Id { + require.Equal(t, f.ChannelId, o3.ChannelId, "Unexpected channel ID") + require.Equal(t, f.Path, "file3.txt", "Unexpected filename") + } else { + require.Fail(t, "unexpected file returned") + } + } + }) +} + +func testFileInfoStoreCountAll(t *testing.T, ss store.Store) { + _, err := ss.FileInfo().PermanentDeleteBatch(model.GetMillis(), 100000) + require.Nil(t, err) + f1, err := ss.FileInfo().Save(&model.FileInfo{ + PostId: model.NewId(), + CreatorId: model.NewId(), + Path: "file1.txt", + }) + require.Nil(t, err) + + _, err = ss.FileInfo().Save(&model.FileInfo{ + PostId: model.NewId(), + CreatorId: model.NewId(), + Path: "file2.txt", + }) + require.Nil(t, err) + _, err = ss.FileInfo().Save(&model.FileInfo{ + PostId: model.NewId(), + CreatorId: model.NewId(), + Path: "file3.txt", + }) + require.Nil(t, err) + + count, err := ss.FileInfo().CountAll() + require.Nil(t, err) + require.Equal(t, int64(3), count) + + _, err = ss.FileInfo().DeleteForPost(f1.PostId) + require.Nil(t, err) + count, err = ss.FileInfo().CountAll() + require.Nil(t, err) + require.Equal(t, int64(2), count) +} diff --git a/store/storetest/mocks/FileInfoStore.go b/store/storetest/mocks/FileInfoStore.go index 7fce35846f..c7f3c58f76 100644 --- a/store/storetest/mocks/FileInfoStore.go +++ b/store/storetest/mocks/FileInfoStore.go @@ -33,6 +33,27 @@ func (_m *FileInfoStore) ClearCaches() { _m.Called() } +// CountAll provides a mock function with given fields: +func (_m *FileInfoStore) CountAll() (int64, error) { + ret := _m.Called() + + var r0 int64 + if rf, ok := ret.Get(0).(func() int64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int64) + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // DeleteForPost provides a mock function with given fields: postID func (_m *FileInfoStore) DeleteForPost(postID string) (string, error) { ret := _m.Called(postID) @@ -123,6 +144,29 @@ func (_m *FileInfoStore) GetByPath(path string) (*model.FileInfo, error) { return r0, r1 } +// GetFilesBatchForIndexing provides a mock function with given fields: startTime, endTime, limit +func (_m *FileInfoStore) GetFilesBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.FileForIndexing, error) { + ret := _m.Called(startTime, endTime, limit) + + var r0 []*model.FileForIndexing + if rf, ok := ret.Get(0).(func(int64, int64, int) []*model.FileForIndexing); ok { + r0 = rf(startTime, endTime, limit) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*model.FileForIndexing) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(int64, int64, int) error); ok { + r1 = rf(startTime, endTime, limit) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetForPost provides a mock function with given fields: postID, readFromMaster, includeDeleted, allowFromCache func (_m *FileInfoStore) GetForPost(postID string, readFromMaster bool, includeDeleted bool, allowFromCache bool) ([]*model.FileInfo, error) { ret := _m.Called(postID, readFromMaster, includeDeleted, allowFromCache) diff --git a/store/timerlayer/timerlayer.go b/store/timerlayer/timerlayer.go index 99f936d4ff..40af5581e5 100644 --- a/store/timerlayer/timerlayer.go +++ b/store/timerlayer/timerlayer.go @@ -2702,6 +2702,22 @@ func (s *TimerLayerFileInfoStore) ClearCaches() { } } +func (s *TimerLayerFileInfoStore) CountAll() (int64, error) { + start := timemodule.Now() + + result, err := s.FileInfoStore.CountAll() + + elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second) + if s.Root.Metrics != nil { + success := "false" + if err == nil { + success = "true" + } + s.Root.Metrics.ObserveStoreMethodDuration("FileInfoStore.CountAll", success, elapsed) + } + return result, err +} + func (s *TimerLayerFileInfoStore) DeleteForPost(postID string) (string, error) { start := timemodule.Now() @@ -2766,6 +2782,22 @@ func (s *TimerLayerFileInfoStore) GetByPath(path string) (*model.FileInfo, error return result, err } +func (s *TimerLayerFileInfoStore) GetFilesBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.FileForIndexing, error) { + start := timemodule.Now() + + result, err := s.FileInfoStore.GetFilesBatchForIndexing(startTime, endTime, limit) + + elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second) + if s.Root.Metrics != nil { + success := "false" + if err == nil { + success = "true" + } + s.Root.Metrics.ObserveStoreMethodDuration("FileInfoStore.GetFilesBatchForIndexing", success, elapsed) + } + return result, err +} + func (s *TimerLayerFileInfoStore) GetForPost(postID string, readFromMaster bool, includeDeleted bool, allowFromCache bool) ([]*model.FileInfo, error) { start := timemodule.Now()