Adding bulk-indexing and improving a bit the name indexing for bleve and elasticsearch (#16704)

* Adding bulk-indexing and improving a bit the name indexing for bleve and elasticsearch

* Update services/searchengine/bleveengine/bleve.go

Co-authored-by: Mario de Frutos Dieguez <mario@defrutos.org>

* Update store/sqlstore/file_info_store.go

Co-authored-by: Mario de Frutos Dieguez <mario@defrutos.org>

* Update store/sqlstore/file_info_store.go

Co-authored-by: Mario de Frutos Dieguez <mario@defrutos.org>

* Adding tests requested in the PR review

* fixing tests

* Adding a feature flag to avoid indexing files before the feature is released

* Fixing i18n

Co-authored-by: Mario de Frutos Dieguez <mario@defrutos.org>
Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

i18n/en.json

@@ -5158,6 +5158,10 @@
     "id": "app.post.get_direct_posts.app_error",
     "translation": "Unable to get direct posts."
   },
+  {
+    "id": "app.post.get_files_batch_for_indexing.get.app_error",
+    "translation": "Unable to get the files batch for indexing."
+  },
   {
     "id": "app.post.get_flagged_posts.app_error",
     "translation": "Unable to get the flagged posts."
@@ -6078,6 +6082,10 @@
     "id": "bleveengine.purge_channel_index.error",
     "translation": "Failed to purge channel indexes."
   },
+  {
+    "id": "bleveengine.purge_file_index.error",
+    "translation": "Failed to purge file indexes."
+  },
   {
     "id": "bleveengine.purge_post_index.error",
     "translation": "Failed to purge post indexes."
@@ -6434,6 +6442,10 @@
     "id": "ent.elasticsearch.not_started.error",
     "translation": "Elasticsearch is not started"
   },
+  {
+    "id": "ent.elasticsearch.post.get_files_batch_for_indexing.error",
+    "translation": "Unable to get the files batch for indexing."
+  },
   {
     "id": "ent.elasticsearch.post.get_posts_batch_for_indexing.error",
     "translation": "Unable to get the posts batch for indexing."

model/feature_flags.go

@@ -24,6 +24,8 @@ type FeatureFlags struct {
 
 	// Feature flags to control plugin versions
 	PluginIncidentManagement string `plugin_id:"com.mattermost.plugin-incident-management"`
+	// Toggle on and off support for Files search
+	FilesSearch bool
 }
 
 func (f *FeatureFlags) SetDefaults() {
@@ -31,6 +33,7 @@ func (f *FeatureFlags) SetDefaults() {
 	f.TestBoolFeature = false
 	f.CloudDelinquentEmailJobsEnabled = false
 	f.CollapsedThreads = false
+	f.FilesSearch = false
 	f.CustomUserStatuses = false
 	f.PluginIncidentManagement = "1.4.0"
 }

services/searchengine/bleveengine/bleve.go

@@ -257,6 +257,9 @@ func (b *BleveEngine) deleteIndexes() *model.AppError {
 	if err := os.RemoveAll(b.getIndexDir(ChannelIndex)); err != nil {
 		return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_channel_index.error", nil, err.Error(), http.StatusInternalServerError)
 	}
+	if err := os.RemoveAll(b.getIndexDir(FileIndex)); err != nil {
+		return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_file_index.error", nil, err.Error(), http.StatusInternalServerError)
+	}
 	return nil
 }
 

services/searchengine/bleveengine/common.go

@@ -125,6 +125,13 @@ func BLVPostFromPostForIndexing(post *model.PostForIndexing) *BLVPost {
 	}
 }
 
+func splitFilenameWords(name string) string {
+	result := name
+	result = strings.ReplaceAll(result, "-", " ")
+	result = strings.ReplaceAll(result, ".", " ")
+	return result
+}
+
 func BLVFileFromFileInfo(fileInfo *model.FileInfo, channelId string) *BLVFile {
 	return &BLVFile{
 		Id:        fileInfo.Id,
@@ -133,7 +140,7 @@ func BLVFileFromFileInfo(fileInfo *model.FileInfo, channelId string) *BLVFile {
 		CreateAt:  fileInfo.CreateAt,
 		Content:   fileInfo.Content,
 		Extension: fileInfo.Extension,
-		Name:      fileInfo.Name,
+		Name:      fileInfo.Name + " " + splitFilenameWords(fileInfo.Name),
 	}
 }
 
@@ -145,6 +152,6 @@ func BLVFileFromFileForIndexing(file *model.FileForIndexing) *BLVFile {
 		CreateAt:  file.CreateAt,
 		Content:   file.Content,
 		Extension: file.Extension,
-		Name:      file.Name,
+		Name:      file.Name + " " + splitFilenameWords(file.Name),
 	}
 }

services/searchengine/bleveengine/indexer/indexing_job.go

@@ -21,6 +21,7 @@ const (
 	BatchSize             = 1000
 	TimeBetweenBatches    = 100
 	EstimatedPostCount    = 10000000
+	EstimatedFilesCount   = 100000
 	EstimatedChannelCount = 100000
 	EstimatedUserCount    = 10000
 )
@@ -68,6 +69,9 @@ type IndexingProgress struct {
 	TotalPostsCount    int64
 	DonePostsCount     int64
 	DonePosts          bool
+	TotalFilesCount    int64
+	DoneFilesCount     int64
+	DoneFiles          bool
 	TotalChannelsCount int64
 	DoneChannelsCount  int64
 	DoneChannels       bool
@@ -77,11 +81,11 @@ type IndexingProgress struct {
 }
 
 func (ip *IndexingProgress) CurrentProgress() int64 {
-	return (ip.DonePostsCount + ip.DoneChannelsCount + ip.DoneUsersCount) * 100 / (ip.TotalPostsCount + ip.TotalChannelsCount + ip.TotalUsersCount)
+	return (ip.DonePostsCount + ip.DoneChannelsCount + ip.DoneUsersCount + ip.DoneFilesCount) * 100 / (ip.TotalPostsCount + ip.TotalChannelsCount + ip.TotalUsersCount + ip.TotalFilesCount)
 }
 
 func (ip *IndexingProgress) IsDone() bool {
-	return ip.DonePosts && ip.DoneChannels && ip.DoneUsers
+	return ip.DonePosts && ip.DoneChannels && ip.DoneUsers && ip.DoneFiles
 }
 
 func (worker *BleveIndexerWorker) JobChannel() chan<- model.Job {
@@ -139,10 +143,15 @@ func (worker *BleveIndexerWorker) DoJob(job *model.Job) {
 		DonePosts:    false,
 		DoneChannels: false,
 		DoneUsers:    false,
+		DoneFiles:    false,
 		StartAtTime:  0,
 		EndAtTime:    model.GetMillis(),
 	}
 
+	if !worker.jobServer.Config().FeatureFlags.FilesSearch {
+		progress.DoneFiles = true
+	}
+
 	// Extract the start and end times, if they are set.
 	if startString, ok := job.Data["start_time"]; ok {
 		startInt, err := strconv.ParseInt(startString, 10, 64)
@@ -209,6 +218,15 @@ func (worker *BleveIndexerWorker) DoJob(job *model.Job) {
 		progress.TotalUsersCount = count
 	}
 
+	// Counting all files may fail or timeout when the file_info table is large. If this happens, log a warning, but carry
+	// on with the indexing job anyway. The only issue is that the progress % reporting will be inaccurate.
+	if count, err := worker.jobServer.Store.FileInfo().CountAll(); err != nil {
+		mlog.Warn("Worker: Failed to fetch total file info count for job. An estimated value will be used for progress reporting.", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+		progress.TotalFilesCount = EstimatedFilesCount
+	} else {
+		progress.TotalFilesCount = count
+	}
+
 	cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background())
 	cancelWatcherChan := make(chan interface{}, 1)
 	go worker.jobServer.CancellationWatcher(cancelCtx, job.Id, cancelWatcherChan)
@@ -273,6 +291,9 @@ func (worker *BleveIndexerWorker) IndexBatch(progress IndexingProgress) (Indexin
 	if !progress.DoneUsers {
 		return worker.IndexUsersBatch(progress)
 	}
+	if !progress.DoneFiles {
+		return worker.IndexFilesBatch(progress)
+	}
 	return progress, model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.index_batch.nothing_left_to_index.error", nil, "", http.StatusInternalServerError)
 }
 
@@ -354,6 +375,60 @@ func (worker *BleveIndexerWorker) BulkIndexPosts(posts []*model.PostForIndexing,
 	return lastCreateAt, nil
 }
 
+func (worker *BleveIndexerWorker) IndexFilesBatch(progress IndexingProgress) (IndexingProgress, *model.AppError) {
+	endTime := progress.LastEntityTime + int64(*worker.jobServer.Config().BleveSettings.BulkIndexingTimeWindowSeconds*1000)
+
+	var files []*model.FileForIndexing
+
+	tries := 0
+	for files == nil {
+		var err error
+		files, err = worker.jobServer.Store.FileInfo().GetFilesBatchForIndexing(progress.LastEntityTime, endTime, BatchSize)
+		if err != nil {
+			if tries >= 10 {
+				return progress, model.NewAppError("IndexFilesBatch", "app.post.get_files_batch_for_indexing.get.app_error", nil, err.Error(), http.StatusInternalServerError)
+			}
+			mlog.Warn("Failed to get files batch for indexing. Retrying.", mlog.Err(err))
+
+			// Wait a bit before trying again.
+			time.Sleep(15 * time.Second)
+		}
+
+		tries++
+	}
+
+	newLastFileTime, err := worker.BulkIndexFiles(files, progress)
+	if err != nil {
+		return progress, err
+	}
+
+	// Due to the "endTime" parameter in the store query, we might get an incomplete batch before the end. In this
+	// case, set the "newLastFileTime" to the endTime so we don't get stuck running the same query in a loop.
+	if len(files) < BatchSize {
+		newLastFileTime = endTime
+	}
+
+	// When to Stop: we index either until we pass a batch of messages where the last
+	// message is created at or after the specified end time when setting up the batch
+	// index, or until two consecutive full batches have the same end time of their final
+	// messages. This second case is safe as long as the assumption that the database
+	// cannot contain more messages with the same CreateAt time than the batch size holds.
+	if progress.EndAtTime <= newLastFileTime {
+		progress.DoneFiles = true
+		progress.LastEntityTime = progress.StartAtTime
+	} else if progress.LastEntityTime == newLastFileTime && len(files) == BatchSize {
+		mlog.Warn("More files with the same CreateAt time were detected than the permitted batch size. Aborting indexing job.", mlog.Int64("CreateAt", newLastFileTime), mlog.Int("Batch Size", BatchSize))
+		progress.DoneFiles = true
+		progress.LastEntityTime = progress.StartAtTime
+	} else {
+		progress.LastEntityTime = newLastFileTime
+	}
+
+	progress.DoneFilesCount += int64(len(files))
+
+	return progress, nil
+}
+
 func (worker *BleveIndexerWorker) BulkIndexFiles(files []*model.FileForIndexing, progress IndexingProgress) (int64, *model.AppError) {
 	lastCreateAt := int64(0)
 	batch := worker.engine.FileIndex.NewBatch()

store/opentracinglayer/opentracinglayer.go

@@ -2946,6 +2946,24 @@ func (s *OpenTracingLayerFileInfoStore) ClearCaches() {
 
 }
 
+func (s *OpenTracingLayerFileInfoStore) CountAll() (int64, error) {
+	origCtx := s.Root.Store.Context()
+	span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.CountAll")
+	s.Root.Store.SetContext(newCtx)
+	defer func() {
+		s.Root.Store.SetContext(origCtx)
+	}()
+
+	defer span.Finish()
+	result, err := s.FileInfoStore.CountAll()
+	if err != nil {
+		span.LogFields(spanlog.Error(err))
+		ext.Error.Set(span, true)
+	}
+
+	return result, err
+}
+
 func (s *OpenTracingLayerFileInfoStore) DeleteForPost(postID string) (string, error) {
 	origCtx := s.Root.Store.Context()
 	span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.DeleteForPost")
@@ -3018,6 +3036,24 @@ func (s *OpenTracingLayerFileInfoStore) GetByPath(path string) (*model.FileInfo,
 	return result, err
 }
 
+func (s *OpenTracingLayerFileInfoStore) GetFilesBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.FileForIndexing, error) {
+	origCtx := s.Root.Store.Context()
+	span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.GetFilesBatchForIndexing")
+	s.Root.Store.SetContext(newCtx)
+	defer func() {
+		s.Root.Store.SetContext(origCtx)
+	}()
+
+	defer span.Finish()
+	result, err := s.FileInfoStore.GetFilesBatchForIndexing(startTime, endTime, limit)
+	if err != nil {
+		span.LogFields(spanlog.Error(err))
+		ext.Error.Set(span, true)
+	}
+
+	return result, err
+}
+
 func (s *OpenTracingLayerFileInfoStore) GetForPost(postID string, readFromMaster bool, includeDeleted bool, allowFromCache bool) ([]*model.FileInfo, error) {
 	origCtx := s.Root.Store.Context()
 	span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.GetForPost")

store/retrylayer/retrylayer.go

@@ -3150,6 +3150,26 @@ func (s *RetryLayerFileInfoStore) ClearCaches() {
 
 }
 
+func (s *RetryLayerFileInfoStore) CountAll() (int64, error) {
+
+	tries := 0
+	for {
+		result, err := s.FileInfoStore.CountAll()
+		if err == nil {
+			return result, nil
+		}
+		if !isRepeatableError(err) {
+			return result, err
+		}
+		tries++
+		if tries >= 3 {
+			err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
+			return result, err
+		}
+	}
+
+}
+
 func (s *RetryLayerFileInfoStore) DeleteForPost(postID string) (string, error) {
 
 	tries := 0
@@ -3230,6 +3250,26 @@ func (s *RetryLayerFileInfoStore) GetByPath(path string) (*model.FileInfo, error
 
 }
 
+func (s *RetryLayerFileInfoStore) GetFilesBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.FileForIndexing, error) {
+
+	tries := 0
+	for {
+		result, err := s.FileInfoStore.GetFilesBatchForIndexing(startTime, endTime, limit)
+		if err == nil {
+			return result, nil
+		}
+		if !isRepeatableError(err) {
+			return result, err
+		}
+		tries++
+		if tries >= 3 {
+			err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
+			return result, err
+		}
+	}
+
+}
+
 func (s *RetryLayerFileInfoStore) GetForPost(postID string, readFromMaster bool, includeDeleted bool, allowFromCache bool) ([]*model.FileInfo, error) {
 
 	tries := 0

store/sqlstore/file_info_store.go

@@ -593,3 +593,40 @@ func (fs SqlFileInfoStore) Search(paramsList []*model.SearchParams, userId, team
 	list.MakeNonNil()
 	return list, nil
 }
+
+func (fs SqlFileInfoStore) CountAll() (int64, error) {
+	query := fs.getQueryBuilder().
+		Select("COUNT(*)").
+		From("FileInfo").
+		Where("DeleteAt = 0")
+
+	queryString, args, err := query.ToSql()
+	if err != nil {
+		return int64(0), errors.Wrap(err, "count_tosql")
+	}
+
+	count, err := fs.GetReplica().SelectInt(queryString, args...)
+	if err != nil {
+		return int64(0), errors.Wrap(err, "failed to count Files")
+	}
+	return count, nil
+}
+
+func (fs SqlFileInfoStore) GetFilesBatchForIndexing(startTime, endTime int64, limit int) ([]*model.FileForIndexing, error) {
+	var files []*model.FileForIndexing
+	sql, args, _ := fs.getQueryBuilder().
+		Select("fi.*, p.ChannelId").
+		From("FileInfo as fi").
+		LeftJoin("Posts AS p ON fi.PostId = p.Id").
+		Where(sq.GtOrEq{"fi.CreateAt": startTime}).
+		Where(sq.Lt{"fi.CreateAt": endTime}).
+		OrderBy("fi.CreateAt").
+		Limit(uint64(limit)).
+		ToSql()
+	_, err := fs.GetSearchReplica().Select(&files, sql, args...)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to find Files")
+	}
+
+	return files, nil
+}

store/store.go

@@ -580,6 +580,8 @@ type FileInfoStore interface {
 	PermanentDeleteByUser(userId string) (int64, error)
 	SetContent(fileID, content string) error
 	Search(paramsList []*model.SearchParams, userId, teamID string, page, perPage int) (*model.FileInfoList, error)
+	CountAll() (int64, error)
+	GetFilesBatchForIndexing(startTime, endTime int64, limit int) ([]*model.FileForIndexing, error)
 	ClearCaches()
 }
 

store/storetest/file_info_store.go

@@ -27,6 +27,8 @@ func TestFileInfoStore(t *testing.T, ss store.Store) {
 	t.Run("FileInfoPermanentDelete", func(t *testing.T) { testFileInfoPermanentDelete(t, ss) })
 	t.Run("FileInfoPermanentDeleteBatch", func(t *testing.T) { testFileInfoPermanentDeleteBatch(t, ss) })
 	t.Run("FileInfoPermanentDeleteByUser", func(t *testing.T) { testFileInfoPermanentDeleteByUser(t, ss) })
+	t.Run("GetFilesBatchForIndexing", func(t *testing.T) { testFileInfoStoreGetFilesBatchForIndexing(t, ss) })
+	t.Run("CountAll", func(t *testing.T) { testFileInfoStoreCountAll(t, ss) })
 }
 
 func testFileInfoSaveGet(t *testing.T, ss store.Store) {
@@ -602,3 +604,144 @@ func testFileInfoPermanentDeleteByUser(t *testing.T, ss store.Store) {
 	_, err = ss.FileInfo().PermanentDeleteByUser(userId)
 	require.NoError(t, err)
 }
+
+func testFileInfoStoreGetFilesBatchForIndexing(t *testing.T, ss store.Store) {
+	c1 := &model.Channel{}
+	c1.TeamId = model.NewId()
+	c1.DisplayName = "Channel1"
+	c1.Name = "zz" + model.NewId() + "b"
+	c1.Type = model.CHANNEL_OPEN
+	c1, _ = ss.Channel().Save(c1, -1)
+
+	c2 := &model.Channel{}
+	c2.TeamId = model.NewId()
+	c2.DisplayName = "Channel2"
+	c2.Name = "zz" + model.NewId() + "b"
+	c2.Type = model.CHANNEL_OPEN
+	c2, _ = ss.Channel().Save(c2, -1)
+
+	o1 := &model.Post{}
+	o1.ChannelId = c1.Id
+	o1.UserId = model.NewId()
+	o1.Message = "zz" + model.NewId() + "AAAAAAAAAAA"
+	o1, err := ss.Post().Save(o1)
+	require.Nil(t, err)
+	f1, err := ss.FileInfo().Save(&model.FileInfo{
+		PostId:    o1.Id,
+		CreatorId: model.NewId(),
+		Path:      "file1.txt",
+	})
+	require.Nil(t, err)
+	defer func() {
+		ss.FileInfo().PermanentDelete(f1.Id)
+	}()
+	time.Sleep(1 * time.Millisecond)
+
+	o2 := &model.Post{}
+	o2.ChannelId = c2.Id
+	o2.UserId = model.NewId()
+	o2.Message = "zz" + model.NewId() + "CCCCCCCCC"
+	o2, err = ss.Post().Save(o2)
+	require.Nil(t, err)
+
+	f2, err := ss.FileInfo().Save(&model.FileInfo{
+		PostId:    o2.Id,
+		CreatorId: model.NewId(),
+		Path:      "file2.txt",
+	})
+	require.Nil(t, err)
+	defer func() {
+		ss.FileInfo().PermanentDelete(f2.Id)
+	}()
+	time.Sleep(1 * time.Millisecond)
+
+	o3 := &model.Post{}
+	o3.ChannelId = c1.Id
+	o3.UserId = model.NewId()
+	o3.ParentId = o1.Id
+	o3.RootId = o1.Id
+	o3.Message = "zz" + model.NewId() + "QQQQQQQQQQ"
+	o3, err = ss.Post().Save(o3)
+	require.Nil(t, err)
+
+	f3, err := ss.FileInfo().Save(&model.FileInfo{
+		PostId:    o3.Id,
+		CreatorId: model.NewId(),
+		Path:      "file3.txt",
+	})
+	require.Nil(t, err)
+	defer func() {
+		ss.FileInfo().PermanentDelete(f3.Id)
+	}()
+
+	t.Run("get all files", func(t *testing.T) {
+		r, err := ss.FileInfo().GetFilesBatchForIndexing(f1.CreateAt, model.GetMillis()+100000, 100)
+		require.Nil(t, err)
+		require.Len(t, r, 3, "Expected 3 posts in results. Got %v", len(r))
+		for _, f := range r {
+			if f.Id == f1.Id {
+				require.Equal(t, f.ChannelId, o1.ChannelId, "Unexpected channel ID")
+				require.Equal(t, f.Path, "file1.txt", "Unexpected filename")
+			} else if f.Id == f2.Id {
+				require.Equal(t, f.ChannelId, o2.ChannelId, "Unexpected channel ID")
+				require.Equal(t, f.Path, "file2.txt", "Unexpected filename")
+			} else if f.Id == f3.Id {
+				require.Equal(t, f.ChannelId, o3.ChannelId, "Unexpected channel ID")
+				require.Equal(t, f.Path, "file3.txt", "Unexpected filename")
+			} else {
+				require.Fail(t, "unexpected file returned")
+			}
+		}
+	})
+
+	t.Run("get files after certain date", func(t *testing.T) {
+		r, err := ss.FileInfo().GetFilesBatchForIndexing(f1.CreateAt+1, model.GetMillis()+100000, 100)
+		require.Nil(t, err)
+		require.Len(t, r, 2, "Expected 2 posts in results. Got %v", len(r))
+		for _, f := range r {
+			if f.Id == f2.Id {
+				require.Equal(t, f.ChannelId, o2.ChannelId, "Unexpected channel ID")
+				require.Equal(t, f.Path, "file2.txt", "Unexpected filename")
+			} else if f.Id == f3.Id {
+				require.Equal(t, f.ChannelId, o3.ChannelId, "Unexpected channel ID")
+				require.Equal(t, f.Path, "file3.txt", "Unexpected filename")
+			} else {
+				require.Fail(t, "unexpected file returned")
+			}
+		}
+	})
+}
+
+func testFileInfoStoreCountAll(t *testing.T, ss store.Store) {
+	_, err := ss.FileInfo().PermanentDeleteBatch(model.GetMillis(), 100000)
+	require.Nil(t, err)
+	f1, err := ss.FileInfo().Save(&model.FileInfo{
+		PostId:    model.NewId(),
+		CreatorId: model.NewId(),
+		Path:      "file1.txt",
+	})
+	require.Nil(t, err)
+
+	_, err = ss.FileInfo().Save(&model.FileInfo{
+		PostId:    model.NewId(),
+		CreatorId: model.NewId(),
+		Path:      "file2.txt",
+	})
+	require.Nil(t, err)
+	_, err = ss.FileInfo().Save(&model.FileInfo{
+		PostId:    model.NewId(),
+		CreatorId: model.NewId(),
+		Path:      "file3.txt",
+	})
+	require.Nil(t, err)
+
+	count, err := ss.FileInfo().CountAll()
+	require.Nil(t, err)
+	require.Equal(t, int64(3), count)
+
+	_, err = ss.FileInfo().DeleteForPost(f1.PostId)
+	require.Nil(t, err)
+	count, err = ss.FileInfo().CountAll()
+	require.Nil(t, err)
+	require.Equal(t, int64(2), count)
+}

store/storetest/mocks/FileInfoStore.go

@@ -33,6 +33,27 @@ func (_m *FileInfoStore) ClearCaches() {
 	_m.Called()
 }
 
+// CountAll provides a mock function with given fields:
+func (_m *FileInfoStore) CountAll() (int64, error) {
+	ret := _m.Called()
+
+	var r0 int64
+	if rf, ok := ret.Get(0).(func() int64); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(int64)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func() error); ok {
+		r1 = rf()
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // DeleteForPost provides a mock function with given fields: postID
 func (_m *FileInfoStore) DeleteForPost(postID string) (string, error) {
 	ret := _m.Called(postID)
@@ -123,6 +144,29 @@ func (_m *FileInfoStore) GetByPath(path string) (*model.FileInfo, error) {
 	return r0, r1
 }
 
+// GetFilesBatchForIndexing provides a mock function with given fields: startTime, endTime, limit
+func (_m *FileInfoStore) GetFilesBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.FileForIndexing, error) {
+	ret := _m.Called(startTime, endTime, limit)
+
+	var r0 []*model.FileForIndexing
+	if rf, ok := ret.Get(0).(func(int64, int64, int) []*model.FileForIndexing); ok {
+		r0 = rf(startTime, endTime, limit)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*model.FileForIndexing)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(int64, int64, int) error); ok {
+		r1 = rf(startTime, endTime, limit)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // GetForPost provides a mock function with given fields: postID, readFromMaster, includeDeleted, allowFromCache
 func (_m *FileInfoStore) GetForPost(postID string, readFromMaster bool, includeDeleted bool, allowFromCache bool) ([]*model.FileInfo, error) {
 	ret := _m.Called(postID, readFromMaster, includeDeleted, allowFromCache)

store/timerlayer/timerlayer.go

@@ -2702,6 +2702,22 @@ func (s *TimerLayerFileInfoStore) ClearCaches() {
 	}
 }
 
+func (s *TimerLayerFileInfoStore) CountAll() (int64, error) {
+	start := timemodule.Now()
+
+	result, err := s.FileInfoStore.CountAll()
+
+	elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if err == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("FileInfoStore.CountAll", success, elapsed)
+	}
+	return result, err
+}
+
 func (s *TimerLayerFileInfoStore) DeleteForPost(postID string) (string, error) {
 	start := timemodule.Now()
 
@@ -2766,6 +2782,22 @@ func (s *TimerLayerFileInfoStore) GetByPath(path string) (*model.FileInfo, error
 	return result, err
 }
 
+func (s *TimerLayerFileInfoStore) GetFilesBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.FileForIndexing, error) {
+	start := timemodule.Now()
+
+	result, err := s.FileInfoStore.GetFilesBatchForIndexing(startTime, endTime, limit)
+
+	elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if err == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("FileInfoStore.GetFilesBatchForIndexing", success, elapsed)
+	}
+	return result, err
+}
+
 func (s *TimerLayerFileInfoStore) GetForPost(postID string, readFromMaster bool, includeDeleted bool, allowFromCache bool) ([]*model.FileInfo, error) {
 	start := timemodule.Now()