MM-53747: Create job to encode older image paths (#24073)

Bifrost now encodes all image paths. Due to this
one-way translation, we need to encode all the older
image paths as well.

After this is done, we can remove the double-lookup.

https://mattermost.atlassian.net/browse/MM-53747

```release-note
NONE
```

Co-authored-by: Mattermost Build <build@mattermost.com>

server/channels/store/opentracinglayer/opentracinglayer.go

@@ -3608,7 +3608,7 @@ func (s *OpenTracingLayerFileInfoStore) GetByPath(path string) (*model.FileInfo,
 	return result, err
 }
 
-func (s *OpenTracingLayerFileInfoStore) GetFilesBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.FileForIndexing, error) {
+func (s *OpenTracingLayerFileInfoStore) GetFilesBatchForIndexing(startTime int64, startFileID string, includeDeleted bool, limit int) ([]*model.FileForIndexing, error) {
 	origCtx := s.Root.Store.Context()
 	span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.GetFilesBatchForIndexing")
 	s.Root.Store.SetContext(newCtx)
@@ -3617,7 +3617,7 @@ func (s *OpenTracingLayerFileInfoStore) GetFilesBatchForIndexing(startTime int64
 	}()
 
 	defer span.Finish()
-	result, err := s.FileInfoStore.GetFilesBatchForIndexing(startTime, startFileID, limit)
+	result, err := s.FileInfoStore.GetFilesBatchForIndexing(startTime, startFileID, includeDeleted, limit)
 	if err != nil {
 		span.LogFields(spanlog.Error(err))
 		ext.Error.Set(span, true)

server/channels/store/retrylayer/retrylayer.go

@@ -4034,11 +4034,11 @@ func (s *RetryLayerFileInfoStore) GetByPath(path string) (*model.FileInfo, error
 
 }
 
-func (s *RetryLayerFileInfoStore) GetFilesBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.FileForIndexing, error) {
+func (s *RetryLayerFileInfoStore) GetFilesBatchForIndexing(startTime int64, startFileID string, includeDeleted bool, limit int) ([]*model.FileForIndexing, error) {
 
 	tries := 0
 	for {
-		result, err := s.FileInfoStore.GetFilesBatchForIndexing(startTime, startFileID, limit)
+		result, err := s.FileInfoStore.GetFilesBatchForIndexing(startTime, startFileID, includeDeleted, limit)
 		if err == nil {
 			return result, nil
 		}

server/channels/store/sqlstore/file_info_store.go

@@ -691,9 +691,10 @@ func (fs SqlFileInfoStore) CountAll() (int64, error) {
 	return count, nil
 }
 
-func (fs SqlFileInfoStore) GetFilesBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.FileForIndexing, error) {
+func (fs SqlFileInfoStore) GetFilesBatchForIndexing(startTime int64, startFileID string, includeDeleted bool, limit int) ([]*model.FileForIndexing, error) {
 	files := []*model.FileForIndexing{}
-	sql, args, _ := fs.getQueryBuilder().
+
+	query := fs.getQueryBuilder().
 		Select(fs.queryFields...).
 		From("FileInfo").
 		Where(sq.Or{
@@ -704,10 +705,13 @@ func (fs SqlFileInfoStore) GetFilesBatchForIndexing(startTime int64, startFileID
 			},
 		}).
 		OrderBy("FileInfo.CreateAt ASC, FileInfo.Id ASC").
-		Limit(uint64(limit)).
-		ToSql()
+		Limit(uint64(limit))
 
-	err := fs.GetSearchReplicaX().Select(&files, sql, args...)
+	if !includeDeleted {
+		query = query.Where(sq.Eq{"FileInfo.DeleteAt": 0})
+	}
+
+	err := fs.GetSearchReplicaX().SelectBuilder(&files, query)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to find Files")
 	}

server/channels/store/store.go

@@ -703,7 +703,7 @@ type FileInfoStore interface {
 	SetContent(fileID, content string) error
 	Search(paramsList []*model.SearchParams, userID, teamID string, page, perPage int) (*model.FileInfoList, error)
 	CountAll() (int64, error)
-	GetFilesBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.FileForIndexing, error)
+	GetFilesBatchForIndexing(startTime int64, startFileID string, includeDeleted bool, limit int) ([]*model.FileForIndexing, error)
 	ClearCaches()
 	GetStorageUsage(allowFromCache, includeDeleted bool) (int64, error)
 	// GetUptoNSizeFileTime returns the CreateAt time of the last accessible file with a running-total size upto n bytes.

server/channels/store/storetest/file_info_store.go

@@ -748,21 +748,29 @@ func testFileInfoStoreGetFilesBatchForIndexing(t *testing.T, ss store.Store) {
 		ss.FileInfo().PermanentDelete(f3.Id)
 	}()
 
+	// Soft-deleting one file info
+	_, err = ss.FileInfo().DeleteForPost(f1.PostId)
+	require.NoError(t, err)
+
 	// Getting all
-	r, err := ss.FileInfo().GetFilesBatchForIndexing(f1.CreateAt-1, "", 100)
+	r, err := ss.FileInfo().GetFilesBatchForIndexing(f1.CreateAt-1, "", true, 100)
 	require.NoError(t, err)
 	require.Len(t, r, 3, "Expected 3 posts in results. Got %v", len(r))
 
-	// Testing pagination
-	r, err = ss.FileInfo().GetFilesBatchForIndexing(f1.CreateAt-1, "", 2)
+	r, err = ss.FileInfo().GetFilesBatchForIndexing(f1.CreateAt-1, "", false, 100)
 	require.NoError(t, err)
 	require.Len(t, r, 2, "Expected 2 posts in results. Got %v", len(r))
 
-	r, err = ss.FileInfo().GetFilesBatchForIndexing(r[1].CreateAt, r[1].Id, 2)
+	// Testing pagination
+	r, err = ss.FileInfo().GetFilesBatchForIndexing(f1.CreateAt-1, "", true, 2)
+	require.NoError(t, err)
+	require.Len(t, r, 2, "Expected 2 posts in results. Got %v", len(r))
+
+	r, err = ss.FileInfo().GetFilesBatchForIndexing(r[1].CreateAt, r[1].Id, true, 2)
 	require.NoError(t, err)
 	require.Len(t, r, 1, "Expected 1 post in results. Got %v", len(r))
 
-	r, err = ss.FileInfo().GetFilesBatchForIndexing(r[0].CreateAt, r[0].Id, 2)
+	r, err = ss.FileInfo().GetFilesBatchForIndexing(r[0].CreateAt, r[0].Id, true, 2)
 	require.NoError(t, err)
 	require.Len(t, r, 0, "Expected 0 posts in results. Got %v", len(r))
 }

server/channels/store/storetest/mocks/FileInfoStore.go

@@ -159,25 +159,25 @@ func (_m *FileInfoStore) GetByPath(path string) (*model.FileInfo, error) {
 	return r0, r1
 }
 
-// GetFilesBatchForIndexing provides a mock function with given fields: startTime, startFileID, limit
-func (_m *FileInfoStore) GetFilesBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.FileForIndexing, error) {
-	ret := _m.Called(startTime, startFileID, limit)
+// GetFilesBatchForIndexing provides a mock function with given fields: startTime, startFileID, includeDeleted, limit
+func (_m *FileInfoStore) GetFilesBatchForIndexing(startTime int64, startFileID string, includeDeleted bool, limit int) ([]*model.FileForIndexing, error) {
+	ret := _m.Called(startTime, startFileID, includeDeleted, limit)
 
 	var r0 []*model.FileForIndexing
 	var r1 error
-	if rf, ok := ret.Get(0).(func(int64, string, int) ([]*model.FileForIndexing, error)); ok {
-		return rf(startTime, startFileID, limit)
+	if rf, ok := ret.Get(0).(func(int64, string, bool, int) ([]*model.FileForIndexing, error)); ok {
+		return rf(startTime, startFileID, includeDeleted, limit)
 	}
-	if rf, ok := ret.Get(0).(func(int64, string, int) []*model.FileForIndexing); ok {
-		r0 = rf(startTime, startFileID, limit)
+	if rf, ok := ret.Get(0).(func(int64, string, bool, int) []*model.FileForIndexing); ok {
+		r0 = rf(startTime, startFileID, includeDeleted, limit)
 	} else {
 		if ret.Get(0) != nil {
 			r0 = ret.Get(0).([]*model.FileForIndexing)
 		}
 	}
 
-	if rf, ok := ret.Get(1).(func(int64, string, int) error); ok {
-		r1 = rf(startTime, startFileID, limit)
+	if rf, ok := ret.Get(1).(func(int64, string, bool, int) error); ok {
+		r1 = rf(startTime, startFileID, includeDeleted, limit)
 	} else {
 		r1 = ret.Error(1)
 	}

server/channels/store/timerlayer/timerlayer.go

@@ -3303,10 +3303,10 @@ func (s *TimerLayerFileInfoStore) GetByPath(path string) (*model.FileInfo, error
 	return result, err
 }
 
-func (s *TimerLayerFileInfoStore) GetFilesBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.FileForIndexing, error) {
+func (s *TimerLayerFileInfoStore) GetFilesBatchForIndexing(startTime int64, startFileID string, includeDeleted bool, limit int) ([]*model.FileForIndexing, error) {
 	start := time.Now()
 
-	result, err := s.FileInfoStore.GetFilesBatchForIndexing(startTime, startFileID, limit)
+	result, err := s.FileInfoStore.GetFilesBatchForIndexing(startTime, startFileID, includeDeleted, limit)
 
 	elapsed := float64(time.Since(start)) / float64(time.Second)
 	if s.Root.Metrics != nil {