[MM-28423] Implement ImportDelete job (#16588)

* Implement ImportDelete job

* Add missing translation

* Improve logging

* Avoid deleting the file in case of errors

services/filesstore/filesstore.go

@@ -5,6 +5,7 @@ package filesstore
 
 import (
 	"io"
+	"time"
 
 	"github.com/pkg/errors"
 
@@ -28,6 +29,7 @@ type FileBackend interface {
 	WriteFile(fr io.Reader, path string) (int64, error)
 	AppendFile(fr io.Reader, path string) (int64, error)
 	RemoveFile(path string) error
+	FileModTime(path string) (time.Time, error)
 
 	ListDirectory(path string) ([]string, error)
 	RemoveDirectory(path string) error

services/filesstore/filesstore_test.go

@@ -10,6 +10,7 @@ import (
 	"math/rand"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
@@ -388,6 +389,42 @@ func (s *FileBackendTestSuite) TestFileSize() {
 	})
 }
 
+func (s *FileBackendTestSuite) TestFileModTime() {
+	s.Run("nonexistent file", func() {
+		modTime, err := s.backend.FileModTime("tests/nonexistentfile")
+		s.NotNil(err)
+		s.Empty(modTime)
+	})
+
+	s.Run("valid file", func() {
+		path := "tests/" + model.NewId()
+		data := []byte("some data")
+
+		written, err := s.backend.WriteFile(bytes.NewReader(data), path)
+		s.Nil(err)
+		s.EqualValues(len(data), written)
+		defer s.backend.RemoveFile(path)
+
+		modTime, err := s.backend.FileModTime(path)
+		s.Nil(err)
+		s.NotEmpty(modTime)
+
+		// We wait 1 second so that the times will differ enough to be testable.
+		time.Sleep(1 * time.Second)
+
+		path2 := "tests/" + model.NewId()
+		written, err = s.backend.WriteFile(bytes.NewReader(data), path2)
+		s.Nil(err)
+		s.EqualValues(len(data), written)
+		defer s.backend.RemoveFile(path2)
+
+		modTime2, err := s.backend.FileModTime(path2)
+		s.Nil(err)
+		s.NotEmpty(modTime2)
+		s.True(modTime2.After(modTime))
+	})
+}
+
 func BenchmarkS3WriteFile(b *testing.B) {
 	utils.TranslationsPreInit()
 

services/filesstore/localstore.go

@@ -9,6 +9,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/pkg/errors"
 
@@ -71,6 +72,14 @@ func (b *LocalFileBackend) FileSize(path string) (int64, error) {
 	return info.Size(), nil
 }
 
+func (b *LocalFileBackend) FileModTime(path string) (time.Time, error) {
+	info, err := os.Stat(filepath.Join(b.directory, path))
+	if err != nil {
+		return time.Time{}, errors.Wrapf(err, "unable to get modification time for file %s", path)
+	}
+	return info.ModTime(), nil
+}
+
 func (b *LocalFileBackend) CopyFile(oldPath, newPath string) error {
 	if err := utils.CopyFile(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil {
 		return errors.Wrapf(err, "unable to copy file from %s to %s", oldPath, newPath)

services/filesstore/mocks/FileBackend.go

@@ -10,6 +10,8 @@ import (
 	filesstore "github.com/mattermost/mattermost-server/v5/services/filesstore"
 
 	mock "github.com/stretchr/testify/mock"
+
+	time "time"
 )
 
 // FileBackend is an autogenerated mock type for the FileBackend type
@@ -73,6 +75,27 @@ func (_m *FileBackend) FileExists(path string) (bool, error) {
 	return r0, r1
 }
 
+// FileModTime provides a mock function with given fields: path
+func (_m *FileBackend) FileModTime(path string) (time.Time, error) {
+	ret := _m.Called(path)
+
+	var r0 time.Time
+	if rf, ok := ret.Get(0).(func(string) time.Time); ok {
+		r0 = rf(path)
+	} else {
+		r0 = ret.Get(0).(time.Time)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string) error); ok {
+		r1 = rf(path)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // FileSize provides a mock function with given fields: path
 func (_m *FileBackend) FileSize(path string) (int64, error) {
 	ret := _m.Called(path)

services/filesstore/s3store.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	s3 "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
@@ -205,6 +206,17 @@ func (b *S3FileBackend) FileSize(path string) (int64, error) {
 	return info.Size, nil
 }
 
+func (b *S3FileBackend) FileModTime(path string) (time.Time, error) {
+	path = filepath.Join(b.pathPrefix, path)
+
+	info, err := b.client.StatObject(context.Background(), b.bucket, path, s3.StatObjectOptions{})
+	if err != nil {
+		return time.Time{}, errors.Wrapf(err, "unable to get modification time for file %s", path)
+	}
+
+	return info.LastModified, nil
+}
+
 func (b *S3FileBackend) CopyFile(oldPath, newPath string) error {
 	oldPath = filepath.Join(b.pathPrefix, oldPath)
 	newPath = filepath.Join(b.pathPrefix, newPath)