[MM-28423] Implement ImportDelete job (#16588)

* Implement ImportDelete job

* Add missing translation

* Improve logging

* Avoid deleting the file in case of errors
Этот коммит содержится в:
Claudio Costa
2021-01-25 10:40:30 +01:00
коммит произвёл GitHub
родитель b932e0fb25
Коммит 200a56fa5a
22 изменённых файлов: 401 добавлений и 4 удалений

Просмотреть файл

@@ -119,6 +119,9 @@ func (a *App) initJobs() {
if jobsImportProcessInterface != nil { if jobsImportProcessInterface != nil {
a.srv.Jobs.ImportProcess = jobsImportProcessInterface(a) a.srv.Jobs.ImportProcess = jobsImportProcessInterface(a)
} }
if jobsImportDeleteInterface != nil {
a.srv.Jobs.ImportDelete = jobsImportDeleteInterface(a)
}
if jobsActiveUsersInterface != nil { if jobsActiveUsersInterface != nil {
a.srv.Jobs.ActiveUsers = jobsActiveUsersInterface(a) a.srv.Jobs.ActiveUsers = jobsActiveUsersInterface(a)

Просмотреть файл

@@ -498,6 +498,7 @@ type AppIface interface {
FetchSamlMetadataFromIdp(url string) ([]byte, *model.AppError) FetchSamlMetadataFromIdp(url string) ([]byte, *model.AppError)
FileBackend() (filesstore.FileBackend, *model.AppError) FileBackend() (filesstore.FileBackend, *model.AppError)
FileExists(path string) (bool, *model.AppError) FileExists(path string) (bool, *model.AppError)
FileModTime(path string) (time.Time, *model.AppError)
FileSize(path string) (int64, *model.AppError) FileSize(path string) (int64, *model.AppError)
FillInChannelProps(channel *model.Channel) *model.AppError FillInChannelProps(channel *model.Channel) *model.AppError
FillInChannelsProps(channelList *model.ChannelList) *model.AppError FillInChannelsProps(channelList *model.ChannelList) *model.AppError

Просмотреть файл

@@ -114,6 +114,12 @@ func RegisterJobsImportProcessInterface(f func(*App) tjobs.ImportProcessInterfac
jobsImportProcessInterface = f jobsImportProcessInterface = f
} }
var jobsImportDeleteInterface func(*App) tjobs.ImportDeleteInterface
func RegisterJobsImportDeleteInterface(f func(*App) tjobs.ImportDeleteInterface) {
jobsImportDeleteInterface = f
}
var productNoticesJobInterface func(*App) tjobs.ProductNoticesJobInterface var productNoticesJobInterface func(*App) tjobs.ProductNoticesJobInterface
func RegisterProductNoticesJobInterface(f func(*App) tjobs.ProductNoticesJobInterface) { func RegisterProductNoticesJobInterface(f func(*App) tjobs.ProductNoticesJobInterface) {

Просмотреть файл

@@ -158,6 +158,19 @@ func (a *App) FileSize(path string) (int64, *model.AppError) {
return size, nil return size, nil
} }
func (a *App) FileModTime(path string) (time.Time, *model.AppError) {
backend, err := a.FileBackend()
if err != nil {
return time.Time{}, err
}
modTime, nErr := backend.FileModTime(path)
if nErr != nil {
return time.Time{}, model.NewAppError("FileModTime", "api.file.file_mod_time.app_error", nil, nErr.Error(), http.StatusInternalServerError)
}
return modTime, nil
}
func (a *App) MoveFile(oldPath, newPath string) *model.AppError { func (a *App) MoveFile(oldPath, newPath string) *model.AppError {
backend, err := a.FileBackend() backend, err := a.FileBackend()
if err != nil { if err != nil {

Просмотреть файл

@@ -277,7 +277,7 @@ func (a *App) ListImports() ([]string, *model.AppError) {
results := make([]string, 0, len(imports)) results := make([]string, 0, len(imports))
for i := 0; i < len(imports); i++ { for i := 0; i < len(imports); i++ {
filename := filepath.Base(imports[i]) filename := filepath.Base(imports[i])
if !strings.HasSuffix(filename, incompleteUploadSuffix) { if !strings.HasSuffix(filename, IncompleteUploadSuffix) {
results = append(results, filename) results = append(results, filename)
} }
} }

Просмотреть файл

@@ -3689,6 +3689,28 @@ func (a *OpenTracingAppLayer) FileExists(path string) (bool, *model.AppError) {
return resultVar0, resultVar1 return resultVar0, resultVar1
} }
func (a *OpenTracingAppLayer) FileModTime(path string) (time.Time, *model.AppError) {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.FileModTime")
a.ctx = newCtx
a.app.Srv().Store.SetContext(newCtx)
defer func() {
a.app.Srv().Store.SetContext(origCtx)
a.ctx = origCtx
}()
defer span.Finish()
resultVar0, resultVar1 := a.app.FileModTime(path)
if resultVar1 != nil {
span.LogFields(spanlog.Error(resultVar1))
ext.Error.Set(span, true)
}
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) FileReader(path string) (filesstore.ReadCloseSeeker, *model.AppError) { func (a *OpenTracingAppLayer) FileReader(path string) (filesstore.ReadCloseSeeker, *model.AppError) {
origCtx := a.ctx origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.FileReader") span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.FileReader")

Просмотреть файл

@@ -19,7 +19,7 @@ import (
) )
const minFirstPartSize = 5 * 1024 * 1024 // 5MB const minFirstPartSize = 5 * 1024 * 1024 // 5MB
const incompleteUploadSuffix = ".tmp" const IncompleteUploadSuffix = ".tmp"
func (a *App) runPluginsHook(info *model.FileInfo, file io.Reader) *model.AppError { func (a *App) runPluginsHook(info *model.FileInfo, file io.Reader) *model.AppError {
pluginsEnvironment := a.GetPluginsEnvironment() pluginsEnvironment := a.GetPluginsEnvironment()
@@ -111,7 +111,7 @@ func (a *App) CreateUploadSession(us *model.UploadSession) (*model.UploadSession
if us.Type == model.UploadTypeAttachment { if us.Type == model.UploadTypeAttachment {
us.Path = now.Format("20060102") + "/teams/noteam/channels/" + us.ChannelId + "/users/" + us.UserId + "/" + us.Id + "/" + filepath.Base(us.Filename) us.Path = now.Format("20060102") + "/teams/noteam/channels/" + us.ChannelId + "/users/" + us.UserId + "/" + us.Id + "/" + filepath.Base(us.Filename)
} else if us.Type == model.UploadTypeImport { } else if us.Type == model.UploadTypeImport {
us.Path = *a.Config().ImportSettings.Directory + "/" + us.Id + "_" + filepath.Base(us.Filename) us.Path = filepath.Clean(*a.Config().ImportSettings.Directory) + "/" + us.Id + "_" + filepath.Base(us.Filename)
} }
if err := us.IsValid(); err != nil { if err := us.IsValid(); err != nil {
return nil, err return nil, err
@@ -194,7 +194,7 @@ func (a *App) UploadData(us *model.UploadSession, rd io.Reader) (*model.FileInfo
uploadPath := us.Path uploadPath := us.Path
if us.Type == model.UploadTypeImport { if us.Type == model.UploadTypeImport {
uploadPath += incompleteUploadSuffix uploadPath += IncompleteUploadSuffix
} }
// make sure it's not possible to upload more data than what is expected. // make sure it's not possible to upload more data than what is expected.

Просмотреть файл

@@ -1336,6 +1336,10 @@
"id": "api.file.file_exists.app_error", "id": "api.file.file_exists.app_error",
"translation": "Unable to check if the file exists." "translation": "Unable to check if the file exists."
}, },
{
"id": "api.file.file_mod_time.app_error",
"translation": "Unable to get last modification time for file."
},
{ {
"id": "api.file.file_reader.app_error", "id": "api.file.file_reader.app_error",
"translation": "Unable to get a file reader." "translation": "Unable to get a file reader."

Просмотреть файл

@@ -24,4 +24,7 @@ import (
// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty. // This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
_ "github.com/mattermost/mattermost-server/v5/jobs/import_process" _ "github.com/mattermost/mattermost-server/v5/jobs/import_process"
// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
_ "github.com/mattermost/mattermost-server/v5/jobs/import_delete"
) )

51
jobs/import_delete/scheduler.go Обычный файл
Просмотреть файл

@@ -0,0 +1,51 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package import_delete
import (
"time"
"github.com/mattermost/mattermost-server/v5/app"
"github.com/mattermost/mattermost-server/v5/model"
)
const (
jobName = "ImportDelete"
schedFrequency = 24 * time.Hour
)
type Scheduler struct {
app *app.App
}
func (i *ImportDeleteInterfaceImpl) MakeScheduler() model.Scheduler {
return &Scheduler{i.app}
}
func (scheduler *Scheduler) Name() string {
return jobName + "Scheduler"
}
func (scheduler *Scheduler) JobType() string {
return model.JOB_TYPE_IMPORT_DELETE
}
func (scheduler *Scheduler) Enabled(cfg *model.Config) bool {
return *cfg.ImportSettings.Directory != "" && *cfg.ImportSettings.RetentionDays > 0
}
func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
nextTime := time.Now().Add(schedFrequency)
return &nextTime
}
func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) {
data := map[string]string{}
job, err := scheduler.app.Srv().Jobs.CreateJob(model.JOB_TYPE_IMPORT_DELETE, data)
if err != nil {
return nil, err
}
return job, nil
}

172
jobs/import_delete/worker.go Обычный файл
Просмотреть файл

@@ -0,0 +1,172 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package import_delete
import (
"errors"
"path/filepath"
"time"
"github.com/mattermost/mattermost-server/v5/app"
"github.com/mattermost/mattermost-server/v5/jobs"
tjobs "github.com/mattermost/mattermost-server/v5/jobs/interfaces"
"github.com/mattermost/mattermost-server/v5/mlog"
"github.com/mattermost/mattermost-server/v5/model"
"github.com/mattermost/mattermost-server/v5/store"
)
func init() {
app.RegisterJobsImportDeleteInterface(func(a *app.App) tjobs.ImportDeleteInterface {
return &ImportDeleteInterfaceImpl{a}
})
}
type ImportDeleteInterfaceImpl struct {
app *app.App
}
type ImportDeleteWorker struct {
name string
stopChan chan struct{}
stoppedChan chan struct{}
jobsChan chan model.Job
jobServer *jobs.JobServer
app *app.App
}
func (i *ImportDeleteInterfaceImpl) MakeWorker() model.Worker {
return &ImportDeleteWorker{
name: "ImportDelete",
stopChan: make(chan struct{}),
stoppedChan: make(chan struct{}),
jobsChan: make(chan model.Job),
jobServer: i.app.Srv().Jobs,
app: i.app,
}
}
func (w *ImportDeleteWorker) JobChannel() chan<- model.Job {
return w.jobsChan
}
func (w *ImportDeleteWorker) Run() {
mlog.Debug("Worker started", mlog.String("worker", w.name))
defer func() {
mlog.Debug("Worker finished", mlog.String("worker", w.name))
close(w.stoppedChan)
}()
for {
select {
case <-w.stopChan:
mlog.Debug("Worker received stop signal", mlog.String("worker", w.name))
return
case job := <-w.jobsChan:
mlog.Debug("Worker received a new candidate job.", mlog.String("worker", w.name))
w.doJob(&job)
}
}
}
func (w *ImportDeleteWorker) Stop() {
mlog.Debug("Worker stopping", mlog.String("worker", w.name))
close(w.stopChan)
<-w.stoppedChan
}
func (w *ImportDeleteWorker) doJob(job *model.Job) {
if claimed, err := w.jobServer.ClaimJob(job); err != nil {
mlog.Warn("Worker experienced an error while trying to claim job",
mlog.String("worker", w.name),
mlog.String("job_id", job.Id),
mlog.String("error", err.Error()))
return
} else if !claimed {
return
}
importPath := *w.app.Config().ImportSettings.Directory
retentionTime := time.Duration(*w.app.Config().ImportSettings.RetentionDays) * 24 * time.Hour
imports, appErr := w.app.ListDirectory(importPath)
if appErr != nil {
w.setJobError(job, appErr)
return
}
var hasErrs bool
for i := range imports {
filename := filepath.Base(imports[i])
modTime, appErr := w.app.FileModTime(filepath.Join(importPath, filename))
if appErr != nil {
mlog.Debug("Worker: Failed to get file modification time",
mlog.Err(appErr), mlog.String("import", imports[i]))
hasErrs = true
continue
}
if time.Now().After(modTime.Add(retentionTime)) {
// expected format if uploaded through the API is
// ${uploadID}_${filename}${app.IncompleteUploadSuffix}
minLen := 26 + 1 + len(app.IncompleteUploadSuffix)
// check if it's an incomplete upload and attempt to delete its session.
if len(filename) > minLen && filepath.Ext(filename) == app.IncompleteUploadSuffix {
uploadID := filename[:26]
if storeErr := w.app.Srv().Store.UploadSession().Delete(uploadID); storeErr != nil {
mlog.Debug("Worker: Failed to delete UploadSession",
mlog.Err(storeErr), mlog.String("upload_id", uploadID))
hasErrs = true
continue
}
} else {
// check if fileinfo exists and if so delete it.
filePath := filepath.Join(imports[i])
info, storeErr := w.app.Srv().Store.FileInfo().GetByPath(filePath)
var nfErr *store.ErrNotFound
if storeErr != nil && !errors.As(storeErr, &nfErr) {
mlog.Debug("Worker: Failed to get FileInfo",
mlog.Err(storeErr), mlog.String("path", filePath))
hasErrs = true
continue
} else if storeErr == nil {
if storeErr = w.app.Srv().Store.FileInfo().PermanentDelete(info.Id); storeErr != nil {
mlog.Debug("Worker: Failed to delete FileInfo",
mlog.Err(storeErr), mlog.String("file_id", info.Id))
hasErrs = true
continue
}
}
}
// remove file data from storage.
if appErr := w.app.RemoveFile(imports[i]); appErr != nil {
mlog.Debug("Worker: Failed to remove file",
mlog.Err(appErr), mlog.String("import", imports[i]))
hasErrs = true
continue
}
}
}
if hasErrs {
mlog.Warn("Worker: errors occurred")
}
mlog.Info("Worker: Job is complete", mlog.String("worker", w.name), mlog.String("job_id", job.Id))
w.setJobSuccess(job)
}
func (w *ImportDeleteWorker) setJobSuccess(job *model.Job) {
if err := w.app.Srv().Jobs.SetJobSuccess(job); err != nil {
mlog.Error("Worker: Failed to set success for job", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
w.setJobError(job, err)
}
}
func (w *ImportDeleteWorker) setJobError(job *model.Job, appError *model.AppError) {
if err := w.app.Srv().Jobs.SetJobError(job, appError); err != nil {
mlog.Error("Worker: Failed to set job error", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
}
}

11
jobs/interfaces/import_delete_interface.go Обычный файл
Просмотреть файл

@@ -0,0 +1,11 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package interfaces
import "github.com/mattermost/mattermost-server/v5/model"
type ImportDeleteInterface interface {
MakeWorker() model.Worker
MakeScheduler() model.Scheduler
}

Просмотреть файл

@@ -156,6 +156,13 @@ func (watcher *Watcher) PollAndNotify() {
default: default:
} }
} }
} else if job.Type == model.JOB_TYPE_IMPORT_DELETE {
if watcher.workers.ImportDelete != nil {
select {
case watcher.workers.ImportDelete.JobChannel() <- *job:
default:
}
}
} else if job.Type == model.JOB_TYPE_CLOUD { } else if job.Type == model.JOB_TYPE_CLOUD {
if watcher.workers.Cloud != nil { if watcher.workers.Cloud != nil {
select { select {

Просмотреть файл

@@ -69,6 +69,7 @@ func (srv *JobServer) InitSchedulers() *Schedulers {
if activeUsersInterface := srv.ActiveUsers; activeUsersInterface != nil { if activeUsersInterface := srv.ActiveUsers; activeUsersInterface != nil {
schedulers.schedulers = append(schedulers.schedulers, activeUsersInterface.MakeScheduler()) schedulers.schedulers = append(schedulers.schedulers, activeUsersInterface.MakeScheduler())
} }
if productNoticesInterface := srv.ProductNotices; productNoticesInterface != nil { if productNoticesInterface := srv.ProductNotices; productNoticesInterface != nil {
schedulers.schedulers = append(schedulers.schedulers, productNoticesInterface.MakeScheduler()) schedulers.schedulers = append(schedulers.schedulers, productNoticesInterface.MakeScheduler())
} }
@@ -77,6 +78,10 @@ func (srv *JobServer) InitSchedulers() *Schedulers {
schedulers.schedulers = append(schedulers.schedulers, cloudInterface.MakeScheduler()) schedulers.schedulers = append(schedulers.schedulers, cloudInterface.MakeScheduler())
} }
if importDeleteInterface := srv.ImportDelete; importDeleteInterface != nil {
schedulers.schedulers = append(schedulers.schedulers, importDeleteInterface.MakeScheduler())
}
schedulers.nextRunTimes = make([]*time.Time, len(schedulers.schedulers)) schedulers.nextRunTimes = make([]*time.Time, len(schedulers.schedulers))
return schedulers return schedulers
} }

Просмотреть файл

@@ -31,6 +31,7 @@ type JobServer struct {
ProductNotices tjobs.ProductNoticesJobInterface ProductNotices tjobs.ProductNoticesJobInterface
ActiveUsers tjobs.ActiveUsersJobInterface ActiveUsers tjobs.ActiveUsersJobInterface
ImportProcess tjobs.ImportProcessInterface ImportProcess tjobs.ImportProcessInterface
ImportDelete tjobs.ImportDeleteInterface
Cloud ejobs.CloudJobInterface Cloud ejobs.CloudJobInterface
} }

Просмотреть файл

@@ -28,6 +28,7 @@ type Workers struct {
ProductNotices model.Worker ProductNotices model.Worker
ActiveUsers model.Worker ActiveUsers model.Worker
ImportProcess model.Worker ImportProcess model.Worker
ImportDelete model.Worker
Cloud model.Worker Cloud model.Worker
listenerId string listenerId string
@@ -87,6 +88,10 @@ func (srv *JobServer) InitWorkers() *Workers {
workers.ImportProcess = importProcessInterface.MakeWorker() workers.ImportProcess = importProcessInterface.MakeWorker()
} }
if importDeleteInterface := srv.ImportDelete; importDeleteInterface != nil {
workers.ImportDelete = importDeleteInterface.MakeWorker()
}
if cloudInterface := srv.Cloud; cloudInterface != nil { if cloudInterface := srv.Cloud; cloudInterface != nil {
workers.Cloud = cloudInterface.MakeWorker() workers.Cloud = cloudInterface.MakeWorker()
} }
@@ -146,6 +151,10 @@ func (workers *Workers) Start() *Workers {
go workers.ImportProcess.Run() go workers.ImportProcess.Run()
} }
if workers.ImportDelete != nil {
go workers.ImportDelete.Run()
}
if workers.Cloud != nil { if workers.Cloud != nil {
go workers.Cloud.Run() go workers.Cloud.Run()
} }
@@ -263,6 +272,10 @@ func (workers *Workers) Stop() *Workers {
workers.ImportProcess.Stop() workers.ImportProcess.Stop()
} }
if workers.ImportDelete != nil {
workers.ImportDelete.Stop()
}
if workers.Cloud != nil { if workers.Cloud != nil {
workers.Cloud.Stop() workers.Cloud.Stop()
} }

Просмотреть файл

@@ -23,6 +23,7 @@ const (
JOB_TYPE_PRODUCT_NOTICES = "product_notices" JOB_TYPE_PRODUCT_NOTICES = "product_notices"
JOB_TYPE_ACTIVE_USERS = "active_users" JOB_TYPE_ACTIVE_USERS = "active_users"
JOB_TYPE_IMPORT_PROCESS = "import_process" JOB_TYPE_IMPORT_PROCESS = "import_process"
JOB_TYPE_IMPORT_DELETE = "import_delete"
JOB_TYPE_CLOUD = "cloud" JOB_TYPE_CLOUD = "cloud"
JOB_STATUS_PENDING = "pending" JOB_STATUS_PENDING = "pending"
@@ -68,6 +69,7 @@ func (j *Job) IsValid() *AppError {
case JOB_TYPE_EXPIRY_NOTIFY: case JOB_TYPE_EXPIRY_NOTIFY:
case JOB_TYPE_ACTIVE_USERS: case JOB_TYPE_ACTIVE_USERS:
case JOB_TYPE_IMPORT_PROCESS: case JOB_TYPE_IMPORT_PROCESS:
case JOB_TYPE_IMPORT_DELETE:
case JOB_TYPE_CLOUD: case JOB_TYPE_CLOUD:
default: default:
return NewAppError("Job.IsValid", "model.job.is_valid.type.app_error", nil, "id="+j.Id, http.StatusBadRequest) return NewAppError("Job.IsValid", "model.job.is_valid.type.app_error", nil, "id="+j.Id, http.StatusBadRequest)

Просмотреть файл

@@ -5,6 +5,7 @@ package filesstore
import ( import (
"io" "io"
"time"
"github.com/pkg/errors" "github.com/pkg/errors"
@@ -28,6 +29,7 @@ type FileBackend interface {
WriteFile(fr io.Reader, path string) (int64, error) WriteFile(fr io.Reader, path string) (int64, error)
AppendFile(fr io.Reader, path string) (int64, error) AppendFile(fr io.Reader, path string) (int64, error)
RemoveFile(path string) error RemoveFile(path string) error
FileModTime(path string) (time.Time, error)
ListDirectory(path string) ([]string, error) ListDirectory(path string) ([]string, error)
RemoveDirectory(path string) error RemoveDirectory(path string) error

Просмотреть файл

@@ -10,6 +10,7 @@ import (
"math/rand" "math/rand"
"os" "os"
"testing" "testing"
"time"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
@@ -388,6 +389,42 @@ func (s *FileBackendTestSuite) TestFileSize() {
}) })
} }
func (s *FileBackendTestSuite) TestFileModTime() {
s.Run("nonexistent file", func() {
modTime, err := s.backend.FileModTime("tests/nonexistentfile")
s.NotNil(err)
s.Empty(modTime)
})
s.Run("valid file", func() {
path := "tests/" + model.NewId()
data := []byte("some data")
written, err := s.backend.WriteFile(bytes.NewReader(data), path)
s.Nil(err)
s.EqualValues(len(data), written)
defer s.backend.RemoveFile(path)
modTime, err := s.backend.FileModTime(path)
s.Nil(err)
s.NotEmpty(modTime)
// We wait 1 second so that the times will differ enough to be testable.
time.Sleep(1 * time.Second)
path2 := "tests/" + model.NewId()
written, err = s.backend.WriteFile(bytes.NewReader(data), path2)
s.Nil(err)
s.EqualValues(len(data), written)
defer s.backend.RemoveFile(path2)
modTime2, err := s.backend.FileModTime(path2)
s.Nil(err)
s.NotEmpty(modTime2)
s.True(modTime2.After(modTime))
})
}
func BenchmarkS3WriteFile(b *testing.B) { func BenchmarkS3WriteFile(b *testing.B) {
utils.TranslationsPreInit() utils.TranslationsPreInit()

Просмотреть файл

@@ -9,6 +9,7 @@ import (
"io/ioutil" "io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"time"
"github.com/pkg/errors" "github.com/pkg/errors"
@@ -71,6 +72,14 @@ func (b *LocalFileBackend) FileSize(path string) (int64, error) {
return info.Size(), nil return info.Size(), nil
} }
func (b *LocalFileBackend) FileModTime(path string) (time.Time, error) {
info, err := os.Stat(filepath.Join(b.directory, path))
if err != nil {
return time.Time{}, errors.Wrapf(err, "unable to get modification time for file %s", path)
}
return info.ModTime(), nil
}
func (b *LocalFileBackend) CopyFile(oldPath, newPath string) error { func (b *LocalFileBackend) CopyFile(oldPath, newPath string) error {
if err := utils.CopyFile(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil { if err := utils.CopyFile(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil {
return errors.Wrapf(err, "unable to copy file from %s to %s", oldPath, newPath) return errors.Wrapf(err, "unable to copy file from %s to %s", oldPath, newPath)

Просмотреть файл

@@ -10,6 +10,8 @@ import (
filesstore "github.com/mattermost/mattermost-server/v5/services/filesstore" filesstore "github.com/mattermost/mattermost-server/v5/services/filesstore"
mock "github.com/stretchr/testify/mock" mock "github.com/stretchr/testify/mock"
time "time"
) )
// FileBackend is an autogenerated mock type for the FileBackend type // FileBackend is an autogenerated mock type for the FileBackend type
@@ -73,6 +75,27 @@ func (_m *FileBackend) FileExists(path string) (bool, error) {
return r0, r1 return r0, r1
} }
// FileModTime provides a mock function with given fields: path
func (_m *FileBackend) FileModTime(path string) (time.Time, error) {
ret := _m.Called(path)
var r0 time.Time
if rf, ok := ret.Get(0).(func(string) time.Time); ok {
r0 = rf(path)
} else {
r0 = ret.Get(0).(time.Time)
}
var r1 error
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(path)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// FileSize provides a mock function with given fields: path // FileSize provides a mock function with given fields: path
func (_m *FileBackend) FileSize(path string) (int64, error) { func (_m *FileBackend) FileSize(path string) (int64, error) {
ret := _m.Called(path) ret := _m.Called(path)

Просмотреть файл

@@ -10,6 +10,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"time"
s3 "github.com/minio/minio-go/v7" s3 "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/credentials"
@@ -205,6 +206,17 @@ func (b *S3FileBackend) FileSize(path string) (int64, error) {
return info.Size, nil return info.Size, nil
} }
func (b *S3FileBackend) FileModTime(path string) (time.Time, error) {
path = filepath.Join(b.pathPrefix, path)
info, err := b.client.StatObject(context.Background(), b.bucket, path, s3.StatObjectOptions{})
if err != nil {
return time.Time{}, errors.Wrapf(err, "unable to get modification time for file %s", path)
}
return info.LastModified, nil
}
func (b *S3FileBackend) CopyFile(oldPath, newPath string) error { func (b *S3FileBackend) CopyFile(oldPath, newPath string) error {
oldPath = filepath.Join(b.pathPrefix, oldPath) oldPath = filepath.Join(b.pathPrefix, oldPath)
newPath = filepath.Join(b.pathPrefix, newPath) newPath = filepath.Join(b.pathPrefix, newPath)