[MM-28423] Implement ImportDelete job (#16588)
* Implement ImportDelete job * Add missing translation * Improve logging * Avoid deleting the file in case of errors
Этот коммит содержится в:
коммит произвёл
GitHub
родитель
b932e0fb25
Коммит
200a56fa5a
@@ -119,6 +119,9 @@ func (a *App) initJobs() {
|
||||
if jobsImportProcessInterface != nil {
|
||||
a.srv.Jobs.ImportProcess = jobsImportProcessInterface(a)
|
||||
}
|
||||
if jobsImportDeleteInterface != nil {
|
||||
a.srv.Jobs.ImportDelete = jobsImportDeleteInterface(a)
|
||||
}
|
||||
|
||||
if jobsActiveUsersInterface != nil {
|
||||
a.srv.Jobs.ActiveUsers = jobsActiveUsersInterface(a)
|
||||
|
||||
@@ -498,6 +498,7 @@ type AppIface interface {
|
||||
FetchSamlMetadataFromIdp(url string) ([]byte, *model.AppError)
|
||||
FileBackend() (filesstore.FileBackend, *model.AppError)
|
||||
FileExists(path string) (bool, *model.AppError)
|
||||
FileModTime(path string) (time.Time, *model.AppError)
|
||||
FileSize(path string) (int64, *model.AppError)
|
||||
FillInChannelProps(channel *model.Channel) *model.AppError
|
||||
FillInChannelsProps(channelList *model.ChannelList) *model.AppError
|
||||
|
||||
@@ -114,6 +114,12 @@ func RegisterJobsImportProcessInterface(f func(*App) tjobs.ImportProcessInterfac
|
||||
jobsImportProcessInterface = f
|
||||
}
|
||||
|
||||
var jobsImportDeleteInterface func(*App) tjobs.ImportDeleteInterface
|
||||
|
||||
func RegisterJobsImportDeleteInterface(f func(*App) tjobs.ImportDeleteInterface) {
|
||||
jobsImportDeleteInterface = f
|
||||
}
|
||||
|
||||
var productNoticesJobInterface func(*App) tjobs.ProductNoticesJobInterface
|
||||
|
||||
func RegisterProductNoticesJobInterface(f func(*App) tjobs.ProductNoticesJobInterface) {
|
||||
|
||||
13
app/file.go
13
app/file.go
@@ -158,6 +158,19 @@ func (a *App) FileSize(path string) (int64, *model.AppError) {
|
||||
return size, nil
|
||||
}
|
||||
|
||||
func (a *App) FileModTime(path string) (time.Time, *model.AppError) {
|
||||
backend, err := a.FileBackend()
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
modTime, nErr := backend.FileModTime(path)
|
||||
if nErr != nil {
|
||||
return time.Time{}, model.NewAppError("FileModTime", "api.file.file_mod_time.app_error", nil, nErr.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
return modTime, nil
|
||||
}
|
||||
|
||||
func (a *App) MoveFile(oldPath, newPath string) *model.AppError {
|
||||
backend, err := a.FileBackend()
|
||||
if err != nil {
|
||||
|
||||
@@ -277,7 +277,7 @@ func (a *App) ListImports() ([]string, *model.AppError) {
|
||||
results := make([]string, 0, len(imports))
|
||||
for i := 0; i < len(imports); i++ {
|
||||
filename := filepath.Base(imports[i])
|
||||
if !strings.HasSuffix(filename, incompleteUploadSuffix) {
|
||||
if !strings.HasSuffix(filename, IncompleteUploadSuffix) {
|
||||
results = append(results, filename)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3689,6 +3689,28 @@ func (a *OpenTracingAppLayer) FileExists(path string) (bool, *model.AppError) {
|
||||
return resultVar0, resultVar1
|
||||
}
|
||||
|
||||
func (a *OpenTracingAppLayer) FileModTime(path string) (time.Time, *model.AppError) {
|
||||
origCtx := a.ctx
|
||||
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.FileModTime")
|
||||
|
||||
a.ctx = newCtx
|
||||
a.app.Srv().Store.SetContext(newCtx)
|
||||
defer func() {
|
||||
a.app.Srv().Store.SetContext(origCtx)
|
||||
a.ctx = origCtx
|
||||
}()
|
||||
|
||||
defer span.Finish()
|
||||
resultVar0, resultVar1 := a.app.FileModTime(path)
|
||||
|
||||
if resultVar1 != nil {
|
||||
span.LogFields(spanlog.Error(resultVar1))
|
||||
ext.Error.Set(span, true)
|
||||
}
|
||||
|
||||
return resultVar0, resultVar1
|
||||
}
|
||||
|
||||
func (a *OpenTracingAppLayer) FileReader(path string) (filesstore.ReadCloseSeeker, *model.AppError) {
|
||||
origCtx := a.ctx
|
||||
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.FileReader")
|
||||
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
)
|
||||
|
||||
const minFirstPartSize = 5 * 1024 * 1024 // 5MB
|
||||
const incompleteUploadSuffix = ".tmp"
|
||||
const IncompleteUploadSuffix = ".tmp"
|
||||
|
||||
func (a *App) runPluginsHook(info *model.FileInfo, file io.Reader) *model.AppError {
|
||||
pluginsEnvironment := a.GetPluginsEnvironment()
|
||||
@@ -111,7 +111,7 @@ func (a *App) CreateUploadSession(us *model.UploadSession) (*model.UploadSession
|
||||
if us.Type == model.UploadTypeAttachment {
|
||||
us.Path = now.Format("20060102") + "/teams/noteam/channels/" + us.ChannelId + "/users/" + us.UserId + "/" + us.Id + "/" + filepath.Base(us.Filename)
|
||||
} else if us.Type == model.UploadTypeImport {
|
||||
us.Path = *a.Config().ImportSettings.Directory + "/" + us.Id + "_" + filepath.Base(us.Filename)
|
||||
us.Path = filepath.Clean(*a.Config().ImportSettings.Directory) + "/" + us.Id + "_" + filepath.Base(us.Filename)
|
||||
}
|
||||
if err := us.IsValid(); err != nil {
|
||||
return nil, err
|
||||
@@ -194,7 +194,7 @@ func (a *App) UploadData(us *model.UploadSession, rd io.Reader) (*model.FileInfo
|
||||
|
||||
uploadPath := us.Path
|
||||
if us.Type == model.UploadTypeImport {
|
||||
uploadPath += incompleteUploadSuffix
|
||||
uploadPath += IncompleteUploadSuffix
|
||||
}
|
||||
|
||||
// make sure it's not possible to upload more data than what is expected.
|
||||
|
||||
@@ -1336,6 +1336,10 @@
|
||||
"id": "api.file.file_exists.app_error",
|
||||
"translation": "Unable to check if the file exists."
|
||||
},
|
||||
{
|
||||
"id": "api.file.file_mod_time.app_error",
|
||||
"translation": "Unable to get last modification time for file."
|
||||
},
|
||||
{
|
||||
"id": "api.file.file_reader.app_error",
|
||||
"translation": "Unable to get a file reader."
|
||||
|
||||
@@ -24,4 +24,7 @@ import (
|
||||
|
||||
// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
|
||||
_ "github.com/mattermost/mattermost-server/v5/jobs/import_process"
|
||||
|
||||
// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
|
||||
_ "github.com/mattermost/mattermost-server/v5/jobs/import_delete"
|
||||
)
|
||||
|
||||
51
jobs/import_delete/scheduler.go
Обычный файл
51
jobs/import_delete/scheduler.go
Обычный файл
@@ -0,0 +1,51 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package import_delete
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/mattermost/mattermost-server/v5/app"
|
||||
"github.com/mattermost/mattermost-server/v5/model"
|
||||
)
|
||||
|
||||
const (
|
||||
jobName = "ImportDelete"
|
||||
schedFrequency = 24 * time.Hour
|
||||
)
|
||||
|
||||
type Scheduler struct {
|
||||
app *app.App
|
||||
}
|
||||
|
||||
func (i *ImportDeleteInterfaceImpl) MakeScheduler() model.Scheduler {
|
||||
return &Scheduler{i.app}
|
||||
}
|
||||
|
||||
func (scheduler *Scheduler) Name() string {
|
||||
return jobName + "Scheduler"
|
||||
}
|
||||
|
||||
func (scheduler *Scheduler) JobType() string {
|
||||
return model.JOB_TYPE_IMPORT_DELETE
|
||||
}
|
||||
|
||||
func (scheduler *Scheduler) Enabled(cfg *model.Config) bool {
|
||||
return *cfg.ImportSettings.Directory != "" && *cfg.ImportSettings.RetentionDays > 0
|
||||
}
|
||||
|
||||
func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
|
||||
nextTime := time.Now().Add(schedFrequency)
|
||||
return &nextTime
|
||||
}
|
||||
|
||||
func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) {
|
||||
data := map[string]string{}
|
||||
|
||||
job, err := scheduler.app.Srv().Jobs.CreateJob(model.JOB_TYPE_IMPORT_DELETE, data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
172
jobs/import_delete/worker.go
Обычный файл
172
jobs/import_delete/worker.go
Обычный файл
@@ -0,0 +1,172 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package import_delete
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/mattermost/mattermost-server/v5/app"
|
||||
"github.com/mattermost/mattermost-server/v5/jobs"
|
||||
tjobs "github.com/mattermost/mattermost-server/v5/jobs/interfaces"
|
||||
"github.com/mattermost/mattermost-server/v5/mlog"
|
||||
"github.com/mattermost/mattermost-server/v5/model"
|
||||
"github.com/mattermost/mattermost-server/v5/store"
|
||||
)
|
||||
|
||||
func init() {
|
||||
app.RegisterJobsImportDeleteInterface(func(a *app.App) tjobs.ImportDeleteInterface {
|
||||
return &ImportDeleteInterfaceImpl{a}
|
||||
})
|
||||
}
|
||||
|
||||
type ImportDeleteInterfaceImpl struct {
|
||||
app *app.App
|
||||
}
|
||||
|
||||
type ImportDeleteWorker struct {
|
||||
name string
|
||||
stopChan chan struct{}
|
||||
stoppedChan chan struct{}
|
||||
jobsChan chan model.Job
|
||||
jobServer *jobs.JobServer
|
||||
app *app.App
|
||||
}
|
||||
|
||||
func (i *ImportDeleteInterfaceImpl) MakeWorker() model.Worker {
|
||||
return &ImportDeleteWorker{
|
||||
name: "ImportDelete",
|
||||
stopChan: make(chan struct{}),
|
||||
stoppedChan: make(chan struct{}),
|
||||
jobsChan: make(chan model.Job),
|
||||
jobServer: i.app.Srv().Jobs,
|
||||
app: i.app,
|
||||
}
|
||||
}
|
||||
|
||||
func (w *ImportDeleteWorker) JobChannel() chan<- model.Job {
|
||||
return w.jobsChan
|
||||
}
|
||||
|
||||
func (w *ImportDeleteWorker) Run() {
|
||||
mlog.Debug("Worker started", mlog.String("worker", w.name))
|
||||
|
||||
defer func() {
|
||||
mlog.Debug("Worker finished", mlog.String("worker", w.name))
|
||||
close(w.stoppedChan)
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-w.stopChan:
|
||||
mlog.Debug("Worker received stop signal", mlog.String("worker", w.name))
|
||||
return
|
||||
case job := <-w.jobsChan:
|
||||
mlog.Debug("Worker received a new candidate job.", mlog.String("worker", w.name))
|
||||
w.doJob(&job)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *ImportDeleteWorker) Stop() {
|
||||
mlog.Debug("Worker stopping", mlog.String("worker", w.name))
|
||||
close(w.stopChan)
|
||||
<-w.stoppedChan
|
||||
}
|
||||
|
||||
func (w *ImportDeleteWorker) doJob(job *model.Job) {
|
||||
if claimed, err := w.jobServer.ClaimJob(job); err != nil {
|
||||
mlog.Warn("Worker experienced an error while trying to claim job",
|
||||
mlog.String("worker", w.name),
|
||||
mlog.String("job_id", job.Id),
|
||||
mlog.String("error", err.Error()))
|
||||
return
|
||||
} else if !claimed {
|
||||
return
|
||||
}
|
||||
|
||||
importPath := *w.app.Config().ImportSettings.Directory
|
||||
retentionTime := time.Duration(*w.app.Config().ImportSettings.RetentionDays) * 24 * time.Hour
|
||||
imports, appErr := w.app.ListDirectory(importPath)
|
||||
if appErr != nil {
|
||||
w.setJobError(job, appErr)
|
||||
return
|
||||
}
|
||||
|
||||
var hasErrs bool
|
||||
for i := range imports {
|
||||
filename := filepath.Base(imports[i])
|
||||
modTime, appErr := w.app.FileModTime(filepath.Join(importPath, filename))
|
||||
if appErr != nil {
|
||||
mlog.Debug("Worker: Failed to get file modification time",
|
||||
mlog.Err(appErr), mlog.String("import", imports[i]))
|
||||
hasErrs = true
|
||||
continue
|
||||
}
|
||||
|
||||
if time.Now().After(modTime.Add(retentionTime)) {
|
||||
// expected format if uploaded through the API is
|
||||
// ${uploadID}_${filename}${app.IncompleteUploadSuffix}
|
||||
minLen := 26 + 1 + len(app.IncompleteUploadSuffix)
|
||||
|
||||
// check if it's an incomplete upload and attempt to delete its session.
|
||||
if len(filename) > minLen && filepath.Ext(filename) == app.IncompleteUploadSuffix {
|
||||
uploadID := filename[:26]
|
||||
if storeErr := w.app.Srv().Store.UploadSession().Delete(uploadID); storeErr != nil {
|
||||
mlog.Debug("Worker: Failed to delete UploadSession",
|
||||
mlog.Err(storeErr), mlog.String("upload_id", uploadID))
|
||||
hasErrs = true
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
// check if fileinfo exists and if so delete it.
|
||||
filePath := filepath.Join(imports[i])
|
||||
info, storeErr := w.app.Srv().Store.FileInfo().GetByPath(filePath)
|
||||
var nfErr *store.ErrNotFound
|
||||
if storeErr != nil && !errors.As(storeErr, &nfErr) {
|
||||
mlog.Debug("Worker: Failed to get FileInfo",
|
||||
mlog.Err(storeErr), mlog.String("path", filePath))
|
||||
hasErrs = true
|
||||
continue
|
||||
} else if storeErr == nil {
|
||||
if storeErr = w.app.Srv().Store.FileInfo().PermanentDelete(info.Id); storeErr != nil {
|
||||
mlog.Debug("Worker: Failed to delete FileInfo",
|
||||
mlog.Err(storeErr), mlog.String("file_id", info.Id))
|
||||
hasErrs = true
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// remove file data from storage.
|
||||
if appErr := w.app.RemoveFile(imports[i]); appErr != nil {
|
||||
mlog.Debug("Worker: Failed to remove file",
|
||||
mlog.Err(appErr), mlog.String("import", imports[i]))
|
||||
hasErrs = true
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if hasErrs {
|
||||
mlog.Warn("Worker: errors occurred")
|
||||
}
|
||||
|
||||
mlog.Info("Worker: Job is complete", mlog.String("worker", w.name), mlog.String("job_id", job.Id))
|
||||
w.setJobSuccess(job)
|
||||
}
|
||||
|
||||
func (w *ImportDeleteWorker) setJobSuccess(job *model.Job) {
|
||||
if err := w.app.Srv().Jobs.SetJobSuccess(job); err != nil {
|
||||
mlog.Error("Worker: Failed to set success for job", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
|
||||
w.setJobError(job, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *ImportDeleteWorker) setJobError(job *model.Job, appError *model.AppError) {
|
||||
if err := w.app.Srv().Jobs.SetJobError(job, appError); err != nil {
|
||||
mlog.Error("Worker: Failed to set job error", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
|
||||
}
|
||||
}
|
||||
11
jobs/interfaces/import_delete_interface.go
Обычный файл
11
jobs/interfaces/import_delete_interface.go
Обычный файл
@@ -0,0 +1,11 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package interfaces
|
||||
|
||||
import "github.com/mattermost/mattermost-server/v5/model"
|
||||
|
||||
type ImportDeleteInterface interface {
|
||||
MakeWorker() model.Worker
|
||||
MakeScheduler() model.Scheduler
|
||||
}
|
||||
@@ -156,6 +156,13 @@ func (watcher *Watcher) PollAndNotify() {
|
||||
default:
|
||||
}
|
||||
}
|
||||
} else if job.Type == model.JOB_TYPE_IMPORT_DELETE {
|
||||
if watcher.workers.ImportDelete != nil {
|
||||
select {
|
||||
case watcher.workers.ImportDelete.JobChannel() <- *job:
|
||||
default:
|
||||
}
|
||||
}
|
||||
} else if job.Type == model.JOB_TYPE_CLOUD {
|
||||
if watcher.workers.Cloud != nil {
|
||||
select {
|
||||
|
||||
@@ -69,6 +69,7 @@ func (srv *JobServer) InitSchedulers() *Schedulers {
|
||||
if activeUsersInterface := srv.ActiveUsers; activeUsersInterface != nil {
|
||||
schedulers.schedulers = append(schedulers.schedulers, activeUsersInterface.MakeScheduler())
|
||||
}
|
||||
|
||||
if productNoticesInterface := srv.ProductNotices; productNoticesInterface != nil {
|
||||
schedulers.schedulers = append(schedulers.schedulers, productNoticesInterface.MakeScheduler())
|
||||
}
|
||||
@@ -77,6 +78,10 @@ func (srv *JobServer) InitSchedulers() *Schedulers {
|
||||
schedulers.schedulers = append(schedulers.schedulers, cloudInterface.MakeScheduler())
|
||||
}
|
||||
|
||||
if importDeleteInterface := srv.ImportDelete; importDeleteInterface != nil {
|
||||
schedulers.schedulers = append(schedulers.schedulers, importDeleteInterface.MakeScheduler())
|
||||
}
|
||||
|
||||
schedulers.nextRunTimes = make([]*time.Time, len(schedulers.schedulers))
|
||||
return schedulers
|
||||
}
|
||||
|
||||
@@ -31,6 +31,7 @@ type JobServer struct {
|
||||
ProductNotices tjobs.ProductNoticesJobInterface
|
||||
ActiveUsers tjobs.ActiveUsersJobInterface
|
||||
ImportProcess tjobs.ImportProcessInterface
|
||||
ImportDelete tjobs.ImportDeleteInterface
|
||||
Cloud ejobs.CloudJobInterface
|
||||
}
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ type Workers struct {
|
||||
ProductNotices model.Worker
|
||||
ActiveUsers model.Worker
|
||||
ImportProcess model.Worker
|
||||
ImportDelete model.Worker
|
||||
Cloud model.Worker
|
||||
|
||||
listenerId string
|
||||
@@ -87,6 +88,10 @@ func (srv *JobServer) InitWorkers() *Workers {
|
||||
workers.ImportProcess = importProcessInterface.MakeWorker()
|
||||
}
|
||||
|
||||
if importDeleteInterface := srv.ImportDelete; importDeleteInterface != nil {
|
||||
workers.ImportDelete = importDeleteInterface.MakeWorker()
|
||||
}
|
||||
|
||||
if cloudInterface := srv.Cloud; cloudInterface != nil {
|
||||
workers.Cloud = cloudInterface.MakeWorker()
|
||||
}
|
||||
@@ -146,6 +151,10 @@ func (workers *Workers) Start() *Workers {
|
||||
go workers.ImportProcess.Run()
|
||||
}
|
||||
|
||||
if workers.ImportDelete != nil {
|
||||
go workers.ImportDelete.Run()
|
||||
}
|
||||
|
||||
if workers.Cloud != nil {
|
||||
go workers.Cloud.Run()
|
||||
}
|
||||
@@ -263,6 +272,10 @@ func (workers *Workers) Stop() *Workers {
|
||||
workers.ImportProcess.Stop()
|
||||
}
|
||||
|
||||
if workers.ImportDelete != nil {
|
||||
workers.ImportDelete.Stop()
|
||||
}
|
||||
|
||||
if workers.Cloud != nil {
|
||||
workers.Cloud.Stop()
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ const (
|
||||
JOB_TYPE_PRODUCT_NOTICES = "product_notices"
|
||||
JOB_TYPE_ACTIVE_USERS = "active_users"
|
||||
JOB_TYPE_IMPORT_PROCESS = "import_process"
|
||||
JOB_TYPE_IMPORT_DELETE = "import_delete"
|
||||
JOB_TYPE_CLOUD = "cloud"
|
||||
|
||||
JOB_STATUS_PENDING = "pending"
|
||||
@@ -68,6 +69,7 @@ func (j *Job) IsValid() *AppError {
|
||||
case JOB_TYPE_EXPIRY_NOTIFY:
|
||||
case JOB_TYPE_ACTIVE_USERS:
|
||||
case JOB_TYPE_IMPORT_PROCESS:
|
||||
case JOB_TYPE_IMPORT_DELETE:
|
||||
case JOB_TYPE_CLOUD:
|
||||
default:
|
||||
return NewAppError("Job.IsValid", "model.job.is_valid.type.app_error", nil, "id="+j.Id, http.StatusBadRequest)
|
||||
|
||||
@@ -5,6 +5,7 @@ package filesstore
|
||||
|
||||
import (
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
@@ -28,6 +29,7 @@ type FileBackend interface {
|
||||
WriteFile(fr io.Reader, path string) (int64, error)
|
||||
AppendFile(fr io.Reader, path string) (int64, error)
|
||||
RemoveFile(path string) error
|
||||
FileModTime(path string) (time.Time, error)
|
||||
|
||||
ListDirectory(path string) ([]string, error)
|
||||
RemoveDirectory(path string) error
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"math/rand"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
@@ -388,6 +389,42 @@ func (s *FileBackendTestSuite) TestFileSize() {
|
||||
})
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestFileModTime() {
|
||||
s.Run("nonexistent file", func() {
|
||||
modTime, err := s.backend.FileModTime("tests/nonexistentfile")
|
||||
s.NotNil(err)
|
||||
s.Empty(modTime)
|
||||
})
|
||||
|
||||
s.Run("valid file", func() {
|
||||
path := "tests/" + model.NewId()
|
||||
data := []byte("some data")
|
||||
|
||||
written, err := s.backend.WriteFile(bytes.NewReader(data), path)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(data), written)
|
||||
defer s.backend.RemoveFile(path)
|
||||
|
||||
modTime, err := s.backend.FileModTime(path)
|
||||
s.Nil(err)
|
||||
s.NotEmpty(modTime)
|
||||
|
||||
// We wait 1 second so that the times will differ enough to be testable.
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
path2 := "tests/" + model.NewId()
|
||||
written, err = s.backend.WriteFile(bytes.NewReader(data), path2)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(data), written)
|
||||
defer s.backend.RemoveFile(path2)
|
||||
|
||||
modTime2, err := s.backend.FileModTime(path2)
|
||||
s.Nil(err)
|
||||
s.NotEmpty(modTime2)
|
||||
s.True(modTime2.After(modTime))
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkS3WriteFile(b *testing.B) {
|
||||
utils.TranslationsPreInit()
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
@@ -71,6 +72,14 @@ func (b *LocalFileBackend) FileSize(path string) (int64, error) {
|
||||
return info.Size(), nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) FileModTime(path string) (time.Time, error) {
|
||||
info, err := os.Stat(filepath.Join(b.directory, path))
|
||||
if err != nil {
|
||||
return time.Time{}, errors.Wrapf(err, "unable to get modification time for file %s", path)
|
||||
}
|
||||
return info.ModTime(), nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) CopyFile(oldPath, newPath string) error {
|
||||
if err := utils.CopyFile(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil {
|
||||
return errors.Wrapf(err, "unable to copy file from %s to %s", oldPath, newPath)
|
||||
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
filesstore "github.com/mattermost/mattermost-server/v5/services/filesstore"
|
||||
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
|
||||
time "time"
|
||||
)
|
||||
|
||||
// FileBackend is an autogenerated mock type for the FileBackend type
|
||||
@@ -73,6 +75,27 @@ func (_m *FileBackend) FileExists(path string) (bool, error) {
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// FileModTime provides a mock function with given fields: path
|
||||
func (_m *FileBackend) FileModTime(path string) (time.Time, error) {
|
||||
ret := _m.Called(path)
|
||||
|
||||
var r0 time.Time
|
||||
if rf, ok := ret.Get(0).(func(string) time.Time); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
r0 = ret.Get(0).(time.Time)
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(string) error); ok {
|
||||
r1 = rf(path)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// FileSize provides a mock function with given fields: path
|
||||
func (_m *FileBackend) FileSize(path string) (int64, error) {
|
||||
ret := _m.Called(path)
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
s3 "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
@@ -205,6 +206,17 @@ func (b *S3FileBackend) FileSize(path string) (int64, error) {
|
||||
return info.Size, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) FileModTime(path string) (time.Time, error) {
|
||||
path = filepath.Join(b.pathPrefix, path)
|
||||
|
||||
info, err := b.client.StatObject(context.Background(), b.bucket, path, s3.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return time.Time{}, errors.Wrapf(err, "unable to get modification time for file %s", path)
|
||||
}
|
||||
|
||||
return info.LastModified, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) CopyFile(oldPath, newPath string) error {
|
||||
oldPath = filepath.Join(b.pathPrefix, oldPath)
|
||||
newPath = filepath.Join(b.pathPrefix, newPath)
|
||||
|
||||
Ссылка в новой задаче
Block a user