[MM-28423] Implement ImportDelete job (#16588)

* Implement ImportDelete job

* Add missing translation

* Improve logging

* Avoid deleting the file in case of errors
Этот коммит содержится в:
Claudio Costa
2021-01-25 10:40:30 +01:00
коммит произвёл GitHub
родитель b932e0fb25
Коммит 200a56fa5a
22 изменённых файлов: 401 добавлений и 4 удалений

Просмотреть файл

@@ -119,6 +119,9 @@ func (a *App) initJobs() {
if jobsImportProcessInterface != nil {
a.srv.Jobs.ImportProcess = jobsImportProcessInterface(a)
}
if jobsImportDeleteInterface != nil {
a.srv.Jobs.ImportDelete = jobsImportDeleteInterface(a)
}
if jobsActiveUsersInterface != nil {
a.srv.Jobs.ActiveUsers = jobsActiveUsersInterface(a)

Просмотреть файл

@@ -498,6 +498,7 @@ type AppIface interface {
FetchSamlMetadataFromIdp(url string) ([]byte, *model.AppError)
FileBackend() (filesstore.FileBackend, *model.AppError)
FileExists(path string) (bool, *model.AppError)
FileModTime(path string) (time.Time, *model.AppError)
FileSize(path string) (int64, *model.AppError)
FillInChannelProps(channel *model.Channel) *model.AppError
FillInChannelsProps(channelList *model.ChannelList) *model.AppError

Просмотреть файл

@@ -114,6 +114,12 @@ func RegisterJobsImportProcessInterface(f func(*App) tjobs.ImportProcessInterfac
jobsImportProcessInterface = f
}
var jobsImportDeleteInterface func(*App) tjobs.ImportDeleteInterface
func RegisterJobsImportDeleteInterface(f func(*App) tjobs.ImportDeleteInterface) {
jobsImportDeleteInterface = f
}
var productNoticesJobInterface func(*App) tjobs.ProductNoticesJobInterface
func RegisterProductNoticesJobInterface(f func(*App) tjobs.ProductNoticesJobInterface) {

Просмотреть файл

@@ -158,6 +158,19 @@ func (a *App) FileSize(path string) (int64, *model.AppError) {
return size, nil
}
func (a *App) FileModTime(path string) (time.Time, *model.AppError) {
backend, err := a.FileBackend()
if err != nil {
return time.Time{}, err
}
modTime, nErr := backend.FileModTime(path)
if nErr != nil {
return time.Time{}, model.NewAppError("FileModTime", "api.file.file_mod_time.app_error", nil, nErr.Error(), http.StatusInternalServerError)
}
return modTime, nil
}
func (a *App) MoveFile(oldPath, newPath string) *model.AppError {
backend, err := a.FileBackend()
if err != nil {

Просмотреть файл

@@ -277,7 +277,7 @@ func (a *App) ListImports() ([]string, *model.AppError) {
results := make([]string, 0, len(imports))
for i := 0; i < len(imports); i++ {
filename := filepath.Base(imports[i])
if !strings.HasSuffix(filename, incompleteUploadSuffix) {
if !strings.HasSuffix(filename, IncompleteUploadSuffix) {
results = append(results, filename)
}
}

Просмотреть файл

@@ -3689,6 +3689,28 @@ func (a *OpenTracingAppLayer) FileExists(path string) (bool, *model.AppError) {
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) FileModTime(path string) (time.Time, *model.AppError) {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.FileModTime")
a.ctx = newCtx
a.app.Srv().Store.SetContext(newCtx)
defer func() {
a.app.Srv().Store.SetContext(origCtx)
a.ctx = origCtx
}()
defer span.Finish()
resultVar0, resultVar1 := a.app.FileModTime(path)
if resultVar1 != nil {
span.LogFields(spanlog.Error(resultVar1))
ext.Error.Set(span, true)
}
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) FileReader(path string) (filesstore.ReadCloseSeeker, *model.AppError) {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.FileReader")

Просмотреть файл

@@ -19,7 +19,7 @@ import (
)
const minFirstPartSize = 5 * 1024 * 1024 // 5MB
const incompleteUploadSuffix = ".tmp"
const IncompleteUploadSuffix = ".tmp"
func (a *App) runPluginsHook(info *model.FileInfo, file io.Reader) *model.AppError {
pluginsEnvironment := a.GetPluginsEnvironment()
@@ -111,7 +111,7 @@ func (a *App) CreateUploadSession(us *model.UploadSession) (*model.UploadSession
if us.Type == model.UploadTypeAttachment {
us.Path = now.Format("20060102") + "/teams/noteam/channels/" + us.ChannelId + "/users/" + us.UserId + "/" + us.Id + "/" + filepath.Base(us.Filename)
} else if us.Type == model.UploadTypeImport {
us.Path = *a.Config().ImportSettings.Directory + "/" + us.Id + "_" + filepath.Base(us.Filename)
us.Path = filepath.Clean(*a.Config().ImportSettings.Directory) + "/" + us.Id + "_" + filepath.Base(us.Filename)
}
if err := us.IsValid(); err != nil {
return nil, err
@@ -194,7 +194,7 @@ func (a *App) UploadData(us *model.UploadSession, rd io.Reader) (*model.FileInfo
uploadPath := us.Path
if us.Type == model.UploadTypeImport {
uploadPath += incompleteUploadSuffix
uploadPath += IncompleteUploadSuffix
}
// make sure it's not possible to upload more data than what is expected.

Просмотреть файл

@@ -1336,6 +1336,10 @@
"id": "api.file.file_exists.app_error",
"translation": "Unable to check if the file exists."
},
{
"id": "api.file.file_mod_time.app_error",
"translation": "Unable to get last modification time for file."
},
{
"id": "api.file.file_reader.app_error",
"translation": "Unable to get a file reader."

Просмотреть файл

@@ -24,4 +24,7 @@ import (
// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
_ "github.com/mattermost/mattermost-server/v5/jobs/import_process"
// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
_ "github.com/mattermost/mattermost-server/v5/jobs/import_delete"
)

51
jobs/import_delete/scheduler.go Обычный файл
Просмотреть файл

@@ -0,0 +1,51 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package import_delete
import (
"time"
"github.com/mattermost/mattermost-server/v5/app"
"github.com/mattermost/mattermost-server/v5/model"
)
const (
jobName = "ImportDelete"
schedFrequency = 24 * time.Hour
)
type Scheduler struct {
app *app.App
}
func (i *ImportDeleteInterfaceImpl) MakeScheduler() model.Scheduler {
return &Scheduler{i.app}
}
func (scheduler *Scheduler) Name() string {
return jobName + "Scheduler"
}
func (scheduler *Scheduler) JobType() string {
return model.JOB_TYPE_IMPORT_DELETE
}
func (scheduler *Scheduler) Enabled(cfg *model.Config) bool {
return *cfg.ImportSettings.Directory != "" && *cfg.ImportSettings.RetentionDays > 0
}
func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
nextTime := time.Now().Add(schedFrequency)
return &nextTime
}
func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) {
data := map[string]string{}
job, err := scheduler.app.Srv().Jobs.CreateJob(model.JOB_TYPE_IMPORT_DELETE, data)
if err != nil {
return nil, err
}
return job, nil
}

172
jobs/import_delete/worker.go Обычный файл
Просмотреть файл

@@ -0,0 +1,172 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package import_delete
import (
"errors"
"path/filepath"
"time"
"github.com/mattermost/mattermost-server/v5/app"
"github.com/mattermost/mattermost-server/v5/jobs"
tjobs "github.com/mattermost/mattermost-server/v5/jobs/interfaces"
"github.com/mattermost/mattermost-server/v5/mlog"
"github.com/mattermost/mattermost-server/v5/model"
"github.com/mattermost/mattermost-server/v5/store"
)
func init() {
app.RegisterJobsImportDeleteInterface(func(a *app.App) tjobs.ImportDeleteInterface {
return &ImportDeleteInterfaceImpl{a}
})
}
type ImportDeleteInterfaceImpl struct {
app *app.App
}
type ImportDeleteWorker struct {
name string
stopChan chan struct{}
stoppedChan chan struct{}
jobsChan chan model.Job
jobServer *jobs.JobServer
app *app.App
}
func (i *ImportDeleteInterfaceImpl) MakeWorker() model.Worker {
return &ImportDeleteWorker{
name: "ImportDelete",
stopChan: make(chan struct{}),
stoppedChan: make(chan struct{}),
jobsChan: make(chan model.Job),
jobServer: i.app.Srv().Jobs,
app: i.app,
}
}
func (w *ImportDeleteWorker) JobChannel() chan<- model.Job {
return w.jobsChan
}
func (w *ImportDeleteWorker) Run() {
mlog.Debug("Worker started", mlog.String("worker", w.name))
defer func() {
mlog.Debug("Worker finished", mlog.String("worker", w.name))
close(w.stoppedChan)
}()
for {
select {
case <-w.stopChan:
mlog.Debug("Worker received stop signal", mlog.String("worker", w.name))
return
case job := <-w.jobsChan:
mlog.Debug("Worker received a new candidate job.", mlog.String("worker", w.name))
w.doJob(&job)
}
}
}
func (w *ImportDeleteWorker) Stop() {
mlog.Debug("Worker stopping", mlog.String("worker", w.name))
close(w.stopChan)
<-w.stoppedChan
}
func (w *ImportDeleteWorker) doJob(job *model.Job) {
if claimed, err := w.jobServer.ClaimJob(job); err != nil {
mlog.Warn("Worker experienced an error while trying to claim job",
mlog.String("worker", w.name),
mlog.String("job_id", job.Id),
mlog.String("error", err.Error()))
return
} else if !claimed {
return
}
importPath := *w.app.Config().ImportSettings.Directory
retentionTime := time.Duration(*w.app.Config().ImportSettings.RetentionDays) * 24 * time.Hour
imports, appErr := w.app.ListDirectory(importPath)
if appErr != nil {
w.setJobError(job, appErr)
return
}
var hasErrs bool
for i := range imports {
filename := filepath.Base(imports[i])
modTime, appErr := w.app.FileModTime(filepath.Join(importPath, filename))
if appErr != nil {
mlog.Debug("Worker: Failed to get file modification time",
mlog.Err(appErr), mlog.String("import", imports[i]))
hasErrs = true
continue
}
if time.Now().After(modTime.Add(retentionTime)) {
// expected format if uploaded through the API is
// ${uploadID}_${filename}${app.IncompleteUploadSuffix}
minLen := 26 + 1 + len(app.IncompleteUploadSuffix)
// check if it's an incomplete upload and attempt to delete its session.
if len(filename) > minLen && filepath.Ext(filename) == app.IncompleteUploadSuffix {
uploadID := filename[:26]
if storeErr := w.app.Srv().Store.UploadSession().Delete(uploadID); storeErr != nil {
mlog.Debug("Worker: Failed to delete UploadSession",
mlog.Err(storeErr), mlog.String("upload_id", uploadID))
hasErrs = true
continue
}
} else {
// check if fileinfo exists and if so delete it.
filePath := filepath.Join(imports[i])
info, storeErr := w.app.Srv().Store.FileInfo().GetByPath(filePath)
var nfErr *store.ErrNotFound
if storeErr != nil && !errors.As(storeErr, &nfErr) {
mlog.Debug("Worker: Failed to get FileInfo",
mlog.Err(storeErr), mlog.String("path", filePath))
hasErrs = true
continue
} else if storeErr == nil {
if storeErr = w.app.Srv().Store.FileInfo().PermanentDelete(info.Id); storeErr != nil {
mlog.Debug("Worker: Failed to delete FileInfo",
mlog.Err(storeErr), mlog.String("file_id", info.Id))
hasErrs = true
continue
}
}
}
// remove file data from storage.
if appErr := w.app.RemoveFile(imports[i]); appErr != nil {
mlog.Debug("Worker: Failed to remove file",
mlog.Err(appErr), mlog.String("import", imports[i]))
hasErrs = true
continue
}
}
}
if hasErrs {
mlog.Warn("Worker: errors occurred")
}
mlog.Info("Worker: Job is complete", mlog.String("worker", w.name), mlog.String("job_id", job.Id))
w.setJobSuccess(job)
}
func (w *ImportDeleteWorker) setJobSuccess(job *model.Job) {
if err := w.app.Srv().Jobs.SetJobSuccess(job); err != nil {
mlog.Error("Worker: Failed to set success for job", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
w.setJobError(job, err)
}
}
func (w *ImportDeleteWorker) setJobError(job *model.Job, appError *model.AppError) {
if err := w.app.Srv().Jobs.SetJobError(job, appError); err != nil {
mlog.Error("Worker: Failed to set job error", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
}
}

11
jobs/interfaces/import_delete_interface.go Обычный файл
Просмотреть файл

@@ -0,0 +1,11 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package interfaces
import "github.com/mattermost/mattermost-server/v5/model"
type ImportDeleteInterface interface {
MakeWorker() model.Worker
MakeScheduler() model.Scheduler
}

Просмотреть файл

@@ -156,6 +156,13 @@ func (watcher *Watcher) PollAndNotify() {
default:
}
}
} else if job.Type == model.JOB_TYPE_IMPORT_DELETE {
if watcher.workers.ImportDelete != nil {
select {
case watcher.workers.ImportDelete.JobChannel() <- *job:
default:
}
}
} else if job.Type == model.JOB_TYPE_CLOUD {
if watcher.workers.Cloud != nil {
select {

Просмотреть файл

@@ -69,6 +69,7 @@ func (srv *JobServer) InitSchedulers() *Schedulers {
if activeUsersInterface := srv.ActiveUsers; activeUsersInterface != nil {
schedulers.schedulers = append(schedulers.schedulers, activeUsersInterface.MakeScheduler())
}
if productNoticesInterface := srv.ProductNotices; productNoticesInterface != nil {
schedulers.schedulers = append(schedulers.schedulers, productNoticesInterface.MakeScheduler())
}
@@ -77,6 +78,10 @@ func (srv *JobServer) InitSchedulers() *Schedulers {
schedulers.schedulers = append(schedulers.schedulers, cloudInterface.MakeScheduler())
}
if importDeleteInterface := srv.ImportDelete; importDeleteInterface != nil {
schedulers.schedulers = append(schedulers.schedulers, importDeleteInterface.MakeScheduler())
}
schedulers.nextRunTimes = make([]*time.Time, len(schedulers.schedulers))
return schedulers
}

Просмотреть файл

@@ -31,6 +31,7 @@ type JobServer struct {
ProductNotices tjobs.ProductNoticesJobInterface
ActiveUsers tjobs.ActiveUsersJobInterface
ImportProcess tjobs.ImportProcessInterface
ImportDelete tjobs.ImportDeleteInterface
Cloud ejobs.CloudJobInterface
}

Просмотреть файл

@@ -28,6 +28,7 @@ type Workers struct {
ProductNotices model.Worker
ActiveUsers model.Worker
ImportProcess model.Worker
ImportDelete model.Worker
Cloud model.Worker
listenerId string
@@ -87,6 +88,10 @@ func (srv *JobServer) InitWorkers() *Workers {
workers.ImportProcess = importProcessInterface.MakeWorker()
}
if importDeleteInterface := srv.ImportDelete; importDeleteInterface != nil {
workers.ImportDelete = importDeleteInterface.MakeWorker()
}
if cloudInterface := srv.Cloud; cloudInterface != nil {
workers.Cloud = cloudInterface.MakeWorker()
}
@@ -146,6 +151,10 @@ func (workers *Workers) Start() *Workers {
go workers.ImportProcess.Run()
}
if workers.ImportDelete != nil {
go workers.ImportDelete.Run()
}
if workers.Cloud != nil {
go workers.Cloud.Run()
}
@@ -263,6 +272,10 @@ func (workers *Workers) Stop() *Workers {
workers.ImportProcess.Stop()
}
if workers.ImportDelete != nil {
workers.ImportDelete.Stop()
}
if workers.Cloud != nil {
workers.Cloud.Stop()
}

Просмотреть файл

@@ -23,6 +23,7 @@ const (
JOB_TYPE_PRODUCT_NOTICES = "product_notices"
JOB_TYPE_ACTIVE_USERS = "active_users"
JOB_TYPE_IMPORT_PROCESS = "import_process"
JOB_TYPE_IMPORT_DELETE = "import_delete"
JOB_TYPE_CLOUD = "cloud"
JOB_STATUS_PENDING = "pending"
@@ -68,6 +69,7 @@ func (j *Job) IsValid() *AppError {
case JOB_TYPE_EXPIRY_NOTIFY:
case JOB_TYPE_ACTIVE_USERS:
case JOB_TYPE_IMPORT_PROCESS:
case JOB_TYPE_IMPORT_DELETE:
case JOB_TYPE_CLOUD:
default:
return NewAppError("Job.IsValid", "model.job.is_valid.type.app_error", nil, "id="+j.Id, http.StatusBadRequest)

Просмотреть файл

@@ -5,6 +5,7 @@ package filesstore
import (
"io"
"time"
"github.com/pkg/errors"
@@ -28,6 +29,7 @@ type FileBackend interface {
WriteFile(fr io.Reader, path string) (int64, error)
AppendFile(fr io.Reader, path string) (int64, error)
RemoveFile(path string) error
FileModTime(path string) (time.Time, error)
ListDirectory(path string) ([]string, error)
RemoveDirectory(path string) error

Просмотреть файл

@@ -10,6 +10,7 @@ import (
"math/rand"
"os"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
@@ -388,6 +389,42 @@ func (s *FileBackendTestSuite) TestFileSize() {
})
}
func (s *FileBackendTestSuite) TestFileModTime() {
s.Run("nonexistent file", func() {
modTime, err := s.backend.FileModTime("tests/nonexistentfile")
s.NotNil(err)
s.Empty(modTime)
})
s.Run("valid file", func() {
path := "tests/" + model.NewId()
data := []byte("some data")
written, err := s.backend.WriteFile(bytes.NewReader(data), path)
s.Nil(err)
s.EqualValues(len(data), written)
defer s.backend.RemoveFile(path)
modTime, err := s.backend.FileModTime(path)
s.Nil(err)
s.NotEmpty(modTime)
// We wait 1 second so that the times will differ enough to be testable.
time.Sleep(1 * time.Second)
path2 := "tests/" + model.NewId()
written, err = s.backend.WriteFile(bytes.NewReader(data), path2)
s.Nil(err)
s.EqualValues(len(data), written)
defer s.backend.RemoveFile(path2)
modTime2, err := s.backend.FileModTime(path2)
s.Nil(err)
s.NotEmpty(modTime2)
s.True(modTime2.After(modTime))
})
}
func BenchmarkS3WriteFile(b *testing.B) {
utils.TranslationsPreInit()

Просмотреть файл

@@ -9,6 +9,7 @@ import (
"io/ioutil"
"os"
"path/filepath"
"time"
"github.com/pkg/errors"
@@ -71,6 +72,14 @@ func (b *LocalFileBackend) FileSize(path string) (int64, error) {
return info.Size(), nil
}
func (b *LocalFileBackend) FileModTime(path string) (time.Time, error) {
info, err := os.Stat(filepath.Join(b.directory, path))
if err != nil {
return time.Time{}, errors.Wrapf(err, "unable to get modification time for file %s", path)
}
return info.ModTime(), nil
}
func (b *LocalFileBackend) CopyFile(oldPath, newPath string) error {
if err := utils.CopyFile(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil {
return errors.Wrapf(err, "unable to copy file from %s to %s", oldPath, newPath)

Просмотреть файл

@@ -10,6 +10,8 @@ import (
filesstore "github.com/mattermost/mattermost-server/v5/services/filesstore"
mock "github.com/stretchr/testify/mock"
time "time"
)
// FileBackend is an autogenerated mock type for the FileBackend type
@@ -73,6 +75,27 @@ func (_m *FileBackend) FileExists(path string) (bool, error) {
return r0, r1
}
// FileModTime provides a mock function with given fields: path
func (_m *FileBackend) FileModTime(path string) (time.Time, error) {
ret := _m.Called(path)
var r0 time.Time
if rf, ok := ret.Get(0).(func(string) time.Time); ok {
r0 = rf(path)
} else {
r0 = ret.Get(0).(time.Time)
}
var r1 error
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(path)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// FileSize provides a mock function with given fields: path
func (_m *FileBackend) FileSize(path string) (int64, error) {
ret := _m.Called(path)

Просмотреть файл

@@ -10,6 +10,7 @@ import (
"os"
"path/filepath"
"strings"
"time"
s3 "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
@@ -205,6 +206,17 @@ func (b *S3FileBackend) FileSize(path string) (int64, error) {
return info.Size, nil
}
func (b *S3FileBackend) FileModTime(path string) (time.Time, error) {
path = filepath.Join(b.pathPrefix, path)
info, err := b.client.StatObject(context.Background(), b.bucket, path, s3.StatObjectOptions{})
if err != nil {
return time.Time{}, errors.Wrapf(err, "unable to get modification time for file %s", path)
}
return info.LastModified, nil
}
func (b *S3FileBackend) CopyFile(oldPath, newPath string) error {
oldPath = filepath.Join(b.pathPrefix, oldPath)
newPath = filepath.Join(b.pathPrefix, newPath)