Mono repo -> Master (#22553)

Combines the following repositories into one:

https://github.com/mattermost/mattermost-server
https://github.com/mattermost/mattermost-webapp
https://github.com/mattermost/focalboard
https://github.com/mattermost/mattermost-plugin-playbooks

server/channels/jobs/active_users/scheduler.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package active_users
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+)
+
+const schedFreq = 10 * time.Minute
+
+func MakeScheduler(jobServer *jobs.JobServer) model.Scheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.MetricsSettings.Enable
+	}
+	return jobs.NewPeriodicScheduler(jobServer, model.JobTypeActiveUsers, schedFreq, isEnabled)
+}

server/channels/jobs/active_users/worker.go

@@ -0,0 +1,36 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package active_users
+
+import (
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/einterfaces"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+)
+
+const (
+	JobName = "ActiveUsers"
+)
+
+func MakeWorker(jobServer *jobs.JobServer, store store.Store, getMetrics func() einterfaces.MetricsInterface) model.Worker {
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.MetricsSettings.Enable
+	}
+	execute := func(job *model.Job) error {
+		defer jobServer.HandleJobPanic(job)
+
+		count, err := store.User().Count(model.UserCountOptions{IncludeDeleted: false})
+		if err != nil {
+			return err
+		}
+
+		if getMetrics() != nil {
+			getMetrics().ObserveEnabledUsers(count)
+		}
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(JobName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/base_schedulers.go

@@ -0,0 +1,84 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs
+
+import (
+	"crypto/rand"
+	"math/big"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+type PeriodicScheduler struct {
+	jobs        *JobServer
+	period      time.Duration
+	jobType     string
+	enabledFunc func(cfg *model.Config) bool
+}
+
+func NewPeriodicScheduler(jobs *JobServer, jobType string, period time.Duration, enabledFunc func(cfg *model.Config) bool) *PeriodicScheduler {
+	return &PeriodicScheduler{
+		period:      period,
+		jobType:     jobType,
+		enabledFunc: enabledFunc,
+		jobs:        jobs,
+	}
+}
+
+func (scheduler *PeriodicScheduler) Enabled(cfg *model.Config) bool {
+	return scheduler.enabledFunc(cfg)
+}
+
+func (scheduler *PeriodicScheduler) NextScheduleTime(_ *model.Config, _ time.Time /* pendingJobs */, _ bool /* lastSuccessfulJob */, _ *model.Job) *time.Time {
+	nextTime := time.Now().Add(getRandomDelay(jitterRange)).Add(scheduler.period)
+	return &nextTime
+}
+
+func (scheduler *PeriodicScheduler) ScheduleJob(_ *model.Config /* pendingJobs */, _ bool /* lastSuccessfulJob */, _ *model.Job) (*model.Job, *model.AppError) {
+	return scheduler.jobs.CreateJob(scheduler.jobType, nil)
+}
+
+type DailyScheduler struct {
+	jobs          *JobServer
+	startTimeFunc func(cfg *model.Config) *time.Time
+	jobType       string
+	enabledFunc   func(cfg *model.Config) bool
+}
+
+func NewDailyScheduler(jobs *JobServer, jobType string, startTimeFunc func(cfg *model.Config) *time.Time, enabledFunc func(cfg *model.Config) bool) *DailyScheduler {
+	return &DailyScheduler{
+		startTimeFunc: startTimeFunc,
+		jobType:       jobType,
+		enabledFunc:   enabledFunc,
+		jobs:          jobs,
+	}
+}
+
+func (scheduler *DailyScheduler) Enabled(cfg *model.Config) bool {
+	return scheduler.enabledFunc(cfg)
+}
+
+func (scheduler *DailyScheduler) NextScheduleTime(cfg *model.Config, now time.Time /* pendingJobs */, _ bool /* lastSuccessfulJob */, _ *model.Job) *time.Time {
+	scheduledTime := scheduler.startTimeFunc(cfg)
+	if scheduledTime == nil {
+		return nil
+	}
+
+	return GenerateNextStartDateTime(now, *scheduledTime)
+}
+
+func (scheduler *DailyScheduler) ScheduleJob(_ *model.Config /* pendingJobs */, _ bool /* lastSuccessfulJob */, _ *model.Job) (*model.Job, *model.AppError) {
+	return scheduler.jobs.CreateJob(scheduler.jobType, nil)
+}
+
+const jitterRange = 2000 // milliseconds
+
+func getRandomDelay(limit int64) time.Duration {
+	num, err := rand.Int(rand.Reader, big.NewInt(limit))
+	if err != nil {
+		return time.Millisecond
+	}
+	return time.Millisecond * time.Duration(num.Int64())
+}

server/channels/jobs/base_workers.go

@@ -0,0 +1,116 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs
+
+import (
+	"net/http"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+type SimpleWorker struct {
+	name      string
+	stop      chan bool
+	stopped   chan bool
+	jobs      chan model.Job
+	jobServer *JobServer
+	execute   func(job *model.Job) error
+	isEnabled func(cfg *model.Config) bool
+}
+
+func NewSimpleWorker(name string, jobServer *JobServer, execute func(job *model.Job) error, isEnabled func(cfg *model.Config) bool) *SimpleWorker {
+	worker := SimpleWorker{
+		name:      name,
+		stop:      make(chan bool, 1),
+		stopped:   make(chan bool, 1),
+		jobs:      make(chan model.Job),
+		jobServer: jobServer,
+		execute:   execute,
+		isEnabled: isEnabled,
+	}
+	return &worker
+}
+
+func (worker *SimpleWorker) Run() {
+	mlog.Debug("Worker started", mlog.String("worker", worker.name))
+
+	defer func() {
+		mlog.Debug("Worker finished", mlog.String("worker", worker.name))
+		worker.stopped <- true
+	}()
+
+	for {
+		select {
+		case <-worker.stop:
+			mlog.Debug("Worker received stop signal", mlog.String("worker", worker.name))
+			return
+		case job := <-worker.jobs:
+			mlog.Debug("Worker received a new candidate job.", mlog.String("worker", worker.name))
+			worker.DoJob(&job)
+		}
+	}
+}
+
+func (worker *SimpleWorker) Stop() {
+	mlog.Debug("Worker stopping", mlog.String("worker", worker.name))
+	worker.stop <- true
+	<-worker.stopped
+}
+
+func (worker *SimpleWorker) JobChannel() chan<- model.Job {
+	return worker.jobs
+}
+
+func (worker *SimpleWorker) IsEnabled(cfg *model.Config) bool {
+	return worker.isEnabled(cfg)
+}
+
+func (worker *SimpleWorker) DoJob(job *model.Job) {
+	if claimed, err := worker.jobServer.ClaimJob(job); err != nil {
+		mlog.Warn("SimpleWorker experienced an error while trying to claim job",
+			mlog.String("worker", worker.name),
+			mlog.String("job_id", job.Id),
+			mlog.Err(err))
+		return
+	} else if !claimed {
+		return
+	}
+
+	var appErr *model.AppError
+	// We get the job again because ClaimJob changes the job status.
+	job, appErr = worker.jobServer.GetJob(job.Id)
+	if appErr != nil {
+		mlog.Error("SimpleWorker: job execution error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.Err(appErr))
+		worker.setJobError(job, appErr)
+	}
+
+	err := worker.execute(job)
+	if err != nil {
+		mlog.Error("SimpleWorker: job execution error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+		worker.setJobError(job, model.NewAppError("DoJob", "app.job.error", nil, "", http.StatusInternalServerError).Wrap(err))
+		return
+	}
+
+	mlog.Info("SimpleWorker: Job is complete", mlog.String("worker", worker.name), mlog.String("job_id", job.Id))
+	worker.setJobSuccess(job)
+}
+
+func (worker *SimpleWorker) setJobSuccess(job *model.Job) {
+	if err := worker.jobServer.SetJobProgress(job, 100); err != nil {
+		mlog.Error("Worker: Failed to update progress for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+		worker.setJobError(job, err)
+	}
+
+	if err := worker.jobServer.SetJobSuccess(job); err != nil {
+		mlog.Error("SimpleWorker: Failed to set success for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+		worker.setJobError(job, err)
+	}
+}
+
+func (worker *SimpleWorker) setJobError(job *model.Job, appError *model.AppError) {
+	if err := worker.jobServer.SetJobError(job, appError); err != nil {
+		mlog.Error("SimpleWorker: Failed to set job error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+	}
+}

server/channels/jobs/expirynotify/scheduler.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package expirynotify
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+)
+
+const schedFreq = 10 * time.Minute
+
+func MakeScheduler(jobServer *jobs.JobServer) model.Scheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.ServiceSettings.ExtendSessionLengthWithActivity
+	}
+	return jobs.NewPeriodicScheduler(jobServer, model.JobTypeExpiryNotify, schedFreq, isEnabled)
+}

server/channels/jobs/expirynotify/worker.go

@@ -0,0 +1,25 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package expirynotify
+
+import (
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+)
+
+const (
+	JobName = "ExpiryNotify"
+)
+
+func MakeWorker(jobServer *jobs.JobServer, notifySessionsExpired func() error) model.Worker {
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.ServiceSettings.ExtendSessionLengthWithActivity
+	}
+	execute := func(job *model.Job) error {
+		defer jobServer.HandleJobPanic(job)
+
+		return notifySessionsExpired()
+	}
+	return jobs.NewSimpleWorker(JobName, jobServer, execute, isEnabled)
+}

server/channels/jobs/export_delete/scheduler.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package export_delete
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+)
+
+const schedFreq = 24 * time.Hour
+
+func MakeScheduler(jobServer *jobs.JobServer) model.Scheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.ExportSettings.Directory != "" && *cfg.ExportSettings.RetentionDays > 0
+	}
+	return jobs.NewPeriodicScheduler(jobServer, model.JobTypeExportDelete, schedFreq, isEnabled)
+}

server/channels/jobs/export_delete/worker.go

@@ -0,0 +1,70 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package export_delete
+
+import (
+	"path/filepath"
+	"time"
+
+	"github.com/wiggin77/merror"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/configservice"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const jobName = "ExportDelete"
+
+type AppIface interface {
+	configservice.ConfigService
+	ListDirectory(path string) ([]string, *model.AppError)
+	FileModTime(path string) (time.Time, *model.AppError)
+	RemoveFile(path string) *model.AppError
+}
+
+func MakeWorker(jobServer *jobs.JobServer, app AppIface) model.Worker {
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.ExportSettings.Directory != "" && *cfg.ExportSettings.RetentionDays > 0
+	}
+	execute := func(job *model.Job) error {
+		defer jobServer.HandleJobPanic(job)
+
+		exportPath := *app.Config().ExportSettings.Directory
+		retentionTime := time.Duration(*app.Config().ExportSettings.RetentionDays) * 24 * time.Hour
+		exports, appErr := app.ListDirectory(exportPath)
+		if appErr != nil {
+			return appErr
+		}
+
+		errors := merror.New()
+		for i := range exports {
+			filename := filepath.Base(exports[i])
+			modTime, appErr := app.FileModTime(filepath.Join(exportPath, filename))
+			if appErr != nil {
+				mlog.Debug("Worker: Failed to get file modification time",
+					mlog.Err(appErr), mlog.String("export", exports[i]))
+				errors.Append(appErr)
+				continue
+			}
+
+			if time.Now().After(modTime.Add(retentionTime)) {
+				// remove file data from storage.
+				if appErr := app.RemoveFile(exports[i]); appErr != nil {
+					mlog.Debug("Worker: Failed to remove file",
+						mlog.Err(appErr), mlog.String("export", exports[i]))
+					errors.Append(appErr)
+					continue
+				}
+			}
+		}
+
+		if err := errors.ErrorOrNil(); err != nil {
+			mlog.Warn("Worker: errors occurred", mlog.String("job-name", jobName), mlog.Err(err))
+		}
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/export_process/worker.go

@@ -0,0 +1,70 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package export_process
+
+import (
+	"context"
+	"io"
+	"path/filepath"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/app/request"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/configservice"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const jobName = "ExportProcess"
+
+type AppIface interface {
+	configservice.ConfigService
+	WriteFile(fr io.Reader, path string) (int64, *model.AppError)
+	WriteFileContext(ctx context.Context, fr io.Reader, path string) (int64, *model.AppError)
+	BulkExport(ctx request.CTX, writer io.Writer, outPath string, job *model.Job, opts model.BulkExportOpts) *model.AppError
+	Log() *mlog.Logger
+}
+
+func MakeWorker(jobServer *jobs.JobServer, app AppIface) model.Worker {
+	isEnabled := func(cfg *model.Config) bool { return true }
+	execute := func(job *model.Job) error {
+		defer jobServer.HandleJobPanic(job)
+
+		opts := model.BulkExportOpts{
+			CreateArchive: true,
+		}
+
+		includeAttachments, ok := job.Data["include_attachments"]
+		if ok && includeAttachments == "true" {
+			opts.IncludeAttachments = true
+		}
+
+		outPath := *app.Config().ExportSettings.Directory
+		exportFilename := job.Id + "_export.zip"
+
+		rd, wr := io.Pipe()
+
+		go func() {
+			_, appErr := app.WriteFileContext(context.Background(), rd, filepath.Join(outPath, exportFilename))
+			if appErr != nil {
+				// we close the reader here to prevent a deadlock when the bulk exporter tries to
+				// write into the pipe while app.WriteFile has already returned. The error will be
+				// returned by the writer part of the pipe when app.BulkExport tries to call
+				// wr.Write() on it.
+				rd.CloseWithError(appErr) // CloseWithError never returns an error
+			}
+		}()
+
+		logger := app.Log().With(mlog.String("job_id", job.Id))
+		appErr := app.BulkExport(request.EmptyContext(logger), wr, outPath, job, opts)
+		wr.Close() // Close never returns an error
+
+		if appErr != nil {
+			return appErr
+		}
+
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/extract_content/worker.go

@@ -0,0 +1,94 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package extract_content
+
+import (
+	"strconv"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+var ignoredFiles = map[string]bool{
+	"png": true, "jpg": true, "jpeg": true, "gif": true, "wmv": true,
+	"mpg": true, "mpeg": true, "mp3": true, "mp4": true, "ogg": true,
+	"ogv": true, "mov": true, "apk": true, "svg": true, "webm": true,
+	"mkv": true,
+}
+
+const jobName = "ExtractContent"
+
+type AppIface interface {
+	ExtractContentFromFileInfo(fileInfo *model.FileInfo) error
+}
+
+func MakeWorker(jobServer *jobs.JobServer, app AppIface, store store.Store) model.Worker {
+	isEnabled := func(cfg *model.Config) bool {
+		return true
+	}
+	execute := func(job *model.Job) error {
+		jobServer.HandleJobPanic(job)
+
+		var err error
+		var fromTS int64 = 0
+		var toTS int64 = model.GetMillis()
+		if fromStr, ok := job.Data["from"]; ok {
+			if fromTS, err = strconv.ParseInt(fromStr, 10, 64); err != nil {
+				return err
+			}
+			fromTS *= 1000
+		}
+		if toStr, ok := job.Data["to"]; ok {
+			if toTS, err = strconv.ParseInt(toStr, 10, 64); err != nil {
+				return err
+			}
+			toTS *= 1000
+		}
+
+		var nFiles int
+		var nErrs int
+		for {
+			opts := model.GetFileInfosOptions{
+				Since:          fromTS,
+				SortBy:         model.FileinfoSortByCreated,
+				IncludeDeleted: false,
+			}
+			fileInfos, err := store.FileInfo().GetWithOptions(0, 1000, &opts)
+			if err != nil {
+				return err
+			}
+			if len(fileInfos) == 0 {
+				break
+			}
+			for _, fileInfo := range fileInfos {
+				if !ignoredFiles[fileInfo.Extension] {
+					mlog.Debug("extracting file", mlog.String("filename", fileInfo.Name), mlog.String("filepath", fileInfo.Path))
+					err = app.ExtractContentFromFileInfo(fileInfo)
+					if err != nil {
+						mlog.Warn("Failed to extract file content", mlog.Err(err), mlog.String("file_info_id", fileInfo.Id))
+						nErrs++
+					}
+					nFiles++
+				}
+			}
+			lastFileInfo := fileInfos[len(fileInfos)-1]
+			if lastFileInfo.CreateAt > toTS {
+				break
+			}
+			fromTS = lastFileInfo.CreateAt + 1
+		}
+
+		job.Data["errors"] = strconv.Itoa(nErrs)
+		job.Data["processed"] = strconv.Itoa(nFiles)
+
+		if err := jobServer.UpdateInProgressJobData(job); err != nil {
+			mlog.Error("Worker: Failed to update job data", mlog.String("worker", model.JobTypeExtractContent), mlog.String("job_id", job.Id), mlog.Err(err))
+		}
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/hosted_purchase_screening/scheduler.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package hosted_purchase_screening
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+)
+
+const schedFreq = 24 * time.Hour
+
+func MakeScheduler(jobServer *jobs.JobServer, license *model.License) model.Scheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		return model.BuildEnterpriseReady == "true" && license == nil
+	}
+	return jobs.NewPeriodicScheduler(jobServer, model.JobTypeHostedPurchaseScreening, schedFreq, isEnabled)
+}

server/channels/jobs/hosted_purchase_screening/worker.go

@@ -0,0 +1,49 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package hosted_purchase_screening
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+)
+
+const (
+	JobName = "HostedPurchaseScreening"
+	// 3 days matches the expecation given in portal purchase flow.
+	waitForScreeningDuration = 3 * 24 * time.Hour
+)
+
+type ScreenTimeStore interface {
+	GetByName(string) (*model.System, error)
+	PermanentDeleteByName(name string) (*model.System, error)
+}
+
+func MakeWorker(jobServer *jobs.JobServer, license *model.License, screenTimeStore ScreenTimeStore) model.Worker {
+	isEnabled := func(_ *model.Config) bool {
+		return !license.IsCloud()
+	}
+	execute := func(job *model.Job) error {
+		defer jobServer.HandleJobPanic(job)
+
+		now := time.Now()
+		screenTimeValue, err := screenTimeStore.GetByName(model.SystemHostedPurchaseNeedsScreening)
+		if err != nil {
+			return err
+		}
+		screenTime, err := strconv.ParseInt(screenTimeValue.Value, 10, 64)
+		if err != nil {
+			return err
+		}
+
+		if now.After(time.UnixMilli(screenTime).Add(waitForScreeningDuration)) {
+			screenTimeStore.PermanentDeleteByName(model.SystemHostedPurchaseNeedsScreening)
+		}
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(JobName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/import_delete/scheduler.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package import_delete
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+)
+
+const schedFreq = 24 * time.Hour
+
+func MakeScheduler(jobServer *jobs.JobServer) model.Scheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.ImportSettings.Directory != "" && *cfg.ImportSettings.RetentionDays > 0
+	}
+	return jobs.NewPeriodicScheduler(jobServer, model.JobTypeImportDelete, schedFreq, isEnabled)
+}

server/channels/jobs/import_delete/worker.go

@@ -0,0 +1,105 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package import_delete
+
+import (
+	"errors"
+	"path/filepath"
+	"time"
+
+	"github.com/wiggin77/merror"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/configservice"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const jobName = "ImportDelete"
+
+type AppIface interface {
+	configservice.ConfigService
+	ListDirectory(path string) ([]string, *model.AppError)
+	FileModTime(path string) (time.Time, *model.AppError)
+	RemoveFile(path string) *model.AppError
+}
+
+func MakeWorker(jobServer *jobs.JobServer, app AppIface, s store.Store) model.Worker {
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.ImportSettings.Directory != "" && *cfg.ImportSettings.RetentionDays > 0
+	}
+	execute := func(job *model.Job) error {
+		defer jobServer.HandleJobPanic(job)
+
+		importPath := *app.Config().ImportSettings.Directory
+		retentionTime := time.Duration(*app.Config().ImportSettings.RetentionDays) * 24 * time.Hour
+		imports, appErr := app.ListDirectory(importPath)
+		if appErr != nil {
+			return appErr
+		}
+
+		multipleErrors := merror.New()
+		for i := range imports {
+			filename := filepath.Base(imports[i])
+			modTime, appErr := app.FileModTime(filepath.Join(importPath, filename))
+			if appErr != nil {
+				mlog.Debug("Worker: Failed to get file modification time",
+					mlog.Err(appErr), mlog.String("import", imports[i]))
+				multipleErrors.Append(appErr)
+				continue
+			}
+
+			if time.Now().After(modTime.Add(retentionTime)) {
+				// expected format if uploaded through the API is
+				// ${uploadID}_${filename}${model.IncompleteUploadSuffix}
+				minLen := 26 + 1 + len(model.IncompleteUploadSuffix)
+
+				// check if it's an incomplete upload and attempt to delete its session.
+				if len(filename) > minLen && filepath.Ext(filename) == model.IncompleteUploadSuffix {
+					uploadID := filename[:26]
+					if storeErr := s.UploadSession().Delete(uploadID); storeErr != nil {
+						mlog.Debug("Worker: Failed to delete UploadSession",
+							mlog.Err(storeErr), mlog.String("upload_id", uploadID))
+						multipleErrors.Append(storeErr)
+						continue
+					}
+				} else {
+					// check if fileinfo exists and if so delete it.
+					filePath := filepath.Join(imports[i])
+					info, storeErr := s.FileInfo().GetByPath(filePath)
+					var nfErr *store.ErrNotFound
+					if storeErr != nil && !errors.As(storeErr, &nfErr) {
+						mlog.Debug("Worker: Failed to get FileInfo",
+							mlog.Err(storeErr), mlog.String("path", filePath))
+						multipleErrors.Append(storeErr)
+						continue
+					} else if storeErr == nil {
+						if storeErr = s.FileInfo().PermanentDelete(info.Id); storeErr != nil {
+							mlog.Debug("Worker: Failed to delete FileInfo",
+								mlog.Err(storeErr), mlog.String("file_id", info.Id))
+							multipleErrors.Append(storeErr)
+							continue
+						}
+					}
+				}
+
+				// remove file data from storage.
+				if appErr := app.RemoveFile(imports[i]); appErr != nil {
+					mlog.Debug("Worker: Failed to remove file",
+						mlog.Err(appErr), mlog.String("import", imports[i]))
+					multipleErrors.Append(appErr)
+					continue
+				}
+			}
+		}
+
+		if err := multipleErrors.ErrorOrNil(); err != nil {
+			mlog.Warn("Worker: errors occurred", mlog.String("job-name", jobName), mlog.Err(err))
+		}
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/import_process/worker.go

@@ -0,0 +1,118 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package import_process
+
+import (
+	"archive/zip"
+	"io"
+	"net/http"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/app/request"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/configservice"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/filestore"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const jobName = "ImportProcess"
+
+type AppIface interface {
+	configservice.ConfigService
+	RemoveFile(path string) *model.AppError
+	FileExists(path string) (bool, *model.AppError)
+	FileSize(path string) (int64, *model.AppError)
+	FileReader(path string) (filestore.ReadCloseSeeker, *model.AppError)
+	BulkImportWithPath(c *request.Context, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun bool, workers int, importPath string) (*model.AppError, int)
+	Log() *mlog.Logger
+}
+
+func MakeWorker(jobServer *jobs.JobServer, app AppIface) model.Worker {
+	appContext := request.EmptyContext(app.Log())
+	isEnabled := func(cfg *model.Config) bool {
+		return true
+	}
+	execute := func(job *model.Job) error {
+		defer jobServer.HandleJobPanic(job)
+
+		importFileName, ok := job.Data["import_file"]
+		if !ok {
+			return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.missing_file", nil, "", http.StatusBadRequest)
+		}
+
+		importFilePath := filepath.Join(*app.Config().ImportSettings.Directory, importFileName)
+		if ok, err := app.FileExists(importFilePath); err != nil {
+			return err
+		} else if !ok {
+			return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.file_exists", nil, "", http.StatusBadRequest)
+		}
+
+		importFileSize, appErr := app.FileSize(importFilePath)
+		if appErr != nil {
+			return appErr
+		}
+
+		importFile, appErr := app.FileReader(importFilePath)
+		if appErr != nil {
+			return appErr
+		}
+		defer importFile.Close()
+
+		// The import is a long running operation, try to cancel any timeouts attached to the reader.
+		type TimeoutCanceler interface{ CancelTimeout() bool }
+		if tc, ok := importFile.(TimeoutCanceler); ok {
+			if !tc.CancelTimeout() {
+				appContext.Logger().Warn("Could not cancel the timeout for the file reader. The import may fail due to a timeout.")
+			}
+		}
+
+		importZipReader, err := zip.NewReader(importFile.(io.ReaderAt), importFileSize)
+		if err != nil {
+			return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+
+		// find JSONL import file.
+		var jsonFile io.ReadCloser
+		for _, f := range importZipReader.File {
+			if filepath.Ext(f.Name) != ".jsonl" {
+				continue
+			}
+			// avoid "zip slip"
+			if strings.Contains(f.Name, "..") {
+				return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, "jsonFilePath contains path traversal", http.StatusForbidden)
+			}
+
+			jsonFile, err = f.Open()
+			if err != nil {
+				return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+
+			defer jsonFile.Close()
+			break
+		}
+
+		if jsonFile == nil {
+			return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.missing_jsonl", nil, "jsonFile was nil", http.StatusBadRequest)
+		}
+
+		// do the actual import.
+		appErr, lineNumber := app.BulkImportWithPath(appContext, jsonFile, importZipReader, false, runtime.NumCPU(), model.ExportDataDir)
+		if appErr != nil {
+			job.Data["line_number"] = strconv.Itoa(lineNumber)
+			return appErr
+		}
+
+		// remove import file when done.
+		if appErr := app.RemoveFile(importFilePath); appErr != nil {
+			return appErr
+		}
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/jobs.go

@@ -0,0 +1,299 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"runtime/pprof"
+	"strings"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const (
+	CancelWatcherPollingInterval = 5000
+)
+
+func (srv *JobServer) CreateJob(jobType string, jobData map[string]string) (*model.Job, *model.AppError) {
+	job := model.Job{
+		Id:       model.NewId(),
+		Type:     jobType,
+		CreateAt: model.GetMillis(),
+		Status:   model.JobStatusPending,
+		Data:     jobData,
+	}
+
+	if err := job.IsValid(); err != nil {
+		return nil, err
+	}
+
+	if srv.workers.Get(job.Type) == nil {
+		return nil, model.NewAppError("Job.IsValid", "model.job.is_valid.type.app_error", nil, "id="+job.Id, http.StatusBadRequest)
+	}
+
+	if _, err := srv.Store.Job().Save(&job); err != nil {
+		return nil, model.NewAppError("CreateJob", "app.job.save.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	return &job, nil
+}
+
+func (srv *JobServer) GetJob(id string) (*model.Job, *model.AppError) {
+	job, err := srv.Store.Job().Get(id)
+	if err != nil {
+		var nfErr *store.ErrNotFound
+		switch {
+		case errors.As(err, &nfErr):
+			return nil, model.NewAppError("GetJob", "app.job.get.app_error", nil, "", http.StatusNotFound).Wrap(err)
+		default:
+			return nil, model.NewAppError("GetJob", "app.job.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+	}
+
+	return job, nil
+}
+
+func (srv *JobServer) ClaimJob(job *model.Job) (bool, *model.AppError) {
+	updated, err := srv.Store.Job().UpdateStatusOptimistically(job.Id, model.JobStatusPending, model.JobStatusInProgress)
+	if err != nil {
+		return false, model.NewAppError("ClaimJob", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	if updated && srv.metrics != nil {
+		srv.metrics.IncrementJobActive(job.Type)
+	}
+
+	return updated, nil
+}
+
+func (srv *JobServer) SetJobProgress(job *model.Job, progress int64) *model.AppError {
+	job.Status = model.JobStatusInProgress
+	job.Progress = progress
+
+	if _, err := srv.Store.Job().UpdateOptimistically(job, model.JobStatusInProgress); err != nil {
+		return model.NewAppError("SetJobProgress", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+func (srv *JobServer) SetJobWarning(job *model.Job) *model.AppError {
+	if _, err := srv.Store.Job().UpdateStatus(job.Id, model.JobStatusWarning); err != nil {
+		return model.NewAppError("SetJobWarning", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+func (srv *JobServer) SetJobSuccess(job *model.Job) *model.AppError {
+	if _, err := srv.Store.Job().UpdateStatus(job.Id, model.JobStatusSuccess); err != nil {
+		return model.NewAppError("SetJobSuccess", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	if srv.metrics != nil {
+		srv.metrics.DecrementJobActive(job.Type)
+	}
+
+	return nil
+}
+
+func (srv *JobServer) SetJobError(job *model.Job, jobError *model.AppError) *model.AppError {
+	if jobError == nil {
+		_, err := srv.Store.Job().UpdateStatus(job.Id, model.JobStatusError)
+		if err != nil {
+			return model.NewAppError("SetJobError", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+
+		if srv.metrics != nil {
+			srv.metrics.DecrementJobActive(job.Type)
+		}
+
+		return nil
+	}
+
+	job.Status = model.JobStatusError
+	job.Progress = -1
+	if job.Data == nil {
+		job.Data = make(map[string]string)
+	}
+	job.Data["error"] = jobError.Message
+	if jobError.DetailedError != "" {
+		job.Data["error"] += " — " + jobError.DetailedError
+	}
+	if wrapped := jobError.Unwrap(); wrapped != nil {
+		job.Data["error"] += " — " + wrapped.Error()
+	}
+	updated, err := srv.Store.Job().UpdateOptimistically(job, model.JobStatusInProgress)
+	if err != nil {
+		return model.NewAppError("SetJobError", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	if updated && srv.metrics != nil {
+		srv.metrics.DecrementJobActive(job.Type)
+	}
+
+	if !updated {
+		updated, err = srv.Store.Job().UpdateOptimistically(job, model.JobStatusCancelRequested)
+		if err != nil {
+			return model.NewAppError("SetJobError", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+		if !updated {
+			return model.NewAppError("SetJobError", "jobs.set_job_error.update.error", nil, "id="+job.Id, http.StatusInternalServerError)
+		}
+	}
+
+	return nil
+}
+
+func (srv *JobServer) SetJobCanceled(job *model.Job) *model.AppError {
+	if _, err := srv.Store.Job().UpdateStatus(job.Id, model.JobStatusCanceled); err != nil {
+		return model.NewAppError("SetJobCanceled", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	if srv.metrics != nil {
+		srv.metrics.DecrementJobActive(job.Type)
+	}
+
+	return nil
+}
+
+func (srv *JobServer) SetJobPending(job *model.Job) *model.AppError {
+	if _, err := srv.Store.Job().UpdateStatus(job.Id, model.JobStatusPending); err != nil {
+		return model.NewAppError("SetJobPending", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	if srv.metrics != nil {
+		srv.metrics.DecrementJobActive(job.Type)
+	}
+
+	return nil
+}
+
+func (srv *JobServer) UpdateInProgressJobData(job *model.Job) *model.AppError {
+	job.Status = model.JobStatusInProgress
+	job.LastActivityAt = model.GetMillis()
+	if _, err := srv.Store.Job().UpdateOptimistically(job, model.JobStatusInProgress); err != nil {
+		return model.NewAppError("UpdateInProgressJobData", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+// HandleJobPanic is used to handle panics during the execution of a job. It logs the panic and sets the status for the job.
+// After handling, the method repanics! This method is supposed to be `defer`'d at the start of the job.
+func (srv *JobServer) HandleJobPanic(job *model.Job) {
+	r := recover()
+	if r == nil {
+		return
+	}
+
+	sb := &strings.Builder{}
+	pprof.Lookup("goroutine").WriteTo(sb, 2)
+	mlog.Error("Unhandled panic in job", mlog.Any("panic", r), mlog.Any("job", job), mlog.String("stack", sb.String()))
+
+	rerr, ok := r.(error)
+	if !ok {
+		rerr = fmt.Errorf("job panic: %v", r)
+	}
+
+	appErr := srv.SetJobError(job, model.NewAppError("HandleJobPanic", "app.job.update.app_error", nil, "", http.StatusInternalServerError)).Wrap(rerr)
+	if appErr != nil {
+		mlog.Error("Failed to set the job status to 'failed'", mlog.Err(appErr), mlog.Any("job", job))
+	}
+
+	panic(r)
+}
+
+func (srv *JobServer) RequestCancellation(jobId string) *model.AppError {
+	updated, err := srv.Store.Job().UpdateStatusOptimistically(jobId, model.JobStatusPending, model.JobStatusCanceled)
+	if err != nil {
+		return model.NewAppError("RequestCancellation", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	if updated {
+		if srv.metrics != nil {
+			job, err := srv.GetJob(jobId)
+			if err != nil {
+				return model.NewAppError("RequestCancellation", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+
+			srv.metrics.DecrementJobActive(job.Type)
+		}
+
+		return nil
+	}
+
+	updated, err = srv.Store.Job().UpdateStatusOptimistically(jobId, model.JobStatusInProgress, model.JobStatusCancelRequested)
+	if err != nil {
+		return model.NewAppError("RequestCancellation", "app.job.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	if updated {
+		return nil
+	}
+
+	return model.NewAppError("RequestCancellation", "jobs.request_cancellation.status.error", nil, "id="+jobId, http.StatusInternalServerError)
+}
+
+func (srv *JobServer) CancellationWatcher(ctx context.Context, jobId string, cancelChan chan struct{}) {
+	for {
+		select {
+		case <-ctx.Done():
+			mlog.Debug("CancellationWatcher for Job Aborting as job has finished.", mlog.String("job_id", jobId))
+			return
+		case <-time.After(CancelWatcherPollingInterval * time.Millisecond):
+			mlog.Debug("CancellationWatcher for Job started polling.", mlog.String("job_id", jobId))
+			jobStatus, err := srv.Store.Job().Get(jobId)
+			if err != nil {
+				mlog.Warn("Error getting job", mlog.String("job_id", jobId), mlog.Err(err))
+				continue
+			}
+			if jobStatus.Status == model.JobStatusCancelRequested {
+				close(cancelChan)
+				return
+			}
+		}
+	}
+}
+
+func GenerateNextStartDateTime(now time.Time, nextStartTime time.Time) *time.Time {
+	nextTime := time.Date(now.Year(), now.Month(), now.Day(), nextStartTime.Hour(), nextStartTime.Minute(), 0, 0, time.Local)
+
+	if !now.Before(nextTime) {
+		nextTime = nextTime.AddDate(0, 0, 1)
+	}
+
+	return &nextTime
+}
+
+func (srv *JobServer) CheckForPendingJobsByType(jobType string) (bool, *model.AppError) {
+	count, err := srv.Store.Job().GetCountByStatusAndType(model.JobStatusPending, jobType)
+	if err != nil {
+		return false, model.NewAppError("CheckForPendingJobsByType", "app.job.get_count_by_status_and_type.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return count > 0, nil
+}
+
+func (srv *JobServer) GetJobsByTypeAndStatus(jobType string, status string) ([]*model.Job, *model.AppError) {
+	jobs, err := srv.Store.Job().GetAllByTypeAndStatus(jobType, status)
+	if err != nil {
+		return nil, model.NewAppError("GetJobsByTypeAndStatus", "app.job.get_all_jobs_by_type_and_status.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	return jobs, nil
+}
+
+func (srv *JobServer) GetLastSuccessfulJobByType(jobType string) (*model.Job, *model.AppError) {
+	statuses := []string{model.JobStatusSuccess}
+	if jobType == model.JobTypeMessageExport {
+		statuses = []string{model.JobStatusWarning, model.JobStatusSuccess}
+	}
+	job, err := srv.Store.Job().GetNewestJobByStatusesAndType(statuses, jobType)
+	var nfErr *store.ErrNotFound
+	if err != nil && !errors.As(err, &nfErr) {
+		return nil, model.NewAppError("GetLastSuccessfulJobByType", "app.job.get_newest_job_by_status_and_type.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return job, nil
+}

server/channels/jobs/jobs_test.go

@@ -0,0 +1,631 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/einterfaces/mocks"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store/storetest"
+	"github.com/mattermost/mattermost-server/v6/server/channels/utils/testutils"
+)
+
+func makeJobServer(t *testing.T) (*JobServer, *storetest.Store, *mocks.MetricsInterface) {
+	configService := &testutils.StaticConfigService{}
+
+	mockStore := &storetest.Store{}
+	t.Cleanup(func() {
+		mockStore.AssertExpectations(t)
+	})
+
+	mockMetrics := &mocks.MetricsInterface{}
+	t.Cleanup(func() {
+		mockMetrics.AssertExpectations(t)
+	})
+
+	jobServer := &JobServer{
+		ConfigService: configService,
+		Store:         mockStore,
+		metrics:       mockMetrics,
+	}
+
+	return jobServer, mockStore, mockMetrics
+}
+
+func expectErrorId(t *testing.T, errId string, appErr *model.AppError) {
+	t.Helper()
+	require.NotNil(t, appErr)
+	require.Equal(t, errId, appErr.Id)
+}
+
+func makeTeamEditionJobServer(t *testing.T) (*JobServer, *storetest.Store) {
+	configService := &testutils.StaticConfigService{}
+
+	mockStore := &storetest.Store{}
+	t.Cleanup(func() {
+		mockStore.AssertExpectations(t)
+	})
+
+	jobServer := NewJobServer(configService, mockStore, nil)
+
+	return jobServer, mockStore
+}
+
+func TestClaimJob(t *testing.T) {
+	t.Run("error claiming job", func(t *testing.T) {
+		jobServer, mockStore, _ := makeJobServer(t)
+
+		job := &model.Job{
+			Id:   "job_id",
+			Type: "job_type",
+		}
+
+		mockStore.JobStore.On("UpdateStatusOptimistically", "job_id", model.JobStatusPending, model.JobStatusInProgress).Return(false, &model.AppError{Message: "message"})
+
+		updated, err := jobServer.ClaimJob(job)
+		expectErrorId(t, "app.job.update.app_error", err)
+		require.False(t, updated)
+	})
+
+	t.Run("no existing job to update", func(t *testing.T) {
+		jobServer, mockStore, _ := makeJobServer(t)
+
+		job := &model.Job{
+			Id:   "job_id",
+			Type: "job_type",
+		}
+
+		mockStore.JobStore.On("UpdateStatusOptimistically", "job_id", model.JobStatusPending, model.JobStatusInProgress).Return(false, nil)
+
+		updated, err := jobServer.ClaimJob(job)
+		require.Nil(t, err)
+		require.False(t, updated)
+	})
+
+	t.Run("pending job updated", func(t *testing.T) {
+		jobServer, mockStore, mockMetrics := makeJobServer(t)
+
+		job := &model.Job{
+			Id:   "job_id",
+			Type: "job_type",
+		}
+
+		mockStore.JobStore.On("UpdateStatusOptimistically", "job_id", model.JobStatusPending, model.JobStatusInProgress).Return(true, nil)
+		mockMetrics.On("IncrementJobActive", "job_type")
+
+		updated, err := jobServer.ClaimJob(job)
+		require.Nil(t, err)
+		require.True(t, updated)
+	})
+
+	t.Run("pending job updated, nil metrics service", func(t *testing.T) {
+		jobServer, mockStore := makeTeamEditionJobServer(t)
+
+		job := &model.Job{
+			Id:   "job_id",
+			Type: "job_type",
+		}
+
+		mockStore.JobStore.On("UpdateStatusOptimistically", "job_id", model.JobStatusPending, model.JobStatusInProgress).Return(true, nil)
+
+		updated, err := jobServer.ClaimJob(job)
+		require.Nil(t, err)
+		require.True(t, updated)
+	})
+}
+
+func TestSetJobProgress(t *testing.T) {
+	t.Run("error setting progress", func(t *testing.T) {
+		jobServer, mockStore, _ := makeJobServer(t)
+
+		progress := int64(50)
+		job := &model.Job{
+			Id:   "job_id",
+			Type: "job_type",
+		}
+
+		job.Status = model.JobStatusInProgress
+		job.Progress = progress
+
+		mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusInProgress).Return(false, &model.AppError{Message: "message"})
+
+		err := jobServer.SetJobProgress(job, progress)
+		expectErrorId(t, "app.job.update.app_error", err)
+	})
+
+	t.Run("progress updated", func(t *testing.T) {
+		jobServer, mockStore, _ := makeJobServer(t)
+
+		progress := int64(50)
+		job := &model.Job{
+			Id:   "job_id",
+			Type: "job_type",
+		}
+
+		job.Status = model.JobStatusInProgress
+		job.Progress = progress
+
+		mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusInProgress).Return(true, nil)
+
+		err := jobServer.SetJobProgress(job, progress)
+		require.Nil(t, err)
+	})
+}
+
+func TestSetJobWarning(t *testing.T) {
+	t.Run("error setting status", func(t *testing.T) {
+		jobServer, mockStore, _ := makeJobServer(t)
+
+		job := &model.Job{
+			Id:   "job_id",
+			Type: "job_type",
+		}
+
+		mockStore.JobStore.On("UpdateStatus", "job_id", model.JobStatusWarning).Return(job, &model.AppError{Message: "message"})
+
+		err := jobServer.SetJobWarning(job)
+		expectErrorId(t, "app.job.update.app_error", err)
+	})
+
+	t.Run("status updated", func(t *testing.T) {
+		jobServer, mockStore, _ := makeJobServer(t)
+
+		job := &model.Job{
+			Id:   "job_id",
+			Type: "job_type",
+		}
+
+		mockStore.JobStore.On("UpdateStatus", "job_id", model.JobStatusWarning).Return(job, nil)
+
+		err := jobServer.SetJobWarning(job)
+		require.Nil(t, err)
+	})
+}
+
+func TestSetJobSuccess(t *testing.T) {
+	t.Run("error setting status", func(t *testing.T) {
+		jobServer, mockStore, _ := makeJobServer(t)
+
+		job := &model.Job{
+			Id:   "job_id",
+			Type: "job_type",
+		}
+
+		mockStore.JobStore.On("UpdateStatus", "job_id", model.JobStatusSuccess).Return(job, &model.AppError{Message: "message"})
+
+		err := jobServer.SetJobSuccess(job)
+		expectErrorId(t, "app.job.update.app_error", err)
+	})
+
+	t.Run("status updated", func(t *testing.T) {
+		jobServer, mockStore, mockMetrics := makeJobServer(t)
+
+		job := &model.Job{
+			Id:   "job_id",
+			Type: "job_type",
+		}
+
+		mockStore.JobStore.On("UpdateStatus", "job_id", model.JobStatusSuccess).Return(job, nil)
+		mockMetrics.On("DecrementJobActive", "job_type")
+
+		err := jobServer.SetJobSuccess(job)
+		require.Nil(t, err)
+	})
+
+	t.Run("status updated, nil metrics service", func(t *testing.T) {
+		jobServer, mockStore := makeTeamEditionJobServer(t)
+
+		job := &model.Job{
+			Id:   "job_id",
+			Type: "job_type",
+		}
+
+		mockStore.JobStore.On("UpdateStatus", "job_id", model.JobStatusSuccess).Return(job, nil)
+
+		err := jobServer.SetJobSuccess(job)
+		require.Nil(t, err)
+	})
+}
+
+func TestSetJobError(t *testing.T) {
+	t.Run("nil provided job error", func(t *testing.T) {
+		t.Run("error setting status", func(t *testing.T) {
+			jobServer, mockStore, _ := makeJobServer(t)
+
+			job := &model.Job{
+				Id:   "job_id",
+				Type: "job_type",
+			}
+
+			mockStore.JobStore.On("UpdateStatus", "job_id", model.JobStatusError).Return(job, &model.AppError{Message: "message"})
+
+			err := jobServer.SetJobError(job, nil)
+			expectErrorId(t, "app.job.update.app_error", err)
+		})
+
+		t.Run("status updated", func(t *testing.T) {
+			jobServer, mockStore, mockMetrics := makeJobServer(t)
+
+			job := &model.Job{
+				Id:   "job_id",
+				Type: "job_type",
+			}
+
+			mockStore.JobStore.On("UpdateStatus", "job_id", model.JobStatusError).Return(job, nil)
+			mockMetrics.On("DecrementJobActive", "job_type")
+
+			err := jobServer.SetJobError(job, nil)
+			require.Nil(t, err)
+		})
+
+		t.Run("status updated, nil metrics service", func(t *testing.T) {
+			jobServer, mockStore := makeTeamEditionJobServer(t)
+
+			job := &model.Job{
+				Id:   "job_id",
+				Type: "job_type",
+			}
+
+			mockStore.JobStore.On("UpdateStatus", "job_id", model.JobStatusError).Return(job, nil)
+
+			err := jobServer.SetJobError(job, nil)
+			require.Nil(t, err)
+		})
+	})
+
+	t.Run("provided job error", func(t *testing.T) {
+		t.Run("error setting status", func(t *testing.T) {
+			jobServer, mockStore, _ := makeJobServer(t)
+
+			jobError := &model.AppError{Message: "message"}
+
+			job := &model.Job{
+				Id:       "job_id",
+				Type:     "job_type",
+				Progress: -1,
+				Data:     map[string]string{"error": jobError.Message},
+			}
+
+			mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusInProgress).Return(false, &model.AppError{Message: "message"})
+
+			err := jobServer.SetJobError(job, jobError)
+			expectErrorId(t, "app.job.update.app_error", err)
+		})
+
+		t.Run("status updated", func(t *testing.T) {
+			jobServer, mockStore, mockMetrics := makeJobServer(t)
+
+			jobError := &model.AppError{Message: "message"}
+
+			job := &model.Job{
+				Id:       "job_id",
+				Type:     "job_type",
+				Progress: -1,
+				Data:     map[string]string{"error": jobError.Message},
+			}
+
+			mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusInProgress).Return(true, nil)
+			mockMetrics.On("DecrementJobActive", "job_type")
+
+			err := jobServer.SetJobError(job, jobError)
+			require.Nil(t, err)
+		})
+
+		t.Run("status updated, nil metrics service", func(t *testing.T) {
+			jobServer, mockStore := makeTeamEditionJobServer(t)
+
+			jobError := &model.AppError{Message: "message"}
+
+			job := &model.Job{
+				Id:       "job_id",
+				Type:     "job_type",
+				Progress: -1,
+				Data:     map[string]string{"error": jobError.Message},
+			}
+
+			mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusInProgress).Return(true, nil)
+
+			err := jobServer.SetJobError(job, jobError)
+			require.Nil(t, err)
+		})
+
+		t.Run("status not updated, request cancellation, error setting status", func(t *testing.T) {
+			jobServer, mockStore, _ := makeJobServer(t)
+
+			jobError := &model.AppError{Message: "message"}
+
+			job := &model.Job{
+				Id:       "job_id",
+				Type:     "job_type",
+				Progress: -1,
+				Data:     map[string]string{"error": jobError.Message},
+			}
+
+			mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusInProgress).Return(false, nil)
+			mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusCancelRequested).Return(false, &model.AppError{Message: "message"})
+
+			err := jobServer.SetJobError(job, jobError)
+			expectErrorId(t, "app.job.update.app_error", err)
+		})
+
+		t.Run("status not updated, request cancellation, status not updated", func(t *testing.T) {
+			jobServer, mockStore, _ := makeJobServer(t)
+
+			jobError := &model.AppError{Message: "message"}
+
+			job := &model.Job{
+				Id:       "job_id",
+				Type:     "job_type",
+				Progress: -1,
+				Data:     map[string]string{"error": jobError.Message},
+			}
+
+			mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusInProgress).Return(false, nil)
+			mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusCancelRequested).Return(false, nil)
+
+			err := jobServer.SetJobError(job, jobError)
+			expectErrorId(t, "jobs.set_job_error.update.error", err)
+		})
+
+		t.Run("status not updated, request cancellation, status updated", func(t *testing.T) {
+			jobServer, mockStore, _ := makeJobServer(t)
+
+			jobError := &model.AppError{Message: "message"}
+
+			job := &model.Job{
+				Id:       "job_id",
+				Type:     "job_type",
+				Progress: -1,
+				Data:     map[string]string{"error": jobError.Message},
+			}
+
+			mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusInProgress).Return(false, nil)
+			mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusCancelRequested).Return(true, nil)
+
+			err := jobServer.SetJobError(job, jobError)
+			require.Nil(t, err)
+		})
+
+		t.Run("error message set correctly", func(t *testing.T) {
+			jobServer, mockStore, _ := makeJobServer(t)
+
+			jobError := model.NewAppError("anywhere", "not.a.valid.id", nil, "details", http.StatusTeapot).Wrap(errors.New("wrapped"))
+
+			job := &model.Job{
+				Id:       "job_id",
+				Type:     "job_type",
+				Progress: -1,
+				Data:     map[string]string{},
+			}
+
+			mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusInProgress).Return(false, nil)
+			mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusCancelRequested).Return(true, nil)
+
+			err := jobServer.SetJobError(job, jobError)
+			require.Nil(t, err)
+			require.Equal(t, "not.a.valid.id — details — wrapped", job.Data["error"])
+		})
+	})
+}
+
+func TestSetJobCanceled(t *testing.T) {
+	t.Run("error setting status", func(t *testing.T) {
+		jobServer, mockStore, _ := makeJobServer(t)
+
+		job := &model.Job{
+			Id:   "job_id",
+			Type: "job_type",
+		}
+
+		mockStore.JobStore.On("UpdateStatus", "job_id", model.JobStatusCanceled).Return(job, &model.AppError{Message: "message"})
+
+		err := jobServer.SetJobCanceled(job)
+		expectErrorId(t, "app.job.update.app_error", err)
+	})
+
+	t.Run("status updated", func(t *testing.T) {
+		jobServer, mockStore, mockMetrics := makeJobServer(t)
+
+		job := &model.Job{
+			Id:   "job_id",
+			Type: "job_type",
+		}
+
+		mockStore.JobStore.On("UpdateStatus", "job_id", model.JobStatusCanceled).Return(job, nil)
+		mockMetrics.On("DecrementJobActive", "job_type")
+
+		err := jobServer.SetJobCanceled(job)
+		require.Nil(t, err)
+	})
+
+	t.Run("status updated, nil metrics service", func(t *testing.T) {
+		jobServer, mockStore := makeTeamEditionJobServer(t)
+
+		job := &model.Job{
+			Id:   "job_id",
+			Type: "job_type",
+		}
+
+		mockStore.JobStore.On("UpdateStatus", "job_id", model.JobStatusCanceled).Return(job, nil)
+
+		err := jobServer.SetJobCanceled(job)
+		require.Nil(t, err)
+	})
+}
+
+func TestUpdateInProgressJobData(t *testing.T) {
+	t.Run("error updating", func(t *testing.T) {
+		jobServer, mockStore, _ := makeJobServer(t)
+
+		job := &model.Job{
+			Id:   "job_id",
+			Type: "job_type",
+		}
+
+		job.Status = model.JobStatusInProgress
+
+		mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusInProgress).Return(false, &model.AppError{Message: "message"})
+
+		err := jobServer.UpdateInProgressJobData(job)
+		expectErrorId(t, "app.job.update.app_error", err)
+	})
+
+	t.Run("progress updated", func(t *testing.T) {
+		jobServer, mockStore, _ := makeJobServer(t)
+
+		job := &model.Job{
+			Id:   "job_id",
+			Type: "job_type",
+		}
+
+		job.Status = model.JobStatusInProgress
+
+		mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusInProgress).Return(true, nil)
+
+		err := jobServer.UpdateInProgressJobData(job)
+		require.Nil(t, err)
+	})
+}
+
+func TestHandleJobPanic(t *testing.T) {
+	t.Run("no panic", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+
+		job := &model.Job{
+			Type:   model.JobTypeImportProcess,
+			Status: model.JobStatusInProgress,
+		}
+
+		f := func() {
+			defer jobServer.HandleJobPanic(job)
+			fmt.Println("OK")
+		}
+
+		require.NotPanics(t, f)
+		require.Equal(t, model.JobStatusInProgress, job.Status)
+	})
+
+	t.Run("with panic string", func(t *testing.T) {
+		jobServer, mockStore, metrics := makeJobServer(t)
+
+		job := &model.Job{
+			Type:   model.JobTypeImportProcess,
+			Status: model.JobStatusInProgress,
+		}
+
+		f := func() {
+			defer jobServer.HandleJobPanic(job)
+			panic("not OK")
+		}
+
+		mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusInProgress).Return(true, nil)
+		metrics.On("DecrementJobActive", model.JobTypeImportProcess)
+
+		require.Panics(t, f)
+		require.Equal(t, model.JobStatusError, job.Status)
+	})
+
+	t.Run("with panic error", func(t *testing.T) {
+		jobServer, mockStore, metrics := makeJobServer(t)
+
+		job := &model.Job{
+			Type:   model.JobTypeImportProcess,
+			Status: model.JobStatusInProgress,
+		}
+
+		f := func() {
+			defer jobServer.HandleJobPanic(job)
+			panic(fmt.Errorf("not OK"))
+		}
+
+		mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusInProgress).Return(true, nil)
+		metrics.On("DecrementJobActive", model.JobTypeImportProcess)
+
+		require.Panics(t, f)
+		require.Equal(t, model.JobStatusError, job.Status)
+	})
+}
+
+func TestRequestCancellation(t *testing.T) {
+	t.Run("error cancelling", func(t *testing.T) {
+		jobServer, mockStore, _ := makeJobServer(t)
+
+		mockStore.JobStore.On("UpdateStatusOptimistically", "job_id", model.JobStatusPending, model.JobStatusCanceled).Return(false, &model.AppError{Message: "message"})
+
+		err := jobServer.RequestCancellation("job_id")
+		expectErrorId(t, "app.job.update.app_error", err)
+	})
+
+	t.Run("cancelled, job not found", func(t *testing.T) {
+		jobServer, mockStore, _ := makeJobServer(t)
+
+		mockStore.JobStore.On("UpdateStatusOptimistically", "job_id", model.JobStatusPending, model.JobStatusCanceled).Return(true, nil)
+		mockStore.JobStore.On("Get", "job_id").Return(nil, &store.ErrNotFound{})
+
+		err := jobServer.RequestCancellation("job_id")
+		expectErrorId(t, "app.job.update.app_error", err)
+	})
+
+	t.Run("cancelled, success", func(t *testing.T) {
+		jobServer, mockStore, mockMetrics := makeJobServer(t)
+
+		job := &model.Job{
+			Id:   "job_id",
+			Type: "job_type",
+		}
+
+		mockStore.JobStore.On("UpdateStatusOptimistically", "job_id", model.JobStatusPending, model.JobStatusCanceled).Return(true, nil)
+		mockStore.JobStore.On("Get", "job_id").Return(job, nil)
+		mockMetrics.On("DecrementJobActive", "job_type")
+
+		err := jobServer.RequestCancellation("job_id")
+		require.Nil(t, err)
+	})
+
+	t.Run("cancelled, success, nil metrics service", func(t *testing.T) {
+		jobServer, mockStore := makeTeamEditionJobServer(t)
+
+		mockStore.JobStore.On("UpdateStatusOptimistically", "job_id", model.JobStatusPending, model.JobStatusCanceled).Return(true, nil)
+
+		err := jobServer.RequestCancellation("job_id")
+		require.Nil(t, err)
+	})
+
+	t.Run("unable to cancel, requesting cancellation instead, error setting status", func(t *testing.T) {
+		jobServer, mockStore, _ := makeJobServer(t)
+
+		mockStore.JobStore.On("UpdateStatusOptimistically", "job_id", model.JobStatusPending, model.JobStatusCanceled).Return(false, nil)
+		mockStore.JobStore.On("UpdateStatusOptimistically", "job_id", model.JobStatusInProgress, model.JobStatusCancelRequested).Return(false, &model.AppError{Message: "message"})
+
+		err := jobServer.RequestCancellation("job_id")
+		expectErrorId(t, "app.job.update.app_error", err)
+	})
+
+	t.Run("unable to cancel, requesting cancellation instead, success", func(t *testing.T) {
+		jobServer, mockStore, _ := makeJobServer(t)
+
+		mockStore.JobStore.On("UpdateStatusOptimistically", "job_id", model.JobStatusPending, model.JobStatusCanceled).Return(false, nil)
+		mockStore.JobStore.On("UpdateStatusOptimistically", "job_id", model.JobStatusInProgress, model.JobStatusCancelRequested).Return(true, nil)
+
+		err := jobServer.RequestCancellation("job_id")
+		require.Nil(t, err)
+	})
+
+	t.Run("unable to cancel, requesting cancellation instead, unexpected state", func(t *testing.T) {
+		jobServer, mockStore, _ := makeJobServer(t)
+
+		mockStore.JobStore.On("UpdateStatusOptimistically", "job_id", model.JobStatusPending, model.JobStatusCanceled).Return(false, nil)
+		mockStore.JobStore.On("UpdateStatusOptimistically", "job_id", model.JobStatusInProgress, model.JobStatusCancelRequested).Return(false, nil)
+
+		err := jobServer.RequestCancellation("job_id")
+		expectErrorId(t, "jobs.request_cancellation.status.error", err)
+	})
+}

server/channels/jobs/jobs_watcher.go

@@ -0,0 +1,85 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs
+
+import (
+	"math/rand"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+// Default polling interval for jobs termination.
+// (Defining as `var` rather than `const` allows tests to lower the interval.)
+var DefaultWatcherPollingInterval = 15000
+
+type Watcher struct {
+	srv     *JobServer
+	workers *Workers
+
+	stop            chan struct{}
+	stopped         chan struct{}
+	pollingInterval int
+}
+
+func (srv *JobServer) MakeWatcher(workers *Workers, pollingInterval int) *Watcher {
+	return &Watcher{
+		stop:            make(chan struct{}),
+		stopped:         make(chan struct{}),
+		pollingInterval: pollingInterval,
+		workers:         workers,
+		srv:             srv,
+	}
+}
+
+func (watcher *Watcher) Start() {
+	mlog.Debug("Watcher Started")
+	// Delay for some random number of milliseconds before starting to ensure that multiple
+	// instances of the jobserver  don't poll at a time too close to each other.
+	rand.Seed(time.Now().UTC().UnixNano())
+	<-time.After(time.Duration(rand.Intn(watcher.pollingInterval)) * time.Millisecond)
+
+	defer func() {
+		mlog.Debug("Watcher Finished")
+		close(watcher.stopped)
+	}()
+
+	for {
+		select {
+		case <-watcher.stop:
+			mlog.Debug("Watcher: Received stop signal")
+			return
+		case <-time.After(time.Duration(watcher.pollingInterval) * time.Millisecond):
+			watcher.PollAndNotify()
+		}
+	}
+}
+
+func (watcher *Watcher) Stop() {
+	mlog.Debug("Watcher Stopping")
+	close(watcher.stop)
+	<-watcher.stopped
+
+	watcher.stop = make(chan struct{})
+	watcher.stopped = make(chan struct{})
+}
+
+func (watcher *Watcher) PollAndNotify() {
+	jobs, err := watcher.srv.Store.Job().GetAllByStatus(model.JobStatusPending)
+	if err != nil {
+		mlog.Error("Error occurred getting all pending statuses.", mlog.Err(err))
+		return
+	}
+
+	for _, job := range jobs {
+		worker := watcher.workers.Get(job.Type)
+		if worker != nil {
+			select {
+			case worker.JobChannel() <- *job:
+			default:
+			}
+		}
+	}
+}

server/channels/jobs/last_accessible_file/scheduler.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package last_accessible_file
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const schedFreq = 2 * time.Hour
+
+func MakeScheduler(jobServer *jobs.JobServer, license *model.License) model.Scheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		enabled := license != nil && *license.Features.Cloud
+		mlog.Debug("Scheduler: isEnabled: "+strconv.FormatBool(enabled), mlog.String("scheduler", model.JobTypeLastAccessibleFile))
+		return enabled
+	}
+	return jobs.NewPeriodicScheduler(jobServer, model.JobTypeLastAccessibleFile, schedFreq, isEnabled)
+}

server/channels/jobs/last_accessible_file/worker.go

@@ -0,0 +1,30 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package last_accessible_file
+
+import (
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+)
+
+const (
+	JobName = "LastAccessibleFile"
+)
+
+type AppIface interface {
+	ComputeLastAccessibleFileTime() error
+}
+
+func MakeWorker(jobServer *jobs.JobServer, license *model.License, app AppIface) model.Worker {
+	isEnabled := func(_ *model.Config) bool {
+		return license != nil && *license.Features.Cloud
+	}
+	execute := func(job *model.Job) error {
+		defer jobServer.HandleJobPanic(job)
+
+		return app.ComputeLastAccessibleFileTime()
+	}
+	worker := jobs.NewSimpleWorker(JobName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/last_accessible_post/scheduler.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package last_accessible_post
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const schedFreq = 30 * time.Minute
+
+func MakeScheduler(jobServer *jobs.JobServer, license *model.License) model.Scheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		enabled := license != nil && *license.Features.Cloud
+		mlog.Debug("Scheduler: isEnabled: "+strconv.FormatBool(enabled), mlog.String("scheduler", model.JobTypeLastAccessiblePost))
+		return enabled
+	}
+	return jobs.NewPeriodicScheduler(jobServer, model.JobTypeLastAccessiblePost, schedFreq, isEnabled)
+}

server/channels/jobs/last_accessible_post/worker.go

@@ -0,0 +1,30 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package last_accessible_post
+
+import (
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+)
+
+const (
+	JobName = "LastAccessiblePost"
+)
+
+type AppIface interface {
+	ComputeLastAccessiblePostTime() error
+}
+
+func MakeWorker(jobServer *jobs.JobServer, license *model.License, app AppIface) model.Worker {
+	isEnabled := func(_ *model.Config) bool {
+		return license != nil && license.Features != nil && *license.Features.Cloud
+	}
+	execute := func(job *model.Job) error {
+		defer jobServer.HandleJobPanic(job)
+
+		return app.ComputeLastAccessiblePostTime()
+	}
+	worker := jobs.NewSimpleWorker(JobName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/migrations/advanced_permissions_phase_2.go

@@ -0,0 +1,112 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+type AdvancedPermissionsPhase2Progress struct {
+	CurrentTable  string `json:"current_table"`
+	LastTeamId    string `json:"last_team_id"`
+	LastChannelId string `json:"last_channel_id"`
+	LastUserId    string `json:"last_user"`
+}
+
+func (p *AdvancedPermissionsPhase2Progress) ToJSON() string {
+	b, _ := json.Marshal(p)
+	return string(b)
+}
+
+func AdvancedPermissionsPhase2ProgressFromJSON(data io.Reader) *AdvancedPermissionsPhase2Progress {
+	var o *AdvancedPermissionsPhase2Progress
+	err := json.NewDecoder(data).Decode(&o)
+	if err != nil {
+		mlog.Warn("Error decoding advanced permissions phase 2 progress", mlog.Err(err))
+	}
+	return o
+}
+
+func (p *AdvancedPermissionsPhase2Progress) IsValid() bool {
+	if !model.IsValidId(p.LastChannelId) {
+		return false
+	}
+
+	if !model.IsValidId(p.LastTeamId) {
+		return false
+	}
+
+	if !model.IsValidId(p.LastUserId) {
+		return false
+	}
+
+	switch p.CurrentTable {
+	case "TeamMembers":
+	case "ChannelMembers":
+	default:
+		return false
+	}
+
+	return true
+}
+
+func (worker *Worker) runAdvancedPermissionsPhase2Migration(lastDone string) (bool, string, *model.AppError) {
+	var progress *AdvancedPermissionsPhase2Progress
+	if lastDone == "" {
+		// Haven't started the migration yet.
+		progress = &AdvancedPermissionsPhase2Progress{
+			CurrentTable:  "TeamMembers",
+			LastChannelId: strings.Repeat("0", 26),
+			LastTeamId:    strings.Repeat("0", 26),
+			LastUserId:    strings.Repeat("0", 26),
+		}
+	} else {
+		err := json.NewDecoder(strings.NewReader(lastDone)).Decode(&progress)
+		if err != nil {
+			return false, "", model.NewAppError("MigrationsWorker.runAdvancedPermissionsPhase2Migration", "migrations.worker.run_advanced_permissions_phase_2_migration.invalid_progress", map[string]any{"lastDone": lastDone}, "", http.StatusInternalServerError).Wrap(err)
+		}
+		if !progress.IsValid() {
+			return false, "", model.NewAppError("MigrationsWorker.runAdvancedPermissionsPhase2Migration", "migrations.worker.run_advanced_permissions_phase_2_migration.invalid_progress", map[string]any{"progress": progress.ToJSON()}, "", http.StatusInternalServerError)
+		}
+	}
+
+	if progress.CurrentTable == "TeamMembers" {
+		// Run a TeamMembers migration batch.
+		result, err := worker.store.Team().MigrateTeamMembers(progress.LastTeamId, progress.LastUserId)
+		if err != nil {
+			return false, progress.ToJSON(), model.NewAppError("MigrationsWorker.runAdvancedPermissionsPhase2Migration", "app.team.migrate_team_members.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+		if result == nil {
+			// We haven't progressed. That means that we've reached the end of this stage of the migration, and should now advance to the next stage.
+			progress.LastUserId = strings.Repeat("0", 26)
+			progress.CurrentTable = "ChannelMembers"
+			return false, progress.ToJSON(), nil
+		}
+
+		progress.LastTeamId = result["TeamId"]
+		progress.LastUserId = result["UserId"]
+	} else if progress.CurrentTable == "ChannelMembers" {
+		// Run a ChannelMembers migration batch.
+		data, err := worker.store.Channel().MigrateChannelMembers(progress.LastChannelId, progress.LastUserId)
+		if err != nil {
+			return false, progress.ToJSON(), model.NewAppError("MigrationsWorker.runAdvancedPermissionsPhase2Migration", "app.channel.migrate_channel_members.select.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+		if data == nil {
+			// We haven't progressed. That means we've reached the end of this final stage of the migration.
+
+			return true, progress.ToJSON(), nil
+		}
+
+		progress.LastChannelId = data["ChannelId"]
+		progress.LastUserId = data["UserId"]
+	}
+
+	return false, progress.ToJSON(), nil
+}

server/channels/jobs/migrations/helper_test.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+)
+
+func Setup(tb testing.TB) store.Store {
+	store := mainHelper.GetStore()
+	store.DropAllTables()
+	return store
+}
+
+func deleteAllJobsByTypeAndMigrationKey(store store.Store, jobType string, migrationKey string) {
+	jobs, err := store.Job().GetAllByType(model.JobTypeMigrations)
+	if err != nil {
+		panic(err)
+	}
+
+	for _, job := range jobs {
+		if key, ok := job.Data[JobDataKeyMigration]; ok && key == migrationKey {
+			if _, err = store.Job().Delete(job.Id); err != nil {
+				panic(err)
+			}
+		}
+	}
+}

server/channels/jobs/migrations/main_test.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost-server/v6/server/channels/testlib"
+)
+
+var mainHelper *testlib.MainHelper
+
+func TestMain(m *testing.M) {
+	var options = testlib.HelperOptions{
+		EnableStore:     true,
+		EnableResources: true,
+	}
+
+	mainHelper = testlib.NewMainHelperWithOptions(&options)
+	defer mainHelper.Close()
+
+	mainHelper.Main(m)
+}

server/channels/jobs/migrations/migrations.go

@@ -0,0 +1,54 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"net/http"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+)
+
+const (
+	MigrationStateUnscheduled = "unscheduled"
+	MigrationStateInProgress  = "in_progress"
+	MigrationStateCompleted   = "completed"
+
+	JobDataKeyMigration         = "migration_key"
+	JobDataKeyMigrationLastDone = "last_done"
+)
+
+func MakeMigrationsList() []string {
+	return []string{
+		model.MigrationKeyAdvancedPermissionsPhase2,
+	}
+}
+
+func GetMigrationState(migration string, store store.Store) (string, *model.Job, *model.AppError) {
+	if _, err := store.System().GetByName(migration); err == nil {
+		return MigrationStateCompleted, nil, nil
+	}
+
+	jobs, err := store.Job().GetAllByType(model.JobTypeMigrations)
+	if err != nil {
+		return "", nil, model.NewAppError("GetMigrationState", "app.job.get_all.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	for _, job := range jobs {
+		if key, ok := job.Data[JobDataKeyMigration]; ok {
+			if key != migration {
+				continue
+			}
+
+			switch job.Status {
+			case model.JobStatusInProgress, model.JobStatusPending:
+				return MigrationStateInProgress, job, nil
+			default:
+				return MigrationStateUnscheduled, job, nil
+			}
+		}
+	}
+
+	return MigrationStateUnscheduled, nil, nil
+}

server/channels/jobs/migrations/migrations_test.go

@@ -0,0 +1,103 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+func TestGetMigrationState(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+	}
+	store := Setup(t)
+
+	migrationKey := model.NewId()
+
+	deleteAllJobsByTypeAndMigrationKey(store, model.JobTypeMigrations, migrationKey)
+
+	// Test with no job yet.
+	state, job, err := GetMigrationState(migrationKey, store)
+	assert.Nil(t, err)
+	assert.Nil(t, job)
+	assert.Equal(t, "unscheduled", state)
+
+	// Test with the system table showing the migration as done.
+	system := model.System{
+		Name:  migrationKey,
+		Value: "true",
+	}
+	nErr := store.System().Save(&system)
+	assert.NoError(t, nErr)
+
+	state, job, err = GetMigrationState(migrationKey, store)
+	assert.Nil(t, err)
+	assert.Nil(t, job)
+	assert.Equal(t, "completed", state)
+
+	_, nErr = store.System().PermanentDeleteByName(migrationKey)
+	assert.NoError(t, nErr)
+
+	// Test with a job scheduled in "pending" state.
+	j1 := &model.Job{
+		Id:       model.NewId(),
+		CreateAt: model.GetMillis(),
+		Data: map[string]string{
+			JobDataKeyMigration: migrationKey,
+		},
+		Status: model.JobStatusPending,
+		Type:   model.JobTypeMigrations,
+	}
+
+	j1, nErr = store.Job().Save(j1)
+	require.NoError(t, nErr)
+
+	state, job, err = GetMigrationState(migrationKey, store)
+	assert.Nil(t, err)
+	assert.Equal(t, j1.Id, job.Id)
+	assert.Equal(t, "in_progress", state)
+
+	// Test with a job scheduled in "in progress" state.
+	j2 := &model.Job{
+		Id:       model.NewId(),
+		CreateAt: j1.CreateAt + 1,
+		Data: map[string]string{
+			JobDataKeyMigration: migrationKey,
+		},
+		Status: model.JobStatusInProgress,
+		Type:   model.JobTypeMigrations,
+	}
+
+	j2, nErr = store.Job().Save(j2)
+	require.NoError(t, nErr)
+
+	state, job, err = GetMigrationState(migrationKey, store)
+	assert.Nil(t, err)
+	assert.Equal(t, j2.Id, job.Id)
+	assert.Equal(t, "in_progress", state)
+
+	// Test with a job scheduled in "error" state.
+	j3 := &model.Job{
+		Id:       model.NewId(),
+		CreateAt: j2.CreateAt + 1,
+		Data: map[string]string{
+			JobDataKeyMigration: migrationKey,
+		},
+		Status: model.JobStatusError,
+		Type:   model.JobTypeMigrations,
+	}
+
+	j3, nErr = store.Job().Save(j3)
+	require.NoError(t, nErr)
+
+	state, job, err = GetMigrationState(migrationKey, store)
+	assert.Nil(t, err)
+	assert.Equal(t, j3.Id, job.Id)
+	assert.Equal(t, "unscheduled", state)
+}

server/channels/jobs/migrations/scheduler.go

@@ -0,0 +1,105 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const (
+	MigrationJobWedgedTimeoutMilliseconds = 3600000 // 1 hour
+)
+
+type Scheduler struct {
+	jobServer              *jobs.JobServer
+	store                  store.Store
+	allMigrationsCompleted bool
+}
+
+func MakeScheduler(jobServer *jobs.JobServer, store store.Store) model.Scheduler {
+	return &Scheduler{jobServer, store, false}
+}
+
+func (scheduler *Scheduler) Enabled(_ *model.Config) bool {
+	return true
+}
+
+//nolint:unparam
+func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
+	if scheduler.allMigrationsCompleted {
+		return nil
+	}
+
+	nextTime := time.Now().Add(60 * time.Second)
+	return &nextTime
+}
+
+//nolint:unparam
+func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) {
+	mlog.Debug("Scheduling Job", mlog.String("scheduler", model.JobTypeMigrations))
+
+	// Work through the list of migrations in order. Schedule the first one that isn't done (assuming it isn't in progress already).
+	for _, key := range MakeMigrationsList() {
+		state, job, err := GetMigrationState(key, scheduler.store)
+		if err != nil {
+			mlog.Error("Failed to determine status of migration: ", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("migration_key", key), mlog.Err(err))
+			return nil, nil
+		}
+
+		if state == MigrationStateInProgress {
+			// Check the migration job isn't wedged.
+			if job != nil && job.LastActivityAt < model.GetMillis()-MigrationJobWedgedTimeoutMilliseconds && job.CreateAt < model.GetMillis()-MigrationJobWedgedTimeoutMilliseconds {
+				mlog.Warn("Job appears to be wedged. Rescheduling another instance.", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("wedged_job_id", job.Id), mlog.String("migration_key", key))
+				if err := scheduler.jobServer.SetJobError(job, nil); err != nil {
+					mlog.Error("Worker: Failed to set job error", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("job_id", job.Id), mlog.Err(err))
+				}
+				return scheduler.createJob(key, job)
+			}
+
+			return nil, nil
+		}
+
+		if state == MigrationStateCompleted {
+			// This migration is done. Continue to check the next.
+			continue
+		}
+
+		if state == MigrationStateUnscheduled {
+			mlog.Debug("Scheduling a new job for migration.", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("migration_key", key))
+			return scheduler.createJob(key, job)
+		}
+
+		mlog.Error("Unknown migration state. Not doing anything.", mlog.String("migration_state", state))
+		return nil, nil
+	}
+
+	// If we reached here, then there aren't any migrations left to run.
+	scheduler.allMigrationsCompleted = true
+	mlog.Debug("All migrations are complete.", mlog.String("scheduler", model.JobTypeMigrations))
+
+	return nil, nil
+}
+
+func (scheduler *Scheduler) createJob(migrationKey string, lastJob *model.Job) (*model.Job, *model.AppError) {
+	var lastDone string
+	if lastJob != nil {
+		lastDone = lastJob.Data[JobDataKeyMigrationLastDone]
+	}
+
+	data := map[string]string{
+		JobDataKeyMigration:         migrationKey,
+		JobDataKeyMigrationLastDone: lastDone,
+	}
+
+	job, err := scheduler.jobServer.CreateJob(model.JobTypeMigrations, data)
+	if err != nil {
+		return nil, err
+	}
+	return job, nil
+}

server/channels/jobs/migrations/worker.go

@@ -0,0 +1,182 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"context"
+	"net/http"
+	"sync/atomic"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const (
+	TimeBetweenBatches = 100
+)
+
+type Worker struct {
+	name      string
+	stop      chan struct{}
+	stopped   chan bool
+	jobs      chan model.Job
+	jobServer *jobs.JobServer
+	store     store.Store
+	closed    int32
+}
+
+func MakeWorker(jobServer *jobs.JobServer, store store.Store) model.Worker {
+	worker := Worker{
+		name:      "Migrations",
+		stop:      make(chan struct{}),
+		stopped:   make(chan bool, 1),
+		jobs:      make(chan model.Job),
+		jobServer: jobServer,
+		store:     store,
+	}
+
+	return &worker
+}
+
+func (worker *Worker) Run() {
+	// Set to open if closed before. We are not bothered about multiple opens.
+	if atomic.CompareAndSwapInt32(&worker.closed, 1, 0) {
+		worker.stop = make(chan struct{})
+	}
+	mlog.Debug("Worker started", mlog.String("worker", worker.name))
+
+	defer func() {
+		mlog.Debug("Worker finished", mlog.String("worker", worker.name))
+		worker.stopped <- true
+	}()
+
+	for {
+		select {
+		case <-worker.stop:
+			mlog.Debug("Worker received stop signal", mlog.String("worker", worker.name))
+			return
+		case job := <-worker.jobs:
+			mlog.Debug("Worker received a new candidate job.", mlog.String("worker", worker.name))
+			worker.DoJob(&job)
+		}
+	}
+}
+
+func (worker *Worker) Stop() {
+	// Set to close, and if already closed before, then return.
+	if !atomic.CompareAndSwapInt32(&worker.closed, 0, 1) {
+		return
+	}
+	mlog.Debug("Worker stopping", mlog.String("worker", worker.name))
+	close(worker.stop)
+	<-worker.stopped
+}
+
+func (worker *Worker) JobChannel() chan<- model.Job {
+	return worker.jobs
+}
+
+func (worker *Worker) IsEnabled(_ *model.Config) bool {
+	return true
+}
+
+func (worker *Worker) DoJob(job *model.Job) {
+	defer worker.jobServer.HandleJobPanic(job)
+
+	if claimed, err := worker.jobServer.ClaimJob(job); err != nil {
+		mlog.Info("Worker experienced an error while trying to claim job",
+			mlog.String("worker", worker.name),
+			mlog.String("job_id", job.Id),
+			mlog.String("error", err.Error()))
+		return
+	} else if !claimed {
+		return
+	}
+
+	cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background())
+	cancelWatcherChan := make(chan struct{}, 1)
+	go worker.jobServer.CancellationWatcher(cancelCtx, job.Id, cancelWatcherChan)
+
+	defer cancelCancelWatcher()
+
+	for {
+		select {
+		case <-cancelWatcherChan:
+			mlog.Debug("Worker: Job has been canceled via CancellationWatcher", mlog.String("worker", worker.name), mlog.String("job_id", job.Id))
+			worker.setJobCanceled(job)
+			return
+
+		case <-worker.stop:
+			mlog.Debug("Worker: Job has been canceled via Worker Stop", mlog.String("worker", worker.name), mlog.String("job_id", job.Id))
+			worker.setJobCanceled(job)
+			return
+
+		case <-time.After(TimeBetweenBatches * time.Millisecond):
+			done, progress, err := worker.runMigration(job.Data[JobDataKeyMigration], job.Data[JobDataKeyMigrationLastDone])
+			if err != nil {
+				mlog.Error("Worker: Failed to run migration", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
+				worker.setJobError(job, err)
+				return
+			} else if done {
+				mlog.Info("Worker: Job is complete", mlog.String("worker", worker.name), mlog.String("job_id", job.Id))
+				worker.setJobSuccess(job)
+				return
+			} else {
+				job.Data[JobDataKeyMigrationLastDone] = progress
+				if err := worker.jobServer.UpdateInProgressJobData(job); err != nil {
+					mlog.Error("Worker: Failed to update migration status data for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
+					worker.setJobError(job, err)
+					return
+				}
+			}
+		}
+	}
+}
+
+func (worker *Worker) setJobSuccess(job *model.Job) {
+	if err := worker.jobServer.SetJobSuccess(job); err != nil {
+		mlog.Error("Worker: Failed to set success for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
+		worker.setJobError(job, err)
+	}
+}
+
+func (worker *Worker) setJobError(job *model.Job, appError *model.AppError) {
+	if err := worker.jobServer.SetJobError(job, appError); err != nil {
+		mlog.Error("Worker: Failed to set job error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
+	}
+}
+
+func (worker *Worker) setJobCanceled(job *model.Job) {
+	if err := worker.jobServer.SetJobCanceled(job); err != nil {
+		mlog.Error("Worker: Failed to mark job as canceled", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
+	}
+}
+
+// Return parameters:
+// - whether the migration is completed on this run (true) or still incomplete (false).
+// - the updated lastDone string for the migration.
+// - any error which may have occurred while running the migration.
+func (worker *Worker) runMigration(key string, lastDone string) (bool, string, *model.AppError) {
+	var done bool
+	var progress string
+	var err *model.AppError
+
+	switch key {
+	case model.MigrationKeyAdvancedPermissionsPhase2:
+		done, progress, err = worker.runAdvancedPermissionsPhase2Migration(lastDone)
+	default:
+		return false, "", model.NewAppError("MigrationsWorker.runMigration", "migrations.worker.run_migration.unknown_key", map[string]any{"key": key}, "", http.StatusInternalServerError)
+	}
+
+	if done {
+		if nErr := worker.store.System().Save(&model.System{Name: key, Value: "true"}); nErr != nil {
+			return false, "", model.NewAppError("runMigration", "migrations.system.save.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
+		}
+	}
+
+	return done, progress, err
+}

server/channels/jobs/notify_admin/install_plugin_scheduler.go

@@ -0,0 +1,25 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package notify_admin
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const installPluginSchedFreq = 1 * time.Minute
+
+func MakeInstallPluginScheduler(jobServer *jobs.JobServer, license *model.License, jobType string) model.Scheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		enabled := jobType == model.JobTypeInstallPluginNotifyAdmin
+		mlog.Debug("Scheduler: isEnabled: "+strconv.FormatBool(enabled), mlog.String("scheduler", jobType))
+		return enabled
+	}
+	return jobs.NewPeriodicScheduler(jobServer, jobType, installPluginSchedFreq, isEnabled)
+
+}

server/channels/jobs/notify_admin/scheduler.go

@@ -0,0 +1,25 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package notify_admin
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const schedFreq = 24 * time.Hour
+
+func MakeScheduler(jobServer *jobs.JobServer, license *model.License, jobType string) model.Scheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		enabled := license != nil && *license.Features.Cloud
+		mlog.Debug("Scheduler: isEnabled: "+strconv.FormatBool(enabled), mlog.String("scheduler", jobType))
+		return enabled
+	}
+	return jobs.NewPeriodicScheduler(jobServer, jobType, schedFreq, isEnabled)
+
+}

server/channels/jobs/notify_admin/worker.go

@@ -0,0 +1,73 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package notify_admin
+
+import (
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+)
+
+const (
+	UpgradeNotifyJobName = "UpgradeNotifyAdmin"
+	TrialNotifyJobName   = "TrialNotifyAdmin"
+	InstallNotifyJobName = "InstallNotifyAdmin"
+)
+
+type AppIface interface {
+	DoCheckForAdminNotifications(trial bool) *model.AppError
+}
+
+func MakeUpgradeNotifyWorker(jobServer *jobs.JobServer, license *model.License, app AppIface) model.Worker {
+	isEnabled := func(_ *model.Config) bool {
+		return license != nil && license.Features != nil && *license.Features.Cloud
+	}
+	execute := func(job *model.Job) error {
+		defer jobServer.HandleJobPanic(job)
+
+		appErr := app.DoCheckForAdminNotifications(false)
+		if appErr != nil {
+			return appErr
+		}
+
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(UpgradeNotifyJobName, jobServer, execute, isEnabled)
+	return worker
+}
+
+func MakeTrialNotifyWorker(jobServer *jobs.JobServer, license *model.License, app AppIface) model.Worker {
+	isEnabled := func(_ *model.Config) bool {
+		return license != nil && license.Features != nil && *license.Features.Cloud
+	}
+	execute := func(job *model.Job) error {
+		defer jobServer.HandleJobPanic(job)
+
+		appErr := app.DoCheckForAdminNotifications(true)
+		if appErr != nil {
+			return appErr
+		}
+
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(TrialNotifyJobName, jobServer, execute, isEnabled)
+	return worker
+}
+
+func MakeInstallPluginNotifyWorker(jobServer *jobs.JobServer, app AppIface) model.Worker {
+	isEnabled := func(_ *model.Config) bool {
+		return true
+	}
+	execute := func(job *model.Job) error {
+		defer jobServer.HandleJobPanic(job)
+
+		appErr := app.DoCheckForAdminNotifications(false)
+		if appErr != nil {
+			return appErr
+		}
+
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(InstallNotifyJobName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/product_notices/scheduler.go

@@ -0,0 +1,27 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package product_notices
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+)
+
+type Scheduler struct {
+	*jobs.PeriodicScheduler
+}
+
+func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
+	nextTime := time.Now().Add(time.Duration(*cfg.AnnouncementSettings.NoticesFetchFrequency) * time.Second)
+	return &nextTime
+}
+
+func MakeScheduler(jobServer *jobs.JobServer) model.Scheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.AnnouncementSettings.AdminNoticesEnabled || *cfg.AnnouncementSettings.UserNoticesEnabled
+	}
+	return &Scheduler{PeriodicScheduler: jobs.NewPeriodicScheduler(jobServer, model.JobTypeProductNotices, 0, isEnabled)}
+}

server/channels/jobs/product_notices/worker.go

@@ -0,0 +1,33 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package product_notices
+
+import (
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const jobName = "ProductNotices"
+
+type AppIface interface {
+	UpdateProductNotices() *model.AppError
+}
+
+func MakeWorker(jobServer *jobs.JobServer, app AppIface) model.Worker {
+	isEnabled := func(cfg *model.Config) bool {
+		return *cfg.AnnouncementSettings.AdminNoticesEnabled || *cfg.AnnouncementSettings.UserNoticesEnabled
+	}
+	execute := func(job *model.Job) error {
+		defer jobServer.HandleJobPanic(job)
+
+		if err := app.UpdateProductNotices(); err != nil {
+			mlog.Error("Worker: Failed to fetch product notices", mlog.String("worker", model.JobTypeProductNotices), mlog.String("job_id", job.Id), mlog.Err(err))
+			return err
+		}
+		return nil
+	}
+	worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled)
+	return worker
+}

server/channels/jobs/resend_invitation_email/worker.go

@@ -0,0 +1,209 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+package resend_invitation_email
+
+import (
+	"encoding/json"
+	"net/http"
+	"os"
+	"strconv"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/configservice"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/telemetry"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const FourtyEightHoursInMillis int64 = 172800000
+
+type AppIface interface {
+	configservice.ConfigService
+	GetUserByEmail(email string) (*model.User, *model.AppError)
+	GetTeamMembersByIds(teamID string, userIDs []string, restrictions *model.ViewUsersRestrictions) ([]*model.TeamMember, *model.AppError)
+	InviteNewUsersToTeamGracefully(memberInvite *model.MemberInvite, teamID, senderId string, reminderInterval string) ([]*model.EmailInviteWithError, *model.AppError)
+}
+
+type ResendInvitationEmailWorker struct {
+	name             string
+	stop             chan bool
+	stopped          chan bool
+	jobs             chan model.Job
+	jobServer        *jobs.JobServer
+	app              AppIface
+	store            store.Store
+	telemetryService *telemetry.TelemetryService
+}
+
+func MakeWorker(jobServer *jobs.JobServer, app AppIface, store store.Store, telemetryService *telemetry.TelemetryService) model.Worker {
+	worker := ResendInvitationEmailWorker{
+		name:             model.JobTypeResendInvitationEmail,
+		stop:             make(chan bool, 1),
+		stopped:          make(chan bool, 1),
+		jobs:             make(chan model.Job),
+		jobServer:        jobServer,
+		app:              app,
+		store:            store,
+		telemetryService: telemetryService,
+	}
+	return &worker
+}
+
+func (rseworker *ResendInvitationEmailWorker) Run() {
+	mlog.Debug("Worker started", mlog.String("worker", rseworker.name))
+
+	defer func() {
+		mlog.Debug("Worker finished", mlog.String("worker", rseworker.name))
+		rseworker.stopped <- true
+	}()
+
+	for {
+		select {
+		case <-rseworker.stop:
+			mlog.Debug("Worker received stop signal", mlog.String("worker", rseworker.name))
+			return
+		case job := <-rseworker.jobs:
+			mlog.Debug("Worker received a new candidate job.", mlog.String("worker", rseworker.name))
+			rseworker.DoJob(&job)
+		}
+	}
+}
+
+func (rseworker *ResendInvitationEmailWorker) IsEnabled(cfg *model.Config) bool {
+	return *cfg.ServiceSettings.EnableEmailInvitations
+}
+
+func (rseworker *ResendInvitationEmailWorker) Stop() {
+	mlog.Debug("Worker stopping", mlog.String("worker", rseworker.name))
+	rseworker.stop <- true
+	<-rseworker.stopped
+}
+
+func (rseworker *ResendInvitationEmailWorker) JobChannel() chan<- model.Job {
+	return rseworker.jobs
+}
+
+func (rseworker *ResendInvitationEmailWorker) DoJob(job *model.Job) {
+	defer rseworker.jobServer.HandleJobPanic(job)
+
+	elapsedTimeSinceSchedule, DurationInMillis := rseworker.GetDurations(job)
+	if elapsedTimeSinceSchedule > DurationInMillis {
+		rseworker.ResendEmails(job, "48")
+		rseworker.TearDown(job)
+	}
+}
+
+func (rseworker *ResendInvitationEmailWorker) setJobSuccess(job *model.Job) {
+	if err := rseworker.jobServer.SetJobSuccess(job); err != nil {
+		mlog.Error("Worker: Failed to set success for job", mlog.String("worker", rseworker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
+		rseworker.setJobError(job, err)
+	}
+}
+
+func (rseworker *ResendInvitationEmailWorker) setJobError(job *model.Job, appError *model.AppError) {
+	if err := rseworker.jobServer.SetJobError(job, appError); err != nil {
+		mlog.Error("Worker: Failed to set job error", mlog.String("worker", rseworker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
+	}
+}
+
+func (rseworker *ResendInvitationEmailWorker) cleanEmailData(emailStringData string) ([]string, error) {
+	// emailStringData looks like this ["user1@gmail.com","user2@gmail.com"]
+	emails := []string{}
+	err := json.Unmarshal([]byte(emailStringData), &emails)
+	if err != nil {
+		return nil, err
+	}
+
+	return emails, nil
+}
+
+func (rseworker *ResendInvitationEmailWorker) cleanChannelsData(channelStringData string) ([]string, error) {
+	// channelStringData looks like this ["uuuiiiiidddd","uuuiiiiidddd"]
+	channels := []string{}
+	err := json.Unmarshal([]byte(channelStringData), &channels)
+	if err != nil {
+		return nil, err
+	}
+
+	return channels, nil
+}
+
+func (rseworker *ResendInvitationEmailWorker) removeAlreadyJoined(teamID string, emailList []string) []string {
+	var notJoinedYet []string
+	for _, email := range emailList {
+		// check if the user with this email is on the system already
+		user, appErr := rseworker.app.GetUserByEmail(email)
+		if appErr != nil {
+			notJoinedYet = append(notJoinedYet, email)
+			continue
+		}
+		// now we check if they are part of the team already
+		userID := []string{user.Id}
+		members, appErr := rseworker.app.GetTeamMembersByIds(teamID, userID, nil)
+		if len(members) == 0 || appErr != nil {
+			notJoinedYet = append(notJoinedYet, email)
+		}
+	}
+
+	return notJoinedYet
+}
+
+func (rseworker *ResendInvitationEmailWorker) GetDurations(job *model.Job) (int64, int64) {
+	scheduledAt, _ := strconv.ParseInt(job.Data["scheduledAt"], 10, 64)
+	now := model.GetMillis()
+
+	elapsedTimeSinceSchedule := now - scheduledAt
+
+	duration := os.Getenv("MM_RESEND_INVITATION_EMAIL_JOB_DURATION")
+	DurationInMillis, parseError := strconv.ParseInt(duration, 10, 64)
+	if parseError != nil {
+		// default to 48 hours
+		DurationInMillis = FourtyEightHoursInMillis
+	}
+
+	return elapsedTimeSinceSchedule, DurationInMillis
+
+}
+
+func (rseworker *ResendInvitationEmailWorker) TearDown(job *model.Job) {
+	rseworker.store.System().PermanentDeleteByName(job.Id)
+	rseworker.setJobSuccess(job)
+}
+
+func (rseworker *ResendInvitationEmailWorker) ResendEmails(job *model.Job, interval string) {
+	teamID := job.Data["teamID"]
+	emailListData := job.Data["emailList"]
+	channelListData := job.Data["channelList"]
+
+	emailList, err := rseworker.cleanEmailData(emailListData)
+	if err != nil {
+		appErr := model.NewAppError("worker: "+rseworker.name, "job_id: "+job.Id, nil, "", http.StatusInternalServerError).Wrap(err)
+		mlog.Error("Worker: Failed to clean emails string data", mlog.String("worker", rseworker.name), mlog.String("job_id", job.Id), mlog.String("error", appErr.Error()))
+		rseworker.setJobError(job, appErr)
+	}
+
+	channelList, err := rseworker.cleanChannelsData(channelListData)
+	if err != nil {
+		appErr := model.NewAppError("worker: "+rseworker.name, "job_id: "+job.Id, nil, "", http.StatusInternalServerError).Wrap(err)
+		mlog.Error("Worker: Failed to clean channel string data", mlog.String("worker", rseworker.name), mlog.String("job_id", job.Id), mlog.String("error", appErr.Error()))
+		rseworker.setJobError(job, appErr)
+	}
+
+	emailList = rseworker.removeAlreadyJoined(teamID, emailList)
+
+	memberInvite := model.MemberInvite{
+		Emails: emailList,
+	}
+
+	if len(channelList) > 0 {
+		memberInvite.ChannelIds = channelList
+	}
+
+	_, appErr := rseworker.app.InviteNewUsersToTeamGracefully(&memberInvite, teamID, job.Data["senderID"], interval)
+	if appErr != nil {
+		mlog.Error("Worker: Failed to send emails", mlog.String("worker", rseworker.name), mlog.String("job_id", job.Id), mlog.String("error", appErr.Error()))
+		rseworker.setJobError(job, appErr)
+	}
+	rseworker.telemetryService.SendTelemetry("track_invite_email_resend", map[string]any{interval: interval})
+}

server/channels/jobs/schedulers.go

@@ -0,0 +1,188 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs
+
+import (
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+type Schedulers struct {
+	stop                 chan bool
+	stopped              chan bool
+	configChanged        chan *model.Config
+	clusterLeaderChanged chan bool
+	listenerId           string
+	jobs                 *JobServer
+	isLeader             bool
+	running              bool
+
+	schedulers   map[string]model.Scheduler
+	nextRunTimes map[string]*time.Time
+}
+
+var (
+	ErrSchedulersNotRunning    = errors.New("job schedulers are not running")
+	ErrSchedulersRunning       = errors.New("job schedulers are running")
+	ErrSchedulersUninitialized = errors.New("job schedulers are not initialized")
+)
+
+func (schedulers *Schedulers) AddScheduler(name string, scheduler model.Scheduler) {
+	schedulers.schedulers[name] = scheduler
+}
+
+// Start starts the schedulers. This call is not safe for concurrent use.
+// Synchronization should be implemented by the caller.
+func (schedulers *Schedulers) Start() {
+	schedulers.stop = make(chan bool)
+	schedulers.stopped = make(chan bool)
+	schedulers.listenerId = schedulers.jobs.ConfigService.AddConfigListener(schedulers.handleConfigChange)
+
+	go func() {
+		mlog.Info("Starting schedulers.")
+
+		defer func() {
+			mlog.Info("Schedulers stopped.")
+			close(schedulers.stopped)
+		}()
+
+		now := time.Now()
+		for name, scheduler := range schedulers.schedulers {
+			if !scheduler.Enabled(schedulers.jobs.Config()) {
+				schedulers.nextRunTimes[name] = nil
+			} else {
+				schedulers.setNextRunTime(schedulers.jobs.Config(), name, now, false)
+			}
+		}
+
+		for {
+			timer := time.NewTimer(1 * time.Minute)
+			select {
+			case <-schedulers.stop:
+				mlog.Debug("Schedulers received stop signal.")
+				timer.Stop()
+				return
+			case now = <-timer.C:
+				cfg := schedulers.jobs.Config()
+
+				for name, nextTime := range schedulers.nextRunTimes {
+					if nextTime == nil {
+						continue
+					}
+
+					if time.Now().After(*nextTime) {
+						scheduler := schedulers.schedulers[name]
+						if scheduler == nil || !schedulers.isLeader || !scheduler.Enabled(cfg) {
+							continue
+						}
+						if _, err := schedulers.scheduleJob(cfg, name, scheduler); err != nil {
+							mlog.Error("Failed to schedule job", mlog.String("scheduler", name), mlog.Err(err))
+							continue
+						}
+						schedulers.setNextRunTime(cfg, name, now, true)
+					}
+				}
+			case newCfg := <-schedulers.configChanged:
+				for name, scheduler := range schedulers.schedulers {
+					if !schedulers.isLeader || !scheduler.Enabled(newCfg) {
+						schedulers.nextRunTimes[name] = nil
+					} else {
+						schedulers.setNextRunTime(newCfg, name, now, false)
+					}
+				}
+			case isLeader := <-schedulers.clusterLeaderChanged:
+				for name := range schedulers.schedulers {
+					schedulers.isLeader = isLeader
+					if !isLeader {
+						schedulers.nextRunTimes[name] = nil
+					} else {
+						schedulers.setNextRunTime(schedulers.jobs.Config(), name, now, false)
+					}
+				}
+			}
+			timer.Stop()
+		}
+	}()
+
+	schedulers.running = true
+}
+
+// Stop stops the schedulers. This call is not safe for concurrent use.
+// Synchronization should be implemented by the caller.
+func (schedulers *Schedulers) Stop() {
+	mlog.Info("Stopping schedulers.")
+	close(schedulers.stop)
+	<-schedulers.stopped
+	schedulers.jobs.ConfigService.RemoveConfigListener(schedulers.listenerId)
+	schedulers.listenerId = ""
+	schedulers.running = false
+}
+
+func (schedulers *Schedulers) setNextRunTime(cfg *model.Config, name string, now time.Time, pendingJobs bool) {
+	scheduler := schedulers.schedulers[name]
+
+	if !pendingJobs {
+		pj, err := schedulers.jobs.CheckForPendingJobsByType(name)
+		if err != nil {
+			mlog.Error("Failed to set next job run time", mlog.Err(err))
+			schedulers.nextRunTimes[name] = nil
+			return
+		}
+		pendingJobs = pj
+	}
+
+	lastSuccessfulJob, err := schedulers.jobs.GetLastSuccessfulJobByType(name)
+	if err != nil {
+		mlog.Error("Failed to set next job run time", mlog.Err(err))
+		schedulers.nextRunTimes[name] = nil
+		return
+	}
+
+	schedulers.nextRunTimes[name] = scheduler.NextScheduleTime(cfg, now, pendingJobs, lastSuccessfulJob)
+	mlog.Debug("Next run time for scheduler", mlog.String("scheduler_name", name), mlog.String("next_runtime", fmt.Sprintf("%v", schedulers.nextRunTimes[name])))
+}
+
+func (schedulers *Schedulers) scheduleJob(cfg *model.Config, name string, scheduler model.Scheduler) (*model.Job, *model.AppError) {
+	pendingJobs, err := schedulers.jobs.CheckForPendingJobsByType(name)
+	if err != nil {
+		return nil, err
+	}
+
+	lastSuccessfulJob, err2 := schedulers.jobs.GetLastSuccessfulJobByType(name)
+	if err2 != nil {
+		return nil, err
+	}
+
+	return scheduler.ScheduleJob(cfg, pendingJobs, lastSuccessfulJob)
+}
+
+func (schedulers *Schedulers) handleConfigChange(_, newConfig *model.Config) {
+	mlog.Debug("Schedulers received config change.")
+	select {
+	case schedulers.configChanged <- newConfig:
+	case <-schedulers.stop:
+	}
+}
+
+func (schedulers *Schedulers) handleClusterLeaderChange(isLeader bool) {
+	select {
+	case schedulers.clusterLeaderChanged <- isLeader:
+	default:
+		mlog.Debug("Sending cluster leader change message to schedulers failed.")
+
+		// Drain the buffered channel to make room for the latest change.
+		select {
+		case <-schedulers.clusterLeaderChanged:
+		default:
+		}
+
+		// Enqueue the latest change. This operation is safe due to this method
+		// being called under lock.
+		schedulers.clusterLeaderChanged <- isLeader
+	}
+}

server/channels/jobs/schedulers_test.go

@@ -0,0 +1,154 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+package jobs
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/plugin/plugintest/mock"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store/storetest"
+	"github.com/mattermost/mattermost-server/v6/server/channels/utils/testutils"
+)
+
+type MockScheduler struct {
+	mock.Mock
+}
+
+func (scheduler *MockScheduler) Enabled(cfg *model.Config) bool {
+	return true
+}
+
+func (scheduler *MockScheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
+	nextTime := time.Now().Add(60 * time.Second)
+	return &nextTime
+}
+
+func (scheduler *MockScheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) {
+	return nil, nil
+}
+
+func TestScheduler(t *testing.T) {
+	mockStore := &storetest.Store{}
+	defer mockStore.AssertExpectations(t)
+
+	job := &model.Job{
+		Id:       model.NewId(),
+		CreateAt: model.GetMillis(),
+		Status:   model.JobStatusPending,
+		Type:     model.JobTypeMessageExport,
+	}
+	// mock job store doesn't return a previously successful job, forcing fallback to config
+	mockStore.JobStore.On("GetNewestJobByStatusesAndType", mock.AnythingOfType("[]string"), mock.AnythingOfType("string")).Return(job, nil)
+	mockStore.JobStore.On("GetCountByStatusAndType", mock.AnythingOfType("string"), mock.AnythingOfType("string")).Return(int64(1), nil)
+
+	jobServer := &JobServer{
+		Store: mockStore,
+		ConfigService: &testutils.StaticConfigService{
+			Cfg: &model.Config{
+				// mock config
+				DataRetentionSettings: model.DataRetentionSettings{
+					EnableMessageDeletion: model.NewBool(true),
+				},
+				MessageExportSettings: model.MessageExportSettings{
+					EnableExport: model.NewBool(true),
+				},
+			},
+		},
+	}
+
+	jobServer.initSchedulers()
+	jobServer.RegisterJobType(model.JobTypeDataRetention, nil, new(MockScheduler))
+	jobServer.RegisterJobType(model.JobTypeMessageExport, nil, new(MockScheduler))
+
+	t.Run("Base", func(t *testing.T) {
+		jobServer.StartSchedulers()
+		time.Sleep(time.Second)
+
+		jobServer.StopSchedulers()
+		// They should be all on here
+		for _, element := range jobServer.schedulers.nextRunTimes {
+			assert.NotNil(t, element)
+		}
+	})
+
+	t.Run("ClusterLeaderChanged", func(t *testing.T) {
+		jobServer.initSchedulers()
+		jobServer.StartSchedulers()
+		time.Sleep(time.Second)
+		jobServer.HandleClusterLeaderChange(false)
+		jobServer.StopSchedulers()
+		// They should be turned off
+		for _, element := range jobServer.schedulers.nextRunTimes {
+			assert.Nil(t, element)
+		}
+	})
+
+	t.Run("ClusterLeaderChangedBeforeStart", func(t *testing.T) {
+		jobServer.initSchedulers()
+		jobServer.HandleClusterLeaderChange(false)
+		jobServer.StartSchedulers()
+		time.Sleep(time.Second)
+		jobServer.StopSchedulers()
+		for _, element := range jobServer.schedulers.nextRunTimes {
+			assert.Nil(t, element)
+		}
+	})
+
+	t.Run("DoubleClusterLeaderChangedBeforeStart", func(t *testing.T) {
+		jobServer.initSchedulers()
+		jobServer.HandleClusterLeaderChange(false)
+		jobServer.HandleClusterLeaderChange(true)
+		jobServer.StartSchedulers()
+		time.Sleep(time.Second)
+		jobServer.StopSchedulers()
+		for _, element := range jobServer.schedulers.nextRunTimes {
+			assert.NotNil(t, element)
+		}
+	})
+
+	t.Run("ConfigChanged", func(t *testing.T) {
+		jobServer.initSchedulers()
+		jobServer.StartSchedulers()
+		time.Sleep(time.Second)
+		jobServer.HandleClusterLeaderChange(false)
+		// After running a config change, they should stay off
+		jobServer.schedulers.handleConfigChange(nil, nil)
+		jobServer.StopSchedulers()
+		for _, element := range jobServer.schedulers.nextRunTimes {
+			assert.Nil(t, element)
+		}
+	})
+
+	t.Run("ConfigChangedDeadlock", func(t *testing.T) {
+		jobServer.initSchedulers()
+		jobServer.StartSchedulers()
+		time.Sleep(time.Second)
+
+		var wg sync.WaitGroup
+		wg.Add(2)
+		go func() {
+			defer wg.Done()
+			jobServer.StopSchedulers()
+		}()
+		go func() {
+			defer wg.Done()
+			jobServer.schedulers.handleConfigChange(nil, nil)
+		}()
+
+		wg.Wait()
+	})
+}
+
+func TestRandomDelay(t *testing.T) {
+	cases := []int64{5, 10, 100}
+	for _, c := range cases {
+		out := getRandomDelay(c)
+		require.Less(t, out.Milliseconds(), c)
+	}
+}

server/channels/jobs/server.go

@@ -0,0 +1,126 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs
+
+import (
+	"sync"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/einterfaces"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/configservice"
+)
+
+type JobServer struct {
+	ConfigService configservice.ConfigService
+	Store         store.Store
+	metrics       einterfaces.MetricsInterface
+
+	// mut is used to protect the following fields from concurrent access.
+	mut        sync.Mutex
+	workers    *Workers
+	schedulers *Schedulers
+}
+
+func NewJobServer(configService configservice.ConfigService, store store.Store, metrics einterfaces.MetricsInterface) *JobServer {
+	srv := &JobServer{
+		ConfigService: configService,
+		Store:         store,
+		metrics:       metrics,
+	}
+	srv.initWorkers()
+	srv.initSchedulers()
+	return srv
+}
+
+func (srv *JobServer) initWorkers() {
+	workers := NewWorkers(srv.ConfigService)
+	workers.Watcher = srv.MakeWatcher(workers, DefaultWatcherPollingInterval)
+	srv.workers = workers
+}
+
+func (srv *JobServer) initSchedulers() {
+	schedulers := &Schedulers{
+		configChanged:        make(chan *model.Config),
+		clusterLeaderChanged: make(chan bool, 1),
+		jobs:                 srv,
+		isLeader:             true,
+		schedulers:           make(map[string]model.Scheduler),
+		nextRunTimes:         make(map[string]*time.Time),
+	}
+
+	srv.schedulers = schedulers
+}
+
+func (srv *JobServer) Config() *model.Config {
+	return srv.ConfigService.Config()
+}
+
+func (srv *JobServer) RegisterJobType(name string, worker model.Worker, scheduler model.Scheduler) {
+	srv.mut.Lock()
+	defer srv.mut.Unlock()
+	if worker != nil {
+		srv.workers.AddWorker(name, worker)
+	}
+	if scheduler != nil {
+		srv.schedulers.AddScheduler(name, scheduler)
+	}
+}
+
+func (srv *JobServer) StartWorkers() error {
+	srv.mut.Lock()
+	defer srv.mut.Unlock()
+	if srv.workers == nil {
+		return ErrWorkersUninitialized
+	} else if srv.workers.running {
+		return ErrWorkersRunning
+	}
+	srv.workers.Start()
+	return nil
+}
+
+func (srv *JobServer) StartSchedulers() error {
+	srv.mut.Lock()
+	defer srv.mut.Unlock()
+	if srv.schedulers == nil {
+		return ErrSchedulersUninitialized
+	} else if srv.schedulers.running {
+		return ErrSchedulersRunning
+	}
+	srv.schedulers.Start()
+	return nil
+}
+
+func (srv *JobServer) StopWorkers() error {
+	srv.mut.Lock()
+	defer srv.mut.Unlock()
+	if srv.workers == nil {
+		return ErrWorkersUninitialized
+	} else if !srv.workers.running {
+		return ErrWorkersNotRunning
+	}
+	srv.workers.Stop()
+	return nil
+}
+
+func (srv *JobServer) StopSchedulers() error {
+	srv.mut.Lock()
+	defer srv.mut.Unlock()
+	if srv.schedulers == nil {
+		return ErrSchedulersUninitialized
+	} else if !srv.schedulers.running {
+		return ErrSchedulersNotRunning
+	}
+	srv.schedulers.Stop()
+	return nil
+}
+
+func (srv *JobServer) HandleClusterLeaderChange(isLeader bool) {
+	srv.mut.Lock()
+	defer srv.mut.Unlock()
+	if srv.schedulers != nil {
+		srv.schedulers.handleClusterLeaderChange(isLeader)
+	}
+}

server/channels/jobs/server_test.go

@@ -0,0 +1,123 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestStartWorkers(t *testing.T) {
+	t.Run("uninitialized", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		err := jobServer.StartWorkers()
+		require.Equal(t, ErrWorkersUninitialized, err)
+	})
+
+	t.Run("already running", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		jobServer.initWorkers()
+		err := jobServer.StartWorkers()
+		require.NoError(t, err)
+		err = jobServer.StartWorkers()
+		require.Equal(t, ErrWorkersRunning, err)
+		// Parking the go routing to let the worker watcher start
+		time.Sleep(1 * time.Millisecond)
+		err = jobServer.StopWorkers()
+		require.NoError(t, err)
+	})
+
+	t.Run("not running", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		jobServer.initWorkers()
+		err := jobServer.StartWorkers()
+		require.NoError(t, err)
+		// Parking the go routing to let the worker watcher start
+		time.Sleep(1 * time.Millisecond)
+		err = jobServer.StopWorkers()
+		require.NoError(t, err)
+	})
+}
+
+func TestStopWorkers(t *testing.T) {
+	t.Run("uninitialized", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		err := jobServer.StopWorkers()
+		require.Equal(t, ErrWorkersUninitialized, err)
+	})
+
+	t.Run("not running", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		jobServer.initWorkers()
+		err := jobServer.StopWorkers()
+		require.Equal(t, ErrWorkersNotRunning, err)
+	})
+
+	t.Run("running", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		jobServer.initWorkers()
+		err := jobServer.StartWorkers()
+		require.NoError(t, err)
+		// Parking the go routing to let the worker watcher start
+		time.Sleep(1 * time.Millisecond)
+		err = jobServer.StopWorkers()
+		require.NoError(t, err)
+	})
+}
+
+func TestStartSchedulers(t *testing.T) {
+	t.Run("uninitialized", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		err := jobServer.StartSchedulers()
+		require.Equal(t, ErrSchedulersUninitialized, err)
+	})
+
+	t.Run("initialized", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		jobServer.initSchedulers()
+		err := jobServer.StartSchedulers()
+		require.NoError(t, err)
+
+		err = jobServer.StopSchedulers()
+		require.NoError(t, err)
+	})
+
+	t.Run("already running", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		jobServer.initSchedulers()
+		err := jobServer.StartSchedulers()
+		require.NoError(t, err)
+		err = jobServer.StartSchedulers()
+		require.Equal(t, ErrSchedulersRunning, err)
+
+		err = jobServer.StopSchedulers()
+		require.NoError(t, err)
+	})
+}
+
+func TestStopSchedulers(t *testing.T) {
+	t.Run("uninitialized", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		err := jobServer.StopSchedulers()
+		require.Equal(t, ErrSchedulersUninitialized, err)
+	})
+
+	t.Run("not running", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		jobServer.initSchedulers()
+		err := jobServer.StopSchedulers()
+		require.Equal(t, ErrSchedulersNotRunning, err)
+	})
+
+	t.Run("running", func(t *testing.T) {
+		jobServer, _, _ := makeJobServer(t)
+		jobServer.initSchedulers()
+		err := jobServer.StartSchedulers()
+		require.NoError(t, err)
+		err = jobServer.StopSchedulers()
+		require.NoError(t, err)
+	})
+}

server/channels/jobs/workers.go

@@ -0,0 +1,91 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs
+
+import (
+	"errors"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/configservice"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+type Workers struct {
+	ConfigService configservice.ConfigService
+	Watcher       *Watcher
+
+	workers map[string]model.Worker
+
+	listenerId string
+	running    bool
+}
+
+var (
+	ErrWorkersNotRunning    = errors.New("job workers are not running")
+	ErrWorkersRunning       = errors.New("job workers are running")
+	ErrWorkersUninitialized = errors.New("job workers are not initialized")
+)
+
+func NewWorkers(configService configservice.ConfigService) *Workers {
+	return &Workers{
+		ConfigService: configService,
+		workers:       make(map[string]model.Worker),
+	}
+}
+
+func (workers *Workers) AddWorker(name string, worker model.Worker) {
+	workers.workers[name] = worker
+}
+
+func (workers *Workers) Get(name string) model.Worker {
+	return workers.workers[name]
+}
+
+// Start starts the workers. This call is not safe for concurrent use.
+// Synchronization should be implemented by the caller.
+func (workers *Workers) Start() {
+	mlog.Info("Starting workers")
+
+	for _, w := range workers.workers {
+		if w.IsEnabled(workers.ConfigService.Config()) {
+			go w.Run()
+		}
+	}
+
+	go workers.Watcher.Start()
+
+	workers.listenerId = workers.ConfigService.AddConfigListener(workers.handleConfigChange)
+	workers.running = true
+}
+
+func (workers *Workers) handleConfigChange(oldConfig *model.Config, newConfig *model.Config) {
+	mlog.Debug("Workers received config change.")
+
+	for _, w := range workers.workers {
+		if w.IsEnabled(oldConfig) && !w.IsEnabled(newConfig) {
+			w.Stop()
+		}
+		if !w.IsEnabled(oldConfig) && w.IsEnabled(newConfig) {
+			go w.Run()
+		}
+	}
+}
+
+// Stop stops the workers. This call is not safe for concurrent use.
+// Synchronization should be implemented by the caller.
+func (workers *Workers) Stop() {
+	workers.ConfigService.RemoveConfigListener(workers.listenerId)
+
+	workers.Watcher.Stop()
+
+	for _, w := range workers.workers {
+		if w.IsEnabled(workers.ConfigService.Config()) {
+			w.Stop()
+		}
+	}
+
+	workers.running = false
+
+	mlog.Info("Stopped workers")
+}