From 2c3e289509ed1ec43f3e5a4798ab0207bb82b831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Mon, 14 Feb 2022 18:21:18 +0100 Subject: [PATCH] Working on refactoring jobs service (#19205) * Working on refactoring jobs service * Making more consistent with the previous existing code * Remove no longer needed functions * Making a base PeridicScheduler to use it in most of the schedulers implementations * Removing accidental complexity from on of the jobs * Removing accidental complexity from expirynotify * Fixing compilation from previous commit * Remove accidental complexity from the export_delete job * Simplifying the workers by making a reusable worker * Using simple worker for export_delete job * Simpliying export process job * Simpliying extract content job * Simpliying import delete job * Simpliying import process job * Simpliying product noticies job * Simpliying fix crt channel unreads job (only removing the uneeded register function) * Simpliying migrations job (only removing the uneeded register function) * fixup * Simpliying plugins job (only removing the uneeded register function) * Simpliying bleve indexing job (only removing the uneeded register function) * Simpliying resend invitation email job (only removing the uneeded register function) * Fixing tests * Simplifying migration tests infrastructure * Adding missed license to files * Adding an empty file to imports package to ensure this package exist even without enterprise repo * Regenerating einterfaces mocks * Adding missed license to files * Updating i18n/en.json file * help fixing enterprise tests compilation * Adding new DailyScheduler * Fixing typo and changing the waitTime type for periodic sechduler * Making the daily scheduler more generic * Adding comments to clarify not used parameters in interface scheduler interface implementations * Using merror to handle multiple errors in jobs workers * Fixing linter errors * Addressing PR review comments * Reverting go.tools.mod changes * Removing the static check for worker type in the model (moving it to the insertion of new jobs * Moving migrations job to the jobs directory * Fixing (and improving a bit) tests * Apply suggestions from code review Co-authored-by: Doug Lauder * Fixing enterprise tests * Removing unneeded InitWorkers/InitSchedulers calls * Fix expirenotify job when error happens * Fixing govet errors Co-authored-by: Mattermod Co-authored-by: Doug Lauder --- api4/apitestlib.go | 2 - api4/job_test.go | 36 +- app/app_iface.go | 4 +- app/enterprise.go | 78 +---- app/expirynotify.go | 2 +- app/expirynotify_test.go | 5 +- app/export.go | 15 +- app/export_test.go | 20 +- app/import.go | 2 +- app/import_test.go | 2 +- app/opentracing/opentracing_layer.go | 4 +- app/server.go | 150 ++++++--- app/upload.go | 3 +- cmd/mattermost/commands/export.go | 3 +- .../jobs}/indexer_interface.go | 2 +- einterfaces/jobs/resend_invitation_email.go | 11 - einterfaces/mocks/IndexerJobInterface.go | 31 ++ i18n/en.json | 12 - imports/placeholder.go | 42 +-- jobs/active_users/scheduler.go | 43 +-- jobs/active_users/worker.go | 118 +------ jobs/base_schedulers.go | 72 ++++ jobs/base_workers.go | 103 ++++++ jobs/expirynotify/expirynotify.go | 20 -- jobs/expirynotify/scheduler.go | 43 +-- jobs/expirynotify/worker.go | 89 +---- jobs/export_delete/scheduler.go | 43 +-- jobs/export_delete/worker.go | 151 +++------ jobs/export_process/worker.go | 165 +++------- jobs/extract_content/worker.go | 212 ++++-------- jobs/import_delete/scheduler.go | 43 +-- jobs/import_delete/worker.go | 217 +++++------- jobs/import_process/worker.go | 235 +++++-------- jobs/interfaces/active_users_interface.go | 13 - jobs/interfaces/expirynotify_interface.go | 13 - jobs/interfaces/export_delete_interface.go | 11 - jobs/interfaces/export_process_interface.go | 12 - jobs/interfaces/extract_content_interface.go | 12 - jobs/interfaces/import_delete_interface.go | 11 - jobs/interfaces/import_process_interface.go | 12 - jobs/interfaces/migrations_interface.go | 13 - jobs/interfaces/plugins_interface.go | 13 - jobs/interfaces/product_notices_interface.go | 13 - jobs/jobs.go | 4 + jobs/jobs_watcher.go | 130 +------- .../advanced_permissions_phase_2.go | 4 +- jobs/migrations/helper_test.go | 32 ++ {migrations => jobs/migrations}/main_test.go | 0 {migrations => jobs/migrations}/migrations.go | 12 - .../migrations}/migrations_test.go | 25 +- {migrations => jobs/migrations}/scheduler.go | 36 +- {migrations => jobs/migrations}/worker.go | 26 +- jobs/product_notices/product_notices.go | 20 -- jobs/product_notices/scheduler.go | 32 +- jobs/product_notices/worker.go | 95 +----- .../resend_invitation_email.go | 19 -- jobs/resend_invitation_email/scheduler.go | 42 --- jobs/resend_invitation_email/worker.go | 67 ++-- jobs/schedulers.go | 113 ++----- jobs/schedulers_test.go | 20 +- jobs/server.go | 32 +- jobs/workers.go | 308 ++---------------- migrations/helper_test.go | 265 --------------- model/bulk_export.go | 13 + model/job.go | 26 +- model/upload_session.go | 5 +- plugin/scheduler/plugin.go | 20 -- plugin/scheduler/scheduler.go | 42 +-- plugin/scheduler/worker.go | 21 +- .../bleveengine/indexer/indexing_job.go | 30 +- .../bleveengine/indexer/indexing_job_test.go | 2 +- 71 files changed, 968 insertions(+), 2574 deletions(-) rename {jobs/interfaces => einterfaces/jobs}/indexer_interface.go (92%) delete mode 100644 einterfaces/jobs/resend_invitation_email.go create mode 100644 einterfaces/mocks/IndexerJobInterface.go create mode 100644 jobs/base_schedulers.go create mode 100644 jobs/base_workers.go delete mode 100644 jobs/expirynotify/expirynotify.go delete mode 100644 jobs/interfaces/active_users_interface.go delete mode 100644 jobs/interfaces/expirynotify_interface.go delete mode 100644 jobs/interfaces/export_delete_interface.go delete mode 100644 jobs/interfaces/export_process_interface.go delete mode 100644 jobs/interfaces/extract_content_interface.go delete mode 100644 jobs/interfaces/import_delete_interface.go delete mode 100644 jobs/interfaces/import_process_interface.go delete mode 100644 jobs/interfaces/migrations_interface.go delete mode 100644 jobs/interfaces/plugins_interface.go delete mode 100644 jobs/interfaces/product_notices_interface.go rename {migrations => jobs/migrations}/advanced_permissions_phase_2.go (93%) create mode 100644 jobs/migrations/helper_test.go rename {migrations => jobs/migrations}/main_test.go (100%) rename {migrations => jobs/migrations}/migrations.go (80%) rename {migrations => jobs/migrations}/migrations_test.go (72%) rename {migrations => jobs/migrations}/scheduler.go (71%) rename {migrations => jobs/migrations}/worker.go (87%) delete mode 100644 jobs/product_notices/product_notices.go delete mode 100644 jobs/resend_invitation_email/resend_invitation_email.go delete mode 100644 jobs/resend_invitation_email/scheduler.go delete mode 100644 migrations/helper_test.go create mode 100644 model/bulk_export.go delete mode 100644 plugin/scheduler/plugin.go diff --git a/api4/apitestlib.go b/api4/apitestlib.go index e9022a5b93..4dc006c37d 100644 --- a/api4/apitestlib.go +++ b/api4/apitestlib.go @@ -189,8 +189,6 @@ func setupTestHelper(dbStore store.Store, searchEngine *searchengine.Broker, ent if enterprise { th.App.Srv().SetLicense(model.NewTestLicense()) - th.App.Srv().Jobs.InitWorkers() - th.App.Srv().Jobs.InitSchedulers() } else { th.App.Srv().SetLicense(nil) } diff --git a/api4/job_test.go b/api4/job_test.go index 422843050d..e73d247327 100644 --- a/api4/job_test.go +++ b/api4/job_test.go @@ -19,33 +19,29 @@ func TestCreateJob(t *testing.T) { defer th.TearDown() job := &model.Job{ - Type: model.JobTypeMessageExport, + Type: model.JobTypeActiveUsers, Data: map[string]string{ "thing": "stuff", }, } - _, resp, err := th.SystemManagerClient.CreateJob(job) - require.Error(t, err) - CheckForbiddenStatus(t, resp) + t.Run("valid job as user without permissions", func(t *testing.T) { + _, resp, err := th.SystemManagerClient.CreateJob(job) + require.Error(t, err) + CheckForbiddenStatus(t, resp) + }) - received, _, err := th.SystemAdminClient.CreateJob(job) - require.NoError(t, err) + t.Run("valid job as user with permissions", func(t *testing.T) { + received, _, err := th.SystemAdminClient.CreateJob(job) + require.NoError(t, err) + defer th.App.Srv().Store.Job().Delete(received.Id) + }) - defer th.App.Srv().Store.Job().Delete(received.Id) - - job = &model.Job{ - Type: model.NewId(), - } - - _, resp, err = th.SystemAdminClient.CreateJob(job) - require.Error(t, err) - CheckBadRequestStatus(t, resp) - - job.Type = model.JobTypeElasticsearchPostIndexing - _, resp, err = th.Client.CreateJob(job) - require.Error(t, err) - CheckForbiddenStatus(t, resp) + t.Run("invalid job type as user without permissions", func(t *testing.T) { + _, resp, err := th.SystemAdminClient.CreateJob(&model.Job{Type: model.NewId()}) + require.Error(t, err) + CheckBadRequestStatus(t, resp) + }) } func TestGetJob(t *testing.T) { diff --git a/app/app_iface.go b/app/app_iface.go index d0f37ae980..97bbffe4ec 100644 --- a/app/app_iface.go +++ b/app/app_iface.go @@ -236,7 +236,7 @@ type AppIface interface { // NewWebConn returns a new WebConn instance. NewWebConn(cfg *WebConnConfig) *WebConn // NotifySessionsExpired is called periodically from the job server to notify any mobile sessions that have expired. - NotifySessionsExpired() *model.AppError + NotifySessionsExpired() error // OverrideIconURLIfEmoji changes the post icon override URL prop, if it has an emoji icon, // so that it points to the URL (relative) of the emoji - static if emoji is default, /api if custom. OverrideIconURLIfEmoji(post *model.Post) @@ -405,7 +405,7 @@ type AppIface interface { BuildPostReactions(postID string) (*[]ReactionImportData, *model.AppError) BuildPushNotificationMessage(contentsConfig string, post *model.Post, user *model.User, channel *model.Channel, channelName string, senderName string, explicitMention bool, channelWideMention bool, replyToThreadType string) (*model.PushNotification, *model.AppError) BuildSamlMetadataObject(idpMetadata []byte) (*model.SamlMetadataResponse, *model.AppError) - BulkExport(writer io.Writer, outPath string, opts BulkExportOpts) *model.AppError + BulkExport(writer io.Writer, outPath string, opts model.BulkExportOpts) *model.AppError BulkImport(c *request.Context, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun bool, workers int) (*model.AppError, int) BulkImportWithPath(c *request.Context, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun bool, workers int, importPath string) (*model.AppError, int) CancelJob(jobId string) *model.AppError diff --git a/app/enterprise.go b/app/enterprise.go index 7d912278a2..7d85752ad3 100644 --- a/app/enterprise.go +++ b/app/enterprise.go @@ -6,7 +6,6 @@ package app import ( "github.com/mattermost/mattermost-server/v6/einterfaces" ejobs "github.com/mattermost/mattermost-server/v6/einterfaces/jobs" - tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces" "github.com/mattermost/mattermost-server/v6/model" "github.com/mattermost/mattermost-server/v6/services/searchengine" "github.com/mattermost/mattermost-server/v6/shared/mlog" @@ -60,9 +59,9 @@ func RegisterJobsElasticsearchAggregatorInterface(f func(*Server) ejobs.Elastics jobsElasticsearchAggregatorInterface = f } -var jobsElasticsearchIndexerInterface func(*Server) tjobs.IndexerJobInterface +var jobsElasticsearchIndexerInterface func(*Server) ejobs.IndexerJobInterface -func RegisterJobsElasticsearchIndexerInterface(f func(*Server) tjobs.IndexerJobInterface) { +func RegisterJobsElasticsearchIndexerInterface(f func(*Server) ejobs.IndexerJobInterface) { jobsElasticsearchIndexerInterface = f } @@ -72,85 +71,12 @@ func RegisterJobsLdapSyncInterface(f func(*Server) ejobs.LdapSyncInterface) { jobsLdapSyncInterface = f } -var jobsMigrationsInterface func(*Server) tjobs.MigrationsJobInterface - -func RegisterJobsMigrationsJobInterface(f func(*Server) tjobs.MigrationsJobInterface) { - jobsMigrationsInterface = f -} - -var jobsPluginsInterface func(*Server) tjobs.PluginsJobInterface - -func RegisterJobsPluginsJobInterface(f func(*Server) tjobs.PluginsJobInterface) { - jobsPluginsInterface = f -} - -var jobsBleveIndexerInterface func(*Server) tjobs.IndexerJobInterface - -func RegisterJobsBleveIndexerInterface(f func(*Server) tjobs.IndexerJobInterface) { - jobsBleveIndexerInterface = f -} - -var jobsActiveUsersInterface func(*Server) tjobs.ActiveUsersJobInterface - -func RegisterJobsActiveUsersInterface(f func(*Server) tjobs.ActiveUsersJobInterface) { - jobsActiveUsersInterface = f -} - -var jobsResendInvitationEmailInterface func(*Server) ejobs.ResendInvitationEmailJobInterface - -// RegisterJobsResendInvitationEmailInterface is used to register or initialize the jobsResendInvitationEmailInterface -func RegisterJobsResendInvitationEmailInterface(f func(*Server) ejobs.ResendInvitationEmailJobInterface) { - jobsResendInvitationEmailInterface = f -} - var jobsCloudInterface func(*Server) ejobs.CloudJobInterface func RegisterJobsCloudInterface(f func(*Server) ejobs.CloudJobInterface) { jobsCloudInterface = f } -var jobsExpiryNotifyInterface func(*Server) tjobs.ExpiryNotifyJobInterface - -func RegisterJobsExpiryNotifyJobInterface(f func(*Server) tjobs.ExpiryNotifyJobInterface) { - jobsExpiryNotifyInterface = f -} - -var jobsImportProcessInterface func(*Server) tjobs.ImportProcessInterface - -func RegisterJobsImportProcessInterface(f func(*Server) tjobs.ImportProcessInterface) { - jobsImportProcessInterface = f -} - -var jobsImportDeleteInterface func(*Server) tjobs.ImportDeleteInterface - -func RegisterJobsImportDeleteInterface(f func(*Server) tjobs.ImportDeleteInterface) { - jobsImportDeleteInterface = f -} - -var jobsExportProcessInterface func(*Server) tjobs.ExportProcessInterface - -func RegisterJobsExportProcessInterface(f func(*Server) tjobs.ExportProcessInterface) { - jobsExportProcessInterface = f -} - -var jobsExportDeleteInterface func(*Server) tjobs.ExportDeleteInterface - -func RegisterJobsExportDeleteInterface(f func(*Server) tjobs.ExportDeleteInterface) { - jobsExportDeleteInterface = f -} - -var jobsExtractContentInterface func(*Server) tjobs.ExtractContentInterface - -func RegisterJobsExtractContentInterface(f func(*Server) tjobs.ExtractContentInterface) { - jobsExtractContentInterface = f -} - -var productNoticesJobInterface func(*Server) tjobs.ProductNoticesJobInterface - -func RegisterProductNoticesJobInterface(f func(*Server) tjobs.ProductNoticesJobInterface) { - productNoticesJobInterface = f -} - var ldapInterface func(*Server) einterfaces.LdapInterface func RegisterLdapInterface(f func(*Server) einterfaces.LdapInterface) { diff --git a/app/expirynotify.go b/app/expirynotify.go index efee6ce939..bf29a21265 100644 --- a/app/expirynotify.go +++ b/app/expirynotify.go @@ -16,7 +16,7 @@ const ( ) // NotifySessionsExpired is called periodically from the job server to notify any mobile sessions that have expired. -func (a *App) NotifySessionsExpired() *model.AppError { +func (a *App) NotifySessionsExpired() error { if *a.Config().EmailSettings.SendPushNotifications { pushServer := *a.Config().EmailSettings.PushNotificationServer if license := a.ch.srv.License(); pushServer == model.MHPNS && (license == nil || !*license.Features.MHPNS) { diff --git a/app/expirynotify_test.go b/app/expirynotify_test.go index 31e23b8c71..42b74e802b 100644 --- a/app/expirynotify_test.go +++ b/app/expirynotify_test.go @@ -34,7 +34,7 @@ func TestNotifySessionsExpired(t *testing.T) { err := th.App.NotifySessionsExpired() // no error, but also no requests sent - require.Nil(t, err) + require.NoError(t, err) require.Equal(t, 0, handler.numReqs()) }) @@ -65,8 +65,7 @@ func TestNotifySessionsExpired(t *testing.T) { } err := th.App.NotifySessionsExpired() - - require.Nil(t, err) + require.NoError(t, err) require.Equal(t, 2, handler.numReqs()) expected := []string{"22222", "33333"} diff --git a/app/export.go b/app/export.go index 0116a6ea39..08fc6fd38d 100644 --- a/app/export.go +++ b/app/export.go @@ -20,15 +20,6 @@ import ( "github.com/mattermost/mattermost-server/v6/store" ) -type BulkExportOpts struct { - IncludeAttachments bool - CreateArchive bool -} - -// ExportDataDir is the name of the directory were to store additional data -// included with the export (e.g. file attachments). -const ExportDataDir = "data" - // We use this map to identify the exportable preferences. // Here we link the preference category and name, to the name of the relevant field in the import struct. var exportablePreferences = map[ComparablePreference]string{{ @@ -64,7 +55,7 @@ var exportablePreferences = map[ComparablePreference]string{{ }: "EmailInterval", } -func (a *App) BulkExport(writer io.Writer, outPath string, opts BulkExportOpts) *model.AppError { +func (a *App) BulkExport(writer io.Writer, outPath string, opts model.BulkExportOpts) *model.AppError { var zipWr *zip.Writer if opts.CreateArchive { var err error @@ -706,7 +697,7 @@ func (a *App) exportFile(outPath, filePath string, zipWr *zip.Writer) *model.App if zipWr != nil { wr, err = zipWr.CreateHeader(&zip.FileHeader{ - Name: filepath.Join(ExportDataDir, filePath), + Name: filepath.Join(model.ExportDataDir, filePath), Method: zip.Store, }) if err != nil { @@ -714,7 +705,7 @@ func (a *App) exportFile(outPath, filePath string, zipWr *zip.Writer) *model.App nil, "err="+err.Error(), http.StatusInternalServerError) } } else { - filePath = filepath.Join(outPath, ExportDataDir, filePath) + filePath = filepath.Join(outPath, model.ExportDataDir, filePath) if err = os.MkdirAll(filepath.Dir(filePath), 0700); err != nil { return model.NewAppError("exportFileAttachment", "app.export.export_attachment.mkdirall.error", nil, "err="+err.Error(), http.StatusInternalServerError) diff --git a/app/export_test.go b/app/export_test.go index 5f38b8e53d..2baaf4bd6e 100644 --- a/app/export_test.go +++ b/app/export_test.go @@ -179,7 +179,7 @@ func TestExportAllUsers(t *testing.T) { require.Nil(t, err) var b bytes.Buffer - err = th1.App.BulkExport(&b, "somePath", BulkExportOpts{}) + err = th1.App.BulkExport(&b, "somePath", model.BulkExportOpts{}) require.Nil(t, err) th2 := Setup(t) @@ -227,7 +227,7 @@ func TestExportDMChannel(t *testing.T) { th1.CreateDmChannel(th1.BasicUser2) var b bytes.Buffer - err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{}) + err := th1.App.BulkExport(&b, "somePath", model.BulkExportOpts{}) require.Nil(t, err) channels, nErr := th1.App.Srv().Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000") @@ -268,7 +268,7 @@ func TestExportDMChannel(t *testing.T) { th1.App.PermanentDeleteUser(th1.Context, th1.BasicUser) var b bytes.Buffer - err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{}) + err := th1.App.BulkExport(&b, "somePath", model.BulkExportOpts{}) require.Nil(t, err) th2 := Setup(t).InitBasic() @@ -292,7 +292,7 @@ func TestExportDMChannelToSelf(t *testing.T) { th1.CreateDmChannel(th1.BasicUser) var b bytes.Buffer - err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{}) + err := th1.App.BulkExport(&b, "somePath", model.BulkExportOpts{}) require.Nil(t, err) channels, nErr := th1.App.Srv().Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000") @@ -330,7 +330,7 @@ func TestExportGMChannel(t *testing.T) { th1.CreateGroupChannel(user1, user2) var b bytes.Buffer - err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{}) + err := th1.App.BulkExport(&b, "somePath", model.BulkExportOpts{}) require.Nil(t, err) channels, nErr := th1.App.Srv().Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000") @@ -362,7 +362,7 @@ func TestExportGMandDMChannels(t *testing.T) { th1.CreateGroupChannel(user1, user2) var b bytes.Buffer - err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{}) + err := th1.App.BulkExport(&b, "somePath", model.BulkExportOpts{}) require.Nil(t, err) channels, nErr := th1.App.Srv().Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000") @@ -445,7 +445,7 @@ func TestExportDMandGMPost(t *testing.T) { assert.Equal(t, 4, len(posts)) var b bytes.Buffer - appErr := th1.App.BulkExport(&b, "somePath", BulkExportOpts{}) + appErr := th1.App.BulkExport(&b, "somePath", model.BulkExportOpts{}) require.Nil(t, appErr) th1.TearDown() @@ -520,7 +520,7 @@ func TestExportPostWithProps(t *testing.T) { require.NotEmpty(t, posts[1].Props) var b bytes.Buffer - appErr := th1.App.BulkExport(&b, "somePath", BulkExportOpts{}) + appErr := th1.App.BulkExport(&b, "somePath", model.BulkExportOpts{}) require.Nil(t, appErr) th1.TearDown() @@ -558,7 +558,7 @@ func TestExportDMPostWithSelf(t *testing.T) { th1.CreatePost(dmChannel) var b bytes.Buffer - err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{}) + err := th1.App.BulkExport(&b, "somePath", model.BulkExportOpts{}) require.Nil(t, err) posts, nErr := th1.App.Srv().Store.Post().GetDirectPostParentsForExportAfter(1000, "0000000") @@ -622,7 +622,7 @@ func TestBulkExport(t *testing.T) { require.NoError(t, err) defer exportFile.Close() - opts := BulkExportOpts{ + opts := model.BulkExportOpts{ IncludeAttachments: true, CreateArchive: true, } diff --git a/app/import.go b/app/import.go index e8b0efa33e..5a5c02e95e 100644 --- a/app/import.go +++ b/app/import.go @@ -327,7 +327,7 @@ func (a *App) ListImports() ([]string, *model.AppError) { results := make([]string, 0, len(imports)) for i := 0; i < len(imports); i++ { filename := filepath.Base(imports[i]) - if !strings.HasSuffix(filename, IncompleteUploadSuffix) { + if !strings.HasSuffix(filename, model.IncompleteUploadSuffix) { results = append(results, filename) } } diff --git a/app/import_test.go b/app/import_test.go index 74b93b661f..919507bb0b 100644 --- a/app/import_test.go +++ b/app/import_test.go @@ -489,7 +489,7 @@ func TestImportBulkImportWithAttachments(t *testing.T) { th.App.UpdateConfig(func(cfg *model.Config) { cfg.TeamSettings.MaxUsersPerTeam = model.NewInt(1000) }) - appErr, _ := th.App.BulkImportWithPath(th.Context, jsonFile, importZipReader, false, 1, ExportDataDir) + appErr, _ := th.App.BulkImportWithPath(th.Context, jsonFile, importZipReader, false, 1, model.ExportDataDir) require.Nil(t, appErr) adminUser, appErr := th.App.GetUserByUsername("sysadmin") diff --git a/app/opentracing/opentracing_layer.go b/app/opentracing/opentracing_layer.go index 5039ab2ee1..6d7bc81657 100644 --- a/app/opentracing/opentracing_layer.go +++ b/app/opentracing/opentracing_layer.go @@ -973,7 +973,7 @@ func (a *OpenTracingAppLayer) BuildSamlMetadataObject(idpMetadata []byte) (*mode return resultVar0, resultVar1 } -func (a *OpenTracingAppLayer) BulkExport(writer io.Writer, outPath string, opts app.BulkExportOpts) *model.AppError { +func (a *OpenTracingAppLayer) BulkExport(writer io.Writer, outPath string, opts model.BulkExportOpts) *model.AppError { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.BulkExport") @@ -11742,7 +11742,7 @@ func (a *OpenTracingAppLayer) NotifyAndSetWarnMetricAck(warnMetricId string, sen return resultVar0 } -func (a *OpenTracingAppLayer) NotifySessionsExpired() *model.AppError { +func (a *OpenTracingAppLayer) NotifySessionsExpired() error { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.NotifySessionsExpired") diff --git a/app/server.go b/app/server.go index 125f4b18e7..a9b8ec8ea3 100644 --- a/app/server.go +++ b/app/server.go @@ -45,13 +45,25 @@ import ( "github.com/mattermost/mattermost-server/v6/config" "github.com/mattermost/mattermost-server/v6/einterfaces" "github.com/mattermost/mattermost-server/v6/jobs" + "github.com/mattermost/mattermost-server/v6/jobs/active_users" + "github.com/mattermost/mattermost-server/v6/jobs/expirynotify" + "github.com/mattermost/mattermost-server/v6/jobs/export_delete" + "github.com/mattermost/mattermost-server/v6/jobs/export_process" + "github.com/mattermost/mattermost-server/v6/jobs/extract_content" + "github.com/mattermost/mattermost-server/v6/jobs/import_delete" + "github.com/mattermost/mattermost-server/v6/jobs/import_process" + "github.com/mattermost/mattermost-server/v6/jobs/migrations" + "github.com/mattermost/mattermost-server/v6/jobs/product_notices" + "github.com/mattermost/mattermost-server/v6/jobs/resend_invitation_email" "github.com/mattermost/mattermost-server/v6/model" + "github.com/mattermost/mattermost-server/v6/plugin/scheduler" "github.com/mattermost/mattermost-server/v6/services/awsmeter" "github.com/mattermost/mattermost-server/v6/services/cache" "github.com/mattermost/mattermost-server/v6/services/httpservice" "github.com/mattermost/mattermost-server/v6/services/remotecluster" "github.com/mattermost/mattermost-server/v6/services/searchengine" "github.com/mattermost/mattermost-server/v6/services/searchengine/bleveengine" + "github.com/mattermost/mattermost-server/v6/services/searchengine/bleveengine/indexer" "github.com/mattermost/mattermost-server/v6/services/sharedchannel" "github.com/mattermost/mattermost-server/v6/services/telemetry" "github.com/mattermost/mattermost-server/v6/services/timezones" @@ -1862,72 +1874,110 @@ func (ch *Channels) ClientConfigHash() string { func (s *Server) initJobs() { s.Jobs = jobs.NewJobServer(s, s.Store, s.Metrics) + s.Jobs.InitWorkers() + s.Jobs.InitSchedulers() + if jobsDataRetentionJobInterface != nil { - s.Jobs.DataRetentionJob = jobsDataRetentionJobInterface(s) + builder := jobsDataRetentionJobInterface(s) + s.Jobs.RegisterJobType(model.JobTypeDataRetention, builder.MakeWorker(), builder.MakeScheduler()) } + if jobsMessageExportJobInterface != nil { - s.Jobs.MessageExportJob = jobsMessageExportJobInterface(s) + builder := jobsMessageExportJobInterface(s) + s.Jobs.RegisterJobType(model.JobTypeMessageExport, builder.MakeWorker(), builder.MakeScheduler()) } + if jobsElasticsearchAggregatorInterface != nil { - s.Jobs.ElasticsearchAggregator = jobsElasticsearchAggregatorInterface(s) + builder := jobsElasticsearchAggregatorInterface(s) + s.Jobs.RegisterJobType(model.JobTypeElasticsearchPostAggregation, builder.MakeWorker(), builder.MakeScheduler()) } + if jobsElasticsearchIndexerInterface != nil { - s.Jobs.ElasticsearchIndexer = jobsElasticsearchIndexerInterface(s) - } - if jobsBleveIndexerInterface != nil { - s.Jobs.BleveIndexer = jobsBleveIndexerInterface(s) - } - if jobsMigrationsInterface != nil { - s.Jobs.Migrations = jobsMigrationsInterface(s) + builder := jobsElasticsearchIndexerInterface(s) + s.Jobs.RegisterJobType(model.JobTypeElasticsearchPostIndexing, builder.MakeWorker(), nil) } + if jobsLdapSyncInterface != nil { - s.Jobs.LdapSync = jobsLdapSyncInterface(s) - } - if jobsPluginsInterface != nil { - s.Jobs.Plugins = jobsPluginsInterface(s) - } - if jobsExpiryNotifyInterface != nil { - s.Jobs.ExpiryNotify = jobsExpiryNotifyInterface(s) - } - if productNoticesJobInterface != nil { - s.Jobs.ProductNotices = productNoticesJobInterface(s) - } - if jobsImportProcessInterface != nil { - s.Jobs.ImportProcess = jobsImportProcessInterface(s) - } - if jobsImportDeleteInterface != nil { - s.Jobs.ImportDelete = jobsImportDeleteInterface(s) - } - if jobsExportDeleteInterface != nil { - s.Jobs.ExportDelete = jobsExportDeleteInterface(s) - } - - if jobsExportProcessInterface != nil { - s.Jobs.ExportProcess = jobsExportProcessInterface(s) - } - - if jobsExportProcessInterface != nil { - s.Jobs.ExportProcess = jobsExportProcessInterface(s) - } - - if jobsActiveUsersInterface != nil { - s.Jobs.ActiveUsers = jobsActiveUsersInterface(s) + builder := jobsLdapSyncInterface(s) + s.Jobs.RegisterJobType(model.JobTypeLdapSync, builder.MakeWorker(), builder.MakeScheduler()) } if jobsCloudInterface != nil { - s.Jobs.Cloud = jobsCloudInterface(s) + builder := jobsCloudInterface(s) + s.Jobs.RegisterJobType(model.JobTypeCloud, builder.MakeWorker(), builder.MakeScheduler()) } - if jobsResendInvitationEmailInterface != nil { - s.Jobs.ResendInvitationEmails = jobsResendInvitationEmailInterface(s) - } + s.Jobs.RegisterJobType( + model.JobTypeBlevePostIndexing, + indexer.MakeWorker(s.Jobs, s.SearchEngine.BleveEngine.(*bleveengine.BleveEngine)), + nil, + ) - if jobsExtractContentInterface != nil { - s.Jobs.ExtractContent = jobsExtractContentInterface(s) - } + s.Jobs.RegisterJobType( + model.JobTypeMigrations, + migrations.MakeWorker(s.Jobs, s.Store), + migrations.MakeScheduler(s.Jobs, s.Store), + ) - s.Jobs.InitWorkers() - s.Jobs.InitSchedulers() + s.Jobs.RegisterJobType( + model.JobTypePlugins, + scheduler.MakeWorker(s.Jobs, New(ServerConnector(s.Channels()))), + scheduler.MakeScheduler(s.Jobs), + ) + + s.Jobs.RegisterJobType( + model.JobTypeExpiryNotify, + expirynotify.MakeWorker(s.Jobs, New(ServerConnector(s.Channels())).NotifySessionsExpired), + expirynotify.MakeScheduler(s.Jobs), + ) + + s.Jobs.RegisterJobType( + model.JobTypeProductNotices, + product_notices.MakeWorker(s.Jobs, New(ServerConnector(s.Channels()))), + product_notices.MakeScheduler(s.Jobs), + ) + + s.Jobs.RegisterJobType( + model.JobTypeImportProcess, + import_process.MakeWorker(s.Jobs, New(ServerConnector(s.Channels()))), + nil, + ) + + s.Jobs.RegisterJobType( + model.JobTypeImportDelete, + import_delete.MakeWorker(s.Jobs, New(ServerConnector(s.Channels())), s.Store), + import_delete.MakeScheduler(s.Jobs), + ) + + s.Jobs.RegisterJobType( + model.JobTypeExportDelete, + export_delete.MakeWorker(s.Jobs, New(ServerConnector(s.Channels()))), + export_delete.MakeScheduler(s.Jobs), + ) + + s.Jobs.RegisterJobType( + model.JobTypeExportProcess, + export_process.MakeWorker(s.Jobs, New(ServerConnector(s.Channels()))), + nil, + ) + + s.Jobs.RegisterJobType( + model.JobTypeActiveUsers, + active_users.MakeWorker(s.Jobs, s.Store, func() einterfaces.MetricsInterface { return s.Metrics }), + active_users.MakeScheduler(s.Jobs), + ) + + s.Jobs.RegisterJobType( + model.JobTypeResendInvitationEmail, + resend_invitation_email.MakeWorker(s.Jobs, New(ServerConnector(s.Channels())), s.Store, s.telemetryService), + nil, + ) + + s.Jobs.RegisterJobType( + model.JobTypeExtractContent, + extract_content.MakeWorker(s.Jobs, New(ServerConnector(s.Channels())), s.Store), + nil, + ) } func (s *Server) TelemetryId() string { diff --git a/app/upload.go b/app/upload.go index d472a6755a..ed46a107ab 100644 --- a/app/upload.go +++ b/app/upload.go @@ -20,7 +20,6 @@ import ( ) const minFirstPartSize = 5 * 1024 * 1024 // 5MB -const IncompleteUploadSuffix = ".tmp" func (a *App) runPluginsHook(c *request.Context, info *model.FileInfo, file io.Reader) *model.AppError { pluginsEnvironment := a.GetPluginsEnvironment() @@ -195,7 +194,7 @@ func (a *App) UploadData(c *request.Context, us *model.UploadSession, rd io.Read uploadPath := us.Path if us.Type == model.UploadTypeImport { - uploadPath += IncompleteUploadSuffix + uploadPath += model.IncompleteUploadSuffix } // make sure it's not possible to upload more data than what is expected. diff --git a/cmd/mattermost/commands/export.go b/cmd/mattermost/commands/export.go index 62e15da920..baac81757d 100644 --- a/cmd/mattermost/commands/export.go +++ b/cmd/mattermost/commands/export.go @@ -10,7 +10,6 @@ import ( "path/filepath" "time" - "github.com/mattermost/mattermost-server/v6/app" "github.com/mattermost/mattermost-server/v6/audit" "github.com/mattermost/mattermost-server/v6/model" @@ -227,7 +226,7 @@ func bulkExportCmdF(command *cobra.Command, args []string) error { return err } - var opts app.BulkExportOpts + var opts model.BulkExportOpts opts.IncludeAttachments = attachments opts.CreateArchive = archive if err := a.BulkExport(fileWriter, filepath.Dir(outPath), opts); err != nil { diff --git a/jobs/interfaces/indexer_interface.go b/einterfaces/jobs/indexer_interface.go similarity index 92% rename from jobs/interfaces/indexer_interface.go rename to einterfaces/jobs/indexer_interface.go index a85872c41b..09035b84e0 100644 --- a/jobs/interfaces/indexer_interface.go +++ b/einterfaces/jobs/indexer_interface.go @@ -1,7 +1,7 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -package interfaces +package jobs import ( "github.com/mattermost/mattermost-server/v6/model" diff --git a/einterfaces/jobs/resend_invitation_email.go b/einterfaces/jobs/resend_invitation_email.go deleted file mode 100644 index 3d971a494f..0000000000 --- a/einterfaces/jobs/resend_invitation_email.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. -package jobs - -import "github.com/mattermost/mattermost-server/v6/model" - -// ResendInvitationEmailJobInterface defines the interface for the job to resend invitation emails -type ResendInvitationEmailJobInterface interface { - MakeWorker() model.Worker - MakeScheduler() model.Scheduler -} diff --git a/einterfaces/mocks/IndexerJobInterface.go b/einterfaces/mocks/IndexerJobInterface.go new file mode 100644 index 0000000000..1382d77987 --- /dev/null +++ b/einterfaces/mocks/IndexerJobInterface.go @@ -0,0 +1,31 @@ +// Code generated by mockery v1.0.0. DO NOT EDIT. + +// Regenerate this file using `make einterfaces-mocks`. + +package mocks + +import ( + model "github.com/mattermost/mattermost-server/v6/model" + mock "github.com/stretchr/testify/mock" +) + +// IndexerJobInterface is an autogenerated mock type for the IndexerJobInterface type +type IndexerJobInterface struct { + mock.Mock +} + +// MakeWorker provides a mock function with given fields: +func (_m *IndexerJobInterface) MakeWorker() model.Worker { + ret := _m.Called() + + var r0 model.Worker + if rf, ok := ret.Get(0).(func() model.Worker); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(model.Worker) + } + } + + return r0 +} diff --git a/i18n/en.json b/i18n/en.json index 76cbe05ee5..4bc143c2cc 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -7559,18 +7559,6 @@ "id": "error", "translation": "Error" }, - { - "id": "extrac_content.worker.do_job.invalid_input.from", - "translation": "Invalid input value 'from'" - }, - { - "id": "extrac_content.worker.do_job.invalid_input.to", - "translation": "Invalid input value 'to'" - }, - { - "id": "extract_content.worker.do_job.file_info", - "translation": "Failed to get file information for content extraction." - }, { "id": "group_not_associated_to_synced_team", "translation": "Group cannot be associated to the channel until it is first associated to the parent group-synced team." diff --git a/imports/placeholder.go b/imports/placeholder.go index 46d29c79da..f12e716614 100644 --- a/imports/placeholder.go +++ b/imports/placeholder.go @@ -1,42 +1,8 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. +// As part of the enterprise code integration machinery a go file is copied to +// this package, this file ensure the existence of the package anyway, allowing +// the rest of the code to import the package when enterprise code is there and +// when is not. package imports - -import ( - // This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty. - _ "github.com/mattermost/mattermost-server/v6/migrations" - - // This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty. - _ "github.com/mattermost/mattermost-server/v6/plugin/scheduler" - - // This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty. - _ "github.com/mattermost/mattermost-server/v6/services/searchengine/bleveengine/indexer" - - // This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty. - _ "github.com/mattermost/mattermost-server/v6/jobs/expirynotify" - - // This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty. - _ "github.com/mattermost/mattermost-server/v6/jobs/active_users" - - // This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty. - _ "github.com/mattermost/mattermost-server/v6/jobs/product_notices" - - // This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty. - _ "github.com/mattermost/mattermost-server/v6/jobs/import_process" - - // This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty. - _ "github.com/mattermost/mattermost-server/v6/jobs/import_delete" - - // This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty. - _ "github.com/mattermost/mattermost-server/v6/jobs/export_process" - - // This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty. - _ "github.com/mattermost/mattermost-server/v6/jobs/export_delete" - - // This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty. - _ "github.com/mattermost/mattermost-server/v6/jobs/resend_invitation_email" - - // This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty. - _ "github.com/mattermost/mattermost-server/v6/jobs/extract_content" -) diff --git a/jobs/active_users/scheduler.go b/jobs/active_users/scheduler.go index 569f8051f4..ad2d496f53 100644 --- a/jobs/active_users/scheduler.go +++ b/jobs/active_users/scheduler.go @@ -6,46 +6,15 @@ package active_users import ( "time" - "github.com/mattermost/mattermost-server/v6/app" + "github.com/mattermost/mattermost-server/v6/jobs" "github.com/mattermost/mattermost-server/v6/model" ) -const ( - SchedFreqMinutes = 10 -) +const schedFreq = 10 * time.Minute -type Scheduler struct { - App *app.App -} - -func (m *ActiveUsersJobInterfaceImpl) MakeScheduler() model.Scheduler { - return &Scheduler{m.App} -} - -func (scheduler *Scheduler) Name() string { - return JobName + "Scheduler" -} - -func (scheduler *Scheduler) JobType() string { - return model.JobTypeActiveUsers -} - -func (scheduler *Scheduler) Enabled(cfg *model.Config) bool { - // Only enabled when Metrics are enabled. - return *cfg.MetricsSettings.Enable -} - -func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time { - nextTime := time.Now().Add(SchedFreqMinutes * time.Minute) - return &nextTime -} - -func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) { - data := map[string]string{} - - job, err := scheduler.App.Srv().Jobs.CreateJob(model.JobTypeActiveUsers, data) - if err != nil { - return nil, err +func MakeScheduler(jobServer *jobs.JobServer) model.Scheduler { + isEnabled := func(cfg *model.Config) bool { + return *cfg.MetricsSettings.Enable } - return job, nil + return jobs.NewPeriodicScheduler(jobServer, model.JobTypeActiveUsers, schedFreq, isEnabled) } diff --git a/jobs/active_users/worker.go b/jobs/active_users/worker.go index 4fdb26ce9a..8f562dd6a5 100644 --- a/jobs/active_users/worker.go +++ b/jobs/active_users/worker.go @@ -4,117 +4,31 @@ package active_users import ( - "net/http" - - "github.com/mattermost/mattermost-server/v6/app" + "github.com/mattermost/mattermost-server/v6/einterfaces" "github.com/mattermost/mattermost-server/v6/jobs" - tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces" "github.com/mattermost/mattermost-server/v6/model" - "github.com/mattermost/mattermost-server/v6/shared/mlog" + "github.com/mattermost/mattermost-server/v6/store" ) const ( JobName = "ActiveUsers" ) -type Worker struct { - name string - stop chan bool - stopped chan bool - jobs chan model.Job - jobServer *jobs.JobServer - app *app.App -} - -func init() { - app.RegisterJobsActiveUsersInterface(func(s *app.Server) tjobs.ActiveUsersJobInterface { - a := app.New(app.ServerConnector(s.Channels())) - return &ActiveUsersJobInterfaceImpl{a} - }) -} - -type ActiveUsersJobInterfaceImpl struct { - App *app.App -} - -func (m *ActiveUsersJobInterfaceImpl) MakeWorker() model.Worker { - worker := Worker{ - name: JobName, - stop: make(chan bool, 1), - stopped: make(chan bool, 1), - jobs: make(chan model.Job), - jobServer: m.App.Srv().Jobs, - app: m.App, +func MakeWorker(jobServer *jobs.JobServer, store store.Store, getMetrics func() einterfaces.MetricsInterface) model.Worker { + isEnabled := func(cfg *model.Config) bool { + return *cfg.MetricsSettings.Enable } - return &worker -} - -func (worker *Worker) Run() { - mlog.Debug("Worker started", mlog.String("worker", worker.name)) - - defer func() { - mlog.Debug("Worker finished", mlog.String("worker", worker.name)) - worker.stopped <- true - }() - - for { - select { - case <-worker.stop: - mlog.Debug("Worker received stop signal", mlog.String("worker", worker.name)) - return - case job := <-worker.jobs: - mlog.Debug("Worker received a new candidate job.", mlog.String("worker", worker.name)) - worker.DoJob(&job) + execute := func(job *model.Job) error { + count, err := store.User().Count(model.UserCountOptions{IncludeDeleted: false}) + if err != nil { + return err } + + if getMetrics() != nil { + getMetrics().ObserveEnabledUsers(count) + } + return nil } -} - -func (worker *Worker) Stop() { - mlog.Debug("Worker stopping", mlog.String("worker", worker.name)) - worker.stop <- true - <-worker.stopped -} - -func (worker *Worker) JobChannel() chan<- model.Job { - return worker.jobs -} - -func (worker *Worker) DoJob(job *model.Job) { - if claimed, err := worker.jobServer.ClaimJob(job); err != nil { - mlog.Warn("Worker experienced an error while trying to claim job", - mlog.String("worker", worker.name), - mlog.String("job_id", job.Id), - mlog.String("error", err.Error())) - return - } else if !claimed { - return - } - - count, err := worker.app.Srv().Store.User().Count(model.UserCountOptions{IncludeDeleted: false}) - - if err != nil { - mlog.Error("Worker: Failed to get active user count", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) - worker.setJobError(job, model.NewAppError("DoJob", "app.user.get_total_users_count.app_error", nil, err.Error(), http.StatusInternalServerError)) - return - } - - if worker.app.Metrics() != nil { - worker.app.Metrics().ObserveEnabledUsers(count) - } - - mlog.Info("Worker: Job is complete", mlog.String("worker", worker.name), mlog.String("job_id", job.Id)) - worker.setJobSuccess(job) -} - -func (worker *Worker) setJobSuccess(job *model.Job) { - if err := worker.app.Srv().Jobs.SetJobSuccess(job); err != nil { - mlog.Error("Worker: Failed to set success for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) - worker.setJobError(job, err) - } -} - -func (worker *Worker) setJobError(job *model.Job, appError *model.AppError) { - if err := worker.app.Srv().Jobs.SetJobError(job, appError); err != nil { - mlog.Error("Worker: Failed to set job error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) - } + worker := jobs.NewSimpleWorker(JobName, jobServer, execute, isEnabled) + return worker } diff --git a/jobs/base_schedulers.go b/jobs/base_schedulers.go new file mode 100644 index 0000000000..b6416b3bbd --- /dev/null +++ b/jobs/base_schedulers.go @@ -0,0 +1,72 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package jobs + +import ( + "time" + + "github.com/mattermost/mattermost-server/v6/model" +) + +type PeriodicScheduler struct { + jobs *JobServer + period time.Duration + jobType string + enabledFunc func(cfg *model.Config) bool +} + +func NewPeriodicScheduler(jobs *JobServer, jobType string, period time.Duration, enabledFunc func(cfg *model.Config) bool) *PeriodicScheduler { + return &PeriodicScheduler{ + period: period, + jobType: jobType, + enabledFunc: enabledFunc, + jobs: jobs, + } +} + +func (scheduler *PeriodicScheduler) Enabled(cfg *model.Config) bool { + return scheduler.enabledFunc(cfg) +} + +func (scheduler *PeriodicScheduler) NextScheduleTime(_ *model.Config, _ time.Time /* pendingJobs */, _ bool /* lastSuccessfulJob */, _ *model.Job) *time.Time { + nextTime := time.Now().Add(scheduler.period) + return &nextTime +} + +func (scheduler *PeriodicScheduler) ScheduleJob(_ *model.Config /* pendingJobs */, _ bool /* lastSuccessfulJob */, _ *model.Job) (*model.Job, *model.AppError) { + return scheduler.jobs.CreateJob(scheduler.jobType, nil) +} + +type DailyScheduler struct { + jobs *JobServer + startTimeFunc func(cfg *model.Config) *time.Time + jobType string + enabledFunc func(cfg *model.Config) bool +} + +func NewDailyScheduler(jobs *JobServer, jobType string, startTimeFunc func(cfg *model.Config) *time.Time, enabledFunc func(cfg *model.Config) bool) *DailyScheduler { + return &DailyScheduler{ + startTimeFunc: startTimeFunc, + jobType: jobType, + enabledFunc: enabledFunc, + jobs: jobs, + } +} + +func (scheduler *DailyScheduler) Enabled(cfg *model.Config) bool { + return scheduler.enabledFunc(cfg) +} + +func (scheduler *DailyScheduler) NextScheduleTime(cfg *model.Config, now time.Time /* pendingJobs */, _ bool /* lastSuccessfulJob */, _ *model.Job) *time.Time { + scheduledTime := scheduler.startTimeFunc(cfg) + if scheduledTime == nil { + return nil + } + + return GenerateNextStartDateTime(now, *scheduledTime) +} + +func (scheduler *DailyScheduler) ScheduleJob(_ *model.Config /* pendingJobs */, _ bool /* lastSuccessfulJob */, _ *model.Job) (*model.Job, *model.AppError) { + return scheduler.jobs.CreateJob(scheduler.jobType, nil) +} diff --git a/jobs/base_workers.go b/jobs/base_workers.go new file mode 100644 index 0000000000..b5fbd6166c --- /dev/null +++ b/jobs/base_workers.go @@ -0,0 +1,103 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package jobs + +import ( + "net/http" + + "github.com/mattermost/mattermost-server/v6/model" + "github.com/mattermost/mattermost-server/v6/shared/mlog" +) + +type SimpleWorker struct { + name string + stop chan bool + stopped chan bool + jobs chan model.Job + jobServer *JobServer + execute func(job *model.Job) error + isEnabled func(cfg *model.Config) bool +} + +func NewSimpleWorker(name string, jobServer *JobServer, execute func(job *model.Job) error, isEnabled func(cfg *model.Config) bool) *SimpleWorker { + worker := SimpleWorker{ + name: name, + stop: make(chan bool, 1), + stopped: make(chan bool, 1), + jobs: make(chan model.Job), + jobServer: jobServer, + execute: execute, + isEnabled: isEnabled, + } + return &worker +} + +func (worker *SimpleWorker) Run() { + mlog.Debug("Worker started", mlog.String("worker", worker.name)) + + defer func() { + mlog.Debug("Worker finished", mlog.String("worker", worker.name)) + worker.stopped <- true + }() + + for { + select { + case <-worker.stop: + mlog.Debug("Worker received stop signal", mlog.String("worker", worker.name)) + return + case job := <-worker.jobs: + mlog.Debug("Worker received a new candidate job.", mlog.String("worker", worker.name)) + worker.DoJob(&job) + } + } +} + +func (worker *SimpleWorker) Stop() { + mlog.Debug("Worker stopping", mlog.String("worker", worker.name)) + worker.stop <- true + <-worker.stopped +} + +func (worker *SimpleWorker) JobChannel() chan<- model.Job { + return worker.jobs +} + +func (worker *SimpleWorker) IsEnabled(cfg *model.Config) bool { + return worker.isEnabled(cfg) +} + +func (worker *SimpleWorker) DoJob(job *model.Job) { + if claimed, err := worker.jobServer.ClaimJob(job); err != nil { + mlog.Warn("SimpleWorker experienced an error while trying to claim job", + mlog.String("worker", worker.name), + mlog.String("job_id", job.Id), + mlog.Err(err)) + return + } else if !claimed { + return + } + + err := worker.execute(job) + if err != nil { + mlog.Error("SimpleWorker: Failed to get active user count", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.Err(err)) + worker.setJobError(job, model.NewAppError("DoJob", "app.user.get_total_users_count.app_error", nil, err.Error(), http.StatusInternalServerError)) + return + } + + mlog.Info("SimpleWorker: Job is complete", mlog.String("worker", worker.name), mlog.String("job_id", job.Id)) + worker.setJobSuccess(job) +} + +func (worker *SimpleWorker) setJobSuccess(job *model.Job) { + if err := worker.jobServer.SetJobSuccess(job); err != nil { + mlog.Error("SimpleWorker: Failed to set success for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) + worker.setJobError(job, err) + } +} + +func (worker *SimpleWorker) setJobError(job *model.Job, appError *model.AppError) { + if err := worker.jobServer.SetJobError(job, appError); err != nil { + mlog.Error("SimpleWorker: Failed to set job error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.Err(err)) + } +} diff --git a/jobs/expirynotify/expirynotify.go b/jobs/expirynotify/expirynotify.go deleted file mode 100644 index ae8bc98d44..0000000000 --- a/jobs/expirynotify/expirynotify.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package expirynotify - -import ( - "github.com/mattermost/mattermost-server/v6/app" - tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces" -) - -type ExpiryNotifyJobInterfaceImpl struct { - App *app.App -} - -func init() { - app.RegisterJobsExpiryNotifyJobInterface(func(s *app.Server) tjobs.ExpiryNotifyJobInterface { - a := app.New(app.ServerConnector(s.Channels())) - return &ExpiryNotifyJobInterfaceImpl{a} - }) -} diff --git a/jobs/expirynotify/scheduler.go b/jobs/expirynotify/scheduler.go index bc24952655..023e0c0f26 100644 --- a/jobs/expirynotify/scheduler.go +++ b/jobs/expirynotify/scheduler.go @@ -6,46 +6,15 @@ package expirynotify import ( "time" - "github.com/mattermost/mattermost-server/v6/app" + "github.com/mattermost/mattermost-server/v6/jobs" "github.com/mattermost/mattermost-server/v6/model" ) -const ( - SchedFreqMinutes = 10 -) +const schedFreq = 10 * time.Minute -type Scheduler struct { - App *app.App -} - -func (m *ExpiryNotifyJobInterfaceImpl) MakeScheduler() model.Scheduler { - return &Scheduler{m.App} -} - -func (scheduler *Scheduler) Name() string { - return JobName + "Scheduler" -} - -func (scheduler *Scheduler) JobType() string { - return model.JobTypeExpiryNotify -} - -func (scheduler *Scheduler) Enabled(cfg *model.Config) bool { - // Only enabled when ExtendSessionLengthWithActivity is enabled. - return *cfg.ServiceSettings.ExtendSessionLengthWithActivity -} - -func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time { - nextTime := time.Now().Add(SchedFreqMinutes * time.Minute) - return &nextTime -} - -func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) { - data := map[string]string{} - - job, err := scheduler.App.Srv().Jobs.CreateJob(model.JobTypeExpiryNotify, data) - if err != nil { - return nil, err +func MakeScheduler(jobServer *jobs.JobServer) model.Scheduler { + isEnabled := func(cfg *model.Config) bool { + return *cfg.ServiceSettings.ExtendSessionLengthWithActivity } - return job, nil + return jobs.NewPeriodicScheduler(jobServer, model.JobTypeExpiryNotify, schedFreq, isEnabled) } diff --git a/jobs/expirynotify/worker.go b/jobs/expirynotify/worker.go index 37c8f10943..98b9a0b47b 100644 --- a/jobs/expirynotify/worker.go +++ b/jobs/expirynotify/worker.go @@ -4,97 +4,20 @@ package expirynotify import ( - "github.com/mattermost/mattermost-server/v6/app" "github.com/mattermost/mattermost-server/v6/jobs" "github.com/mattermost/mattermost-server/v6/model" - "github.com/mattermost/mattermost-server/v6/shared/mlog" ) const ( JobName = "ExpiryNotify" ) -type Worker struct { - name string - stop chan bool - stopped chan bool - jobs chan model.Job - jobServer *jobs.JobServer - app *app.App -} - -func (m *ExpiryNotifyJobInterfaceImpl) MakeWorker() model.Worker { - worker := Worker{ - name: JobName, - stop: make(chan bool, 1), - stopped: make(chan bool, 1), - jobs: make(chan model.Job), - jobServer: m.App.Srv().Jobs, - app: m.App, +func MakeWorker(jobServer *jobs.JobServer, notifySessionsExpired func() error) model.Worker { + isEnabled := func(cfg *model.Config) bool { + return *cfg.ServiceSettings.ExtendSessionLengthWithActivity } - return &worker -} - -func (worker *Worker) Run() { - mlog.Debug("Worker started", mlog.String("worker", worker.name)) - - defer func() { - mlog.Debug("Worker finished", mlog.String("worker", worker.name)) - worker.stopped <- true - }() - - for { - select { - case <-worker.stop: - mlog.Debug("Worker received stop signal", mlog.String("worker", worker.name)) - return - case job := <-worker.jobs: - mlog.Debug("Worker received a new candidate job.", mlog.String("worker", worker.name)) - worker.DoJob(&job) - } - } -} - -func (worker *Worker) Stop() { - mlog.Debug("Worker stopping", mlog.String("worker", worker.name)) - worker.stop <- true - <-worker.stopped -} - -func (worker *Worker) JobChannel() chan<- model.Job { - return worker.jobs -} - -func (worker *Worker) DoJob(job *model.Job) { - if claimed, err := worker.jobServer.ClaimJob(job); err != nil { - mlog.Warn("Worker experienced an error while trying to claim job", - mlog.String("worker", worker.name), - mlog.String("job_id", job.Id), - mlog.String("error", err.Error())) - return - } else if !claimed { - return - } - - if err := worker.app.NotifySessionsExpired(); err != nil { - mlog.Error("Worker: Failed to notify clients of expired session", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) - worker.setJobError(job, err) - return - } - - mlog.Info("Worker: Job is complete", mlog.String("worker", worker.name), mlog.String("job_id", job.Id)) - worker.setJobSuccess(job) -} - -func (worker *Worker) setJobSuccess(job *model.Job) { - if err := worker.app.Srv().Jobs.SetJobSuccess(job); err != nil { - mlog.Error("Worker: Failed to set success for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) - worker.setJobError(job, err) - } -} - -func (worker *Worker) setJobError(job *model.Job, appError *model.AppError) { - if err := worker.app.Srv().Jobs.SetJobError(job, appError); err != nil { - mlog.Error("Worker: Failed to set job error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) + execute := func(job *model.Job) error { + return notifySessionsExpired() } + return jobs.NewSimpleWorker(JobName, jobServer, execute, isEnabled) } diff --git a/jobs/export_delete/scheduler.go b/jobs/export_delete/scheduler.go index 9f06efdbe1..f15f7339e4 100644 --- a/jobs/export_delete/scheduler.go +++ b/jobs/export_delete/scheduler.go @@ -6,46 +6,15 @@ package export_delete import ( "time" - "github.com/mattermost/mattermost-server/v6/app" + "github.com/mattermost/mattermost-server/v6/jobs" "github.com/mattermost/mattermost-server/v6/model" ) -const ( - jobName = "ExportDelete" - schedFrequency = 24 * time.Hour -) +const schedFreq = 24 * time.Hour -type Scheduler struct { - app *app.App -} - -func (i *ExportDeleteInterfaceImpl) MakeScheduler() model.Scheduler { - return &Scheduler{i.app} -} - -func (scheduler *Scheduler) Name() string { - return jobName + "Scheduler" -} - -func (scheduler *Scheduler) JobType() string { - return model.JobTypeExportDelete -} - -func (scheduler *Scheduler) Enabled(cfg *model.Config) bool { - return *cfg.ExportSettings.Directory != "" && *cfg.ExportSettings.RetentionDays > 0 -} - -func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time { - nextTime := time.Now().Add(schedFrequency) - return &nextTime -} - -func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) { - data := map[string]string{} - - job, err := scheduler.app.Srv().Jobs.CreateJob(model.JobTypeExportDelete, data) - if err != nil { - return nil, err +func MakeScheduler(jobServer *jobs.JobServer) model.Scheduler { + isEnabled := func(cfg *model.Config) bool { + return *cfg.ExportSettings.Directory != "" && *cfg.ExportSettings.RetentionDays > 0 } - return job, nil + return jobs.NewPeriodicScheduler(jobServer, model.JobTypeExportDelete, schedFreq, isEnabled) } diff --git a/jobs/export_delete/worker.go b/jobs/export_delete/worker.go index 099f813c72..8a8c509426 100644 --- a/jobs/export_delete/worker.go +++ b/jobs/export_delete/worker.go @@ -7,132 +7,61 @@ import ( "path/filepath" "time" - "github.com/mattermost/mattermost-server/v6/app" "github.com/mattermost/mattermost-server/v6/jobs" - tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces" "github.com/mattermost/mattermost-server/v6/model" + "github.com/mattermost/mattermost-server/v6/services/configservice" "github.com/mattermost/mattermost-server/v6/shared/mlog" + "github.com/wiggin77/merror" ) -func init() { - app.RegisterJobsExportDeleteInterface(func(s *app.Server) tjobs.ExportDeleteInterface { - a := app.New(app.ServerConnector(s.Channels())) - return &ExportDeleteInterfaceImpl{a} - }) +const jobName = "ExportDelete" + +type AppIface interface { + configservice.ConfigService + ListDirectory(path string) ([]string, *model.AppError) + FileModTime(path string) (time.Time, *model.AppError) + RemoveFile(path string) *model.AppError } -type ExportDeleteInterfaceImpl struct { - app *app.App -} - -type ExportDeleteWorker struct { - name string - stopChan chan struct{} - stoppedChan chan struct{} - jobsChan chan model.Job - jobServer *jobs.JobServer - app *app.App -} - -func (i *ExportDeleteInterfaceImpl) MakeWorker() model.Worker { - return &ExportDeleteWorker{ - name: "ExportDelete", - stopChan: make(chan struct{}), - stoppedChan: make(chan struct{}), - jobsChan: make(chan model.Job), - jobServer: i.app.Srv().Jobs, - app: i.app, +func MakeWorker(jobServer *jobs.JobServer, app AppIface) model.Worker { + isEnabled := func(cfg *model.Config) bool { + return *cfg.ExportSettings.Directory != "" && *cfg.ExportSettings.RetentionDays > 0 } -} - -func (w *ExportDeleteWorker) JobChannel() chan<- model.Job { - return w.jobsChan -} - -func (w *ExportDeleteWorker) Run() { - mlog.Debug("Worker started", mlog.String("worker", w.name)) - - defer func() { - mlog.Debug("Worker finished", mlog.String("worker", w.name)) - close(w.stoppedChan) - }() - - for { - select { - case <-w.stopChan: - mlog.Debug("Worker received stop signal", mlog.String("worker", w.name)) - return - case job := <-w.jobsChan: - mlog.Debug("Worker received a new candidate job.", mlog.String("worker", w.name)) - w.doJob(&job) - } - } -} - -func (w *ExportDeleteWorker) Stop() { - mlog.Debug("Worker stopping", mlog.String("worker", w.name)) - close(w.stopChan) - <-w.stoppedChan -} - -func (w *ExportDeleteWorker) doJob(job *model.Job) { - if claimed, err := w.jobServer.ClaimJob(job); err != nil { - mlog.Warn("Worker experienced an error while trying to claim job", - mlog.String("worker", w.name), - mlog.String("job_id", job.Id), - mlog.String("error", err.Error())) - return - } else if !claimed { - return - } - - exportPath := *w.app.Config().ExportSettings.Directory - retentionTime := time.Duration(*w.app.Config().ExportSettings.RetentionDays) * 24 * time.Hour - exports, appErr := w.app.ListDirectory(exportPath) - if appErr != nil { - w.setJobError(job, appErr) - return - } - - var hasErrs bool - for i := range exports { - filename := filepath.Base(exports[i]) - modTime, appErr := w.app.FileModTime(filepath.Join(exportPath, filename)) + execute := func(job *model.Job) error { + exportPath := *app.Config().ExportSettings.Directory + retentionTime := time.Duration(*app.Config().ExportSettings.RetentionDays) * 24 * time.Hour + exports, appErr := app.ListDirectory(exportPath) if appErr != nil { - mlog.Debug("Worker: Failed to get file modification time", - mlog.Err(appErr), mlog.String("export", exports[i])) - hasErrs = true - continue + return appErr } - if time.Now().After(modTime.Add(retentionTime)) { - // remove file data from storage. - if appErr := w.app.RemoveFile(exports[i]); appErr != nil { - mlog.Debug("Worker: Failed to remove file", + errors := merror.New() + for i := range exports { + filename := filepath.Base(exports[i]) + modTime, appErr := app.FileModTime(filepath.Join(exportPath, filename)) + if appErr != nil { + mlog.Debug("Worker: Failed to get file modification time", mlog.Err(appErr), mlog.String("export", exports[i])) - hasErrs = true + errors.Append(appErr) continue } + + if time.Now().After(modTime.Add(retentionTime)) { + // remove file data from storage. + if appErr := app.RemoveFile(exports[i]); appErr != nil { + mlog.Debug("Worker: Failed to remove file", + mlog.Err(appErr), mlog.String("export", exports[i])) + errors.Append(appErr) + continue + } + } } - } - if hasErrs { - mlog.Warn("Worker: errors occurred") - } - - mlog.Info("Worker: Job is complete", mlog.String("worker", w.name), mlog.String("job_id", job.Id)) - w.setJobSuccess(job) -} - -func (w *ExportDeleteWorker) setJobSuccess(job *model.Job) { - if err := w.app.Srv().Jobs.SetJobSuccess(job); err != nil { - mlog.Error("Worker: Failed to set success for job", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) - w.setJobError(job, err) - } -} - -func (w *ExportDeleteWorker) setJobError(job *model.Job, appError *model.AppError) { - if err := w.app.Srv().Jobs.SetJobError(job, appError); err != nil { - mlog.Error("Worker: Failed to set job error", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) + if err := errors.ErrorOrNil(); err != nil { + mlog.Warn("Worker: errors occurred", mlog.String("job-name", jobName), mlog.Err(err)) + } + return nil } + worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled) + return worker } diff --git a/jobs/export_process/worker.go b/jobs/export_process/worker.go index 14e5e9c7d3..e37e5798ad 100644 --- a/jobs/export_process/worker.go +++ b/jobs/export_process/worker.go @@ -7,133 +7,58 @@ import ( "io" "path/filepath" - "github.com/mattermost/mattermost-server/v6/app" "github.com/mattermost/mattermost-server/v6/jobs" - tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces" "github.com/mattermost/mattermost-server/v6/model" + "github.com/mattermost/mattermost-server/v6/services/configservice" "github.com/mattermost/mattermost-server/v6/shared/mlog" ) -func init() { - app.RegisterJobsExportProcessInterface(func(s *app.Server) tjobs.ExportProcessInterface { - a := app.New(app.ServerConnector(s.Channels())) - return &ExportProcessInterfaceImpl{a} - }) +const jobName = "ExportProcess" + +type AppIface interface { + configservice.ConfigService + WriteFile(fr io.Reader, path string) (int64, *model.AppError) + BulkExport(writer io.Writer, outPath string, opts model.BulkExportOpts) *model.AppError } -type ExportProcessInterfaceImpl struct { - app *app.App -} - -type ExportProcessWorker struct { - name string - stopChan chan struct{} - stoppedChan chan struct{} - jobsChan chan model.Job - jobServer *jobs.JobServer - app *app.App -} - -func (i *ExportProcessInterfaceImpl) MakeWorker() model.Worker { - return &ExportProcessWorker{ - name: "ExportProcess", - stopChan: make(chan struct{}), - stoppedChan: make(chan struct{}), - jobsChan: make(chan model.Job), - jobServer: i.app.Srv().Jobs, - app: i.app, - } -} - -func (w *ExportProcessWorker) JobChannel() chan<- model.Job { - return w.jobsChan -} - -func (w *ExportProcessWorker) Run() { - mlog.Debug("Worker started", mlog.String("worker", w.name)) - - defer func() { - mlog.Debug("Worker finished", mlog.String("worker", w.name)) - close(w.stoppedChan) - }() - - for { - select { - case <-w.stopChan: - mlog.Debug("Worker received stop signal", mlog.String("worker", w.name)) - return - case job := <-w.jobsChan: - mlog.Debug("Worker received a new candidate job.", mlog.String("worker", w.name)) - w.doJob(&job) +func MakeWorker(jobServer *jobs.JobServer, app AppIface) model.Worker { + isEnabled := func(cfg *model.Config) bool { return true } + execute := func(job *model.Job) error { + opts := model.BulkExportOpts{ + CreateArchive: true, } + + includeAttachments, ok := job.Data["include_attachments"] + if ok && includeAttachments == "true" { + opts.IncludeAttachments = true + } + + outPath := *app.Config().ExportSettings.Directory + exportFilename := model.NewId() + "_export.zip" + + rd, wr := io.Pipe() + + errCh := make(chan *model.AppError, 1) + go func() { + defer close(errCh) + _, appErr := app.WriteFile(rd, filepath.Join(outPath, exportFilename)) + errCh <- appErr + }() + + appErr := app.BulkExport(wr, outPath, opts) + if err := wr.Close(); err != nil { + mlog.Warn("Worker: error closing writer") + } + + if appErr != nil { + return appErr + } + + if appErr := <-errCh; appErr != nil { + return appErr + } + return nil } -} - -func (w *ExportProcessWorker) Stop() { - mlog.Debug("Worker stopping", mlog.String("worker", w.name)) - close(w.stopChan) - <-w.stoppedChan -} - -func (w *ExportProcessWorker) doJob(job *model.Job) { - if claimed, err := w.jobServer.ClaimJob(job); err != nil { - mlog.Warn("Worker experienced an error while trying to claim job", - mlog.String("worker", w.name), - mlog.String("job_id", job.Id), - mlog.String("error", err.Error())) - return - } else if !claimed { - return - } - - opts := app.BulkExportOpts{ - CreateArchive: true, - } - - includeAttachments, ok := job.Data["include_attachments"] - if ok && includeAttachments == "true" { - opts.IncludeAttachments = true - } - - outPath := *w.app.Config().ExportSettings.Directory - exportFilename := model.NewId() + "_export.zip" - - rd, wr := io.Pipe() - - errCh := make(chan *model.AppError, 1) - go func() { - defer close(errCh) - _, appErr := w.app.WriteFile(rd, filepath.Join(outPath, exportFilename)) - errCh <- appErr - }() - - appErr := w.app.BulkExport(wr, outPath, opts) - if err := wr.Close(); err != nil { - mlog.Warn("Worker: error closing writer") - } - if appErr != nil { - w.setJobError(job, appErr) - return - } - - if appErr := <-errCh; appErr != nil { - w.setJobError(job, appErr) - return - } - - mlog.Info("Worker: Job is complete", mlog.String("worker", w.name), mlog.String("job_id", job.Id)) - w.setJobSuccess(job) -} - -func (w *ExportProcessWorker) setJobSuccess(job *model.Job) { - if err := w.app.Srv().Jobs.SetJobSuccess(job); err != nil { - mlog.Error("Worker: Failed to set success for job", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) - w.setJobError(job, err) - } -} - -func (w *ExportProcessWorker) setJobError(job *model.Job, appError *model.AppError) { - if err := w.app.Srv().Jobs.SetJobError(job, appError); err != nil { - mlog.Error("Worker: Failed to set job error", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) - } + worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled) + return worker } diff --git a/jobs/extract_content/worker.go b/jobs/extract_content/worker.go index 2352e024a3..c2eb9ad0af 100644 --- a/jobs/extract_content/worker.go +++ b/jobs/extract_content/worker.go @@ -4,15 +4,12 @@ package extract_content import ( - "net/http" "strconv" - "github.com/mattermost/mattermost-server/v6/app" - "github.com/mattermost/mattermost-server/v6/app/request" "github.com/mattermost/mattermost-server/v6/jobs" - tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces" "github.com/mattermost/mattermost-server/v6/model" "github.com/mattermost/mattermost-server/v6/shared/mlog" + "github.com/mattermost/mattermost-server/v6/store" ) var ignoredFiles = map[string]bool{ @@ -22,155 +19,74 @@ var ignoredFiles = map[string]bool{ "mkv": true, } -func init() { - app.RegisterJobsExtractContentInterface(func(s *app.Server) tjobs.ExtractContentInterface { - a := app.New(app.ServerConnector(s.Channels())) - return &ExtractContentInterfaceImpl{a} - }) +const jobName = "ExtractContent" + +type AppIface interface { + ExtractContentFromFileInfo(fileInfo *model.FileInfo) error } -type ExtractContentInterfaceImpl struct { - app *app.App -} - -type ExtractContentWorker struct { - name string - stopChan chan struct{} - stoppedChan chan struct{} - jobsChan chan model.Job - jobServer *jobs.JobServer - app *app.App - appContext *request.Context -} - -func (i *ExtractContentInterfaceImpl) MakeWorker() model.Worker { - return &ExtractContentWorker{ - name: "ExtractContent", - stopChan: make(chan struct{}), - stoppedChan: make(chan struct{}), - jobsChan: make(chan model.Job), - jobServer: i.app.Srv().Jobs, - app: i.app, - appContext: &request.Context{}, +func MakeWorker(jobServer *jobs.JobServer, app AppIface, store store.Store) model.Worker { + isEnabled := func(cfg *model.Config) bool { + return true } -} - -func (w *ExtractContentWorker) JobChannel() chan<- model.Job { - return w.jobsChan -} - -func (w *ExtractContentWorker) Run() { - mlog.Debug("Worker started", mlog.String("worker", w.name)) - - defer func() { - mlog.Debug("Worker finished", mlog.String("worker", w.name)) - close(w.stoppedChan) - }() - - for { - select { - case <-w.stopChan: - mlog.Debug("Worker received stop signal", mlog.String("worker", w.name)) - return - case job := <-w.jobsChan: - mlog.Debug("Worker received a new candidate job.", mlog.String("worker", w.name)) - w.doJob(&job) - } - } -} - -func (w *ExtractContentWorker) Stop() { - mlog.Debug("Worker stopping", mlog.String("worker", w.name)) - close(w.stopChan) - <-w.stoppedChan -} - -func (w *ExtractContentWorker) doJob(job *model.Job) { - if claimed, err := w.jobServer.ClaimJob(job); err != nil { - mlog.Warn("Worker experienced an error while trying to claim job", - mlog.String("worker", w.name), - mlog.String("job_id", job.Id), - mlog.String("error", err.Error())) - return - } else if !claimed { - return - } - - var err error - var fromTS int64 = 0 - var toTS int64 = model.GetMillis() - if fromStr, ok := job.Data["from"]; ok { - if fromTS, err = strconv.ParseInt(fromStr, 10, 64); err != nil { - w.setJobError(job, model.NewAppError("ExtractContentWorker", "extrac_content.worker.do_job.invalid_input.from", nil, "", http.StatusBadRequest)) - return - } - fromTS *= 1000 - } - if toStr, ok := job.Data["to"]; ok { - if toTS, err = strconv.ParseInt(toStr, 10, 64); err != nil { - w.setJobError(job, model.NewAppError("ExtractContentWorker", "extrac_content.worker.do_job.invalid_input.to", nil, "", http.StatusBadRequest)) - return - } - toTS *= 1000 - } - - var nFiles int - var nErrs int - for { - opts := model.GetFileInfosOptions{ - Since: fromTS, - SortBy: model.FileinfoSortByCreated, - IncludeDeleted: false, - } - fileInfos, err := w.app.Srv().Store.FileInfo().GetWithOptions(0, 1000, &opts) - if err != nil { - w.setJobError(job, model.NewAppError("ExtractContentWorker", "extract_content.worker.do_job.file_info", nil, err.Error(), http.StatusInternalServerError)) - return - } - if len(fileInfos) == 0 { - break - } - for _, fileInfo := range fileInfos { - if !ignoredFiles[fileInfo.Extension] { - mlog.Debug("extracting file", mlog.String("filename", fileInfo.Name), mlog.String("filepath", fileInfo.Path)) - err = w.app.ExtractContentFromFileInfo(fileInfo) - if err != nil { - mlog.Warn("Failed to extract file content", mlog.Err(err), mlog.String("file_info_id", fileInfo.Id)) - nErrs++ - } - nFiles++ + execute := func(job *model.Job) error { + var err error + var fromTS int64 = 0 + var toTS int64 = model.GetMillis() + if fromStr, ok := job.Data["from"]; ok { + if fromTS, err = strconv.ParseInt(fromStr, 10, 64); err != nil { + return err } + fromTS *= 1000 } - lastFileInfo := fileInfos[len(fileInfos)-1] - if lastFileInfo.CreateAt > toTS { - break + if toStr, ok := job.Data["to"]; ok { + if toTS, err = strconv.ParseInt(toStr, 10, 64); err != nil { + return err + } + toTS *= 1000 } - fromTS = lastFileInfo.CreateAt + 1 - } - - job.Data["errors"] = strconv.Itoa(nErrs) - job.Data["processed"] = strconv.Itoa(nFiles) - w.updateData(job) - - mlog.Info("Worker: Job is complete", mlog.String("worker", w.name), mlog.String("job_id", job.Id)) - w.setJobSuccess(job) -} - -func (w *ExtractContentWorker) setJobSuccess(job *model.Job) { - if err := w.app.Srv().Jobs.SetJobSuccess(job); err != nil { - mlog.Error("Worker: Failed to set success for job", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) - w.setJobError(job, err) - } -} - -func (w *ExtractContentWorker) setJobError(job *model.Job, appError *model.AppError) { - if err := w.app.Srv().Jobs.SetJobError(job, appError); err != nil { - mlog.Error("Worker: Failed to set job error", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) - } -} - -func (w *ExtractContentWorker) updateData(job *model.Job) { - if err := w.app.Srv().Jobs.UpdateInProgressJobData(job); err != nil { - mlog.Error("Worker: Failed to update job data", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) + + var nFiles int + var nErrs int + for { + opts := model.GetFileInfosOptions{ + Since: fromTS, + SortBy: model.FileinfoSortByCreated, + IncludeDeleted: false, + } + fileInfos, err := store.FileInfo().GetWithOptions(0, 1000, &opts) + if err != nil { + return err + } + if len(fileInfos) == 0 { + break + } + for _, fileInfo := range fileInfos { + if !ignoredFiles[fileInfo.Extension] { + mlog.Debug("extracting file", mlog.String("filename", fileInfo.Name), mlog.String("filepath", fileInfo.Path)) + err = app.ExtractContentFromFileInfo(fileInfo) + if err != nil { + mlog.Warn("Failed to extract file content", mlog.Err(err), mlog.String("file_info_id", fileInfo.Id)) + nErrs++ + } + nFiles++ + } + } + lastFileInfo := fileInfos[len(fileInfos)-1] + if lastFileInfo.CreateAt > toTS { + break + } + fromTS = lastFileInfo.CreateAt + 1 + } + + job.Data["errors"] = strconv.Itoa(nErrs) + job.Data["processed"] = strconv.Itoa(nFiles) + + if err := jobServer.UpdateInProgressJobData(job); err != nil { + mlog.Error("Worker: Failed to update job data", mlog.String("worker", model.JobTypeExtractContent), mlog.String("job_id", job.Id), mlog.Err(err)) + } + return nil } + worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled) + return worker } diff --git a/jobs/import_delete/scheduler.go b/jobs/import_delete/scheduler.go index 5d3e9b95e1..e76f9dc133 100644 --- a/jobs/import_delete/scheduler.go +++ b/jobs/import_delete/scheduler.go @@ -6,46 +6,15 @@ package import_delete import ( "time" - "github.com/mattermost/mattermost-server/v6/app" + "github.com/mattermost/mattermost-server/v6/jobs" "github.com/mattermost/mattermost-server/v6/model" ) -const ( - jobName = "ImportDelete" - schedFrequency = 24 * time.Hour -) +const schedFreq = 24 * time.Hour -type Scheduler struct { - app *app.App -} - -func (i *ImportDeleteInterfaceImpl) MakeScheduler() model.Scheduler { - return &Scheduler{i.app} -} - -func (scheduler *Scheduler) Name() string { - return jobName + "Scheduler" -} - -func (scheduler *Scheduler) JobType() string { - return model.JobTypeImportDelete -} - -func (scheduler *Scheduler) Enabled(cfg *model.Config) bool { - return *cfg.ImportSettings.Directory != "" && *cfg.ImportSettings.RetentionDays > 0 -} - -func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time { - nextTime := time.Now().Add(schedFrequency) - return &nextTime -} - -func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) { - data := map[string]string{} - - job, err := scheduler.app.Srv().Jobs.CreateJob(model.JobTypeImportDelete, data) - if err != nil { - return nil, err +func MakeScheduler(jobServer *jobs.JobServer) model.Scheduler { + isEnabled := func(cfg *model.Config) bool { + return *cfg.ImportSettings.Directory != "" && *cfg.ImportSettings.RetentionDays > 0 } - return job, nil + return jobs.NewPeriodicScheduler(jobServer, model.JobTypeImportDelete, schedFreq, isEnabled) } diff --git a/jobs/import_delete/worker.go b/jobs/import_delete/worker.go index 7ef420baf8..6dc46b4ae8 100644 --- a/jobs/import_delete/worker.go +++ b/jobs/import_delete/worker.go @@ -8,166 +8,95 @@ import ( "path/filepath" "time" - "github.com/mattermost/mattermost-server/v6/app" "github.com/mattermost/mattermost-server/v6/jobs" - tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces" "github.com/mattermost/mattermost-server/v6/model" + "github.com/mattermost/mattermost-server/v6/services/configservice" "github.com/mattermost/mattermost-server/v6/shared/mlog" "github.com/mattermost/mattermost-server/v6/store" + "github.com/wiggin77/merror" ) -func init() { - app.RegisterJobsImportDeleteInterface(func(s *app.Server) tjobs.ImportDeleteInterface { - a := app.New(app.ServerConnector(s.Channels())) - return &ImportDeleteInterfaceImpl{a} - }) +const jobName = "ImportDelete" + +type AppIface interface { + configservice.ConfigService + ListDirectory(path string) ([]string, *model.AppError) + FileModTime(path string) (time.Time, *model.AppError) + RemoveFile(path string) *model.AppError } -type ImportDeleteInterfaceImpl struct { - app *app.App -} - -type ImportDeleteWorker struct { - name string - stopChan chan struct{} - stoppedChan chan struct{} - jobsChan chan model.Job - jobServer *jobs.JobServer - app *app.App -} - -func (i *ImportDeleteInterfaceImpl) MakeWorker() model.Worker { - return &ImportDeleteWorker{ - name: "ImportDelete", - stopChan: make(chan struct{}), - stoppedChan: make(chan struct{}), - jobsChan: make(chan model.Job), - jobServer: i.app.Srv().Jobs, - app: i.app, +func MakeWorker(jobServer *jobs.JobServer, app AppIface, s store.Store) model.Worker { + isEnabled := func(cfg *model.Config) bool { + return *cfg.ImportSettings.Directory != "" && *cfg.ImportSettings.RetentionDays > 0 } -} - -func (w *ImportDeleteWorker) JobChannel() chan<- model.Job { - return w.jobsChan -} - -func (w *ImportDeleteWorker) Run() { - mlog.Debug("Worker started", mlog.String("worker", w.name)) - - defer func() { - mlog.Debug("Worker finished", mlog.String("worker", w.name)) - close(w.stoppedChan) - }() - - for { - select { - case <-w.stopChan: - mlog.Debug("Worker received stop signal", mlog.String("worker", w.name)) - return - case job := <-w.jobsChan: - mlog.Debug("Worker received a new candidate job.", mlog.String("worker", w.name)) - w.doJob(&job) - } - } -} - -func (w *ImportDeleteWorker) Stop() { - mlog.Debug("Worker stopping", mlog.String("worker", w.name)) - close(w.stopChan) - <-w.stoppedChan -} - -func (w *ImportDeleteWorker) doJob(job *model.Job) { - if claimed, err := w.jobServer.ClaimJob(job); err != nil { - mlog.Warn("Worker experienced an error while trying to claim job", - mlog.String("worker", w.name), - mlog.String("job_id", job.Id), - mlog.String("error", err.Error())) - return - } else if !claimed { - return - } - - importPath := *w.app.Config().ImportSettings.Directory - retentionTime := time.Duration(*w.app.Config().ImportSettings.RetentionDays) * 24 * time.Hour - imports, appErr := w.app.ListDirectory(importPath) - if appErr != nil { - w.setJobError(job, appErr) - return - } - - var hasErrs bool - for i := range imports { - filename := filepath.Base(imports[i]) - modTime, appErr := w.app.FileModTime(filepath.Join(importPath, filename)) + execute := func(job *model.Job) error { + importPath := *app.Config().ImportSettings.Directory + retentionTime := time.Duration(*app.Config().ImportSettings.RetentionDays) * 24 * time.Hour + imports, appErr := app.ListDirectory(importPath) if appErr != nil { - mlog.Debug("Worker: Failed to get file modification time", - mlog.Err(appErr), mlog.String("import", imports[i])) - hasErrs = true - continue + return appErr } - if time.Now().After(modTime.Add(retentionTime)) { - // expected format if uploaded through the API is - // ${uploadID}_${filename}${app.IncompleteUploadSuffix} - minLen := 26 + 1 + len(app.IncompleteUploadSuffix) - - // check if it's an incomplete upload and attempt to delete its session. - if len(filename) > minLen && filepath.Ext(filename) == app.IncompleteUploadSuffix { - uploadID := filename[:26] - if storeErr := w.app.Srv().Store.UploadSession().Delete(uploadID); storeErr != nil { - mlog.Debug("Worker: Failed to delete UploadSession", - mlog.Err(storeErr), mlog.String("upload_id", uploadID)) - hasErrs = true - continue - } - } else { - // check if fileinfo exists and if so delete it. - filePath := filepath.Join(imports[i]) - info, storeErr := w.app.Srv().Store.FileInfo().GetByPath(filePath) - var nfErr *store.ErrNotFound - if storeErr != nil && !errors.As(storeErr, &nfErr) { - mlog.Debug("Worker: Failed to get FileInfo", - mlog.Err(storeErr), mlog.String("path", filePath)) - hasErrs = true - continue - } else if storeErr == nil { - if storeErr = w.app.Srv().Store.FileInfo().PermanentDelete(info.Id); storeErr != nil { - mlog.Debug("Worker: Failed to delete FileInfo", - mlog.Err(storeErr), mlog.String("file_id", info.Id)) - hasErrs = true - continue - } - } - } - - // remove file data from storage. - if appErr := w.app.RemoveFile(imports[i]); appErr != nil { - mlog.Debug("Worker: Failed to remove file", + multipleErrors := merror.New() + for i := range imports { + filename := filepath.Base(imports[i]) + modTime, appErr := app.FileModTime(filepath.Join(importPath, filename)) + if appErr != nil { + mlog.Debug("Worker: Failed to get file modification time", mlog.Err(appErr), mlog.String("import", imports[i])) - hasErrs = true + multipleErrors.Append(appErr) continue } + + if time.Now().After(modTime.Add(retentionTime)) { + // expected format if uploaded through the API is + // ${uploadID}_${filename}${model.IncompleteUploadSuffix} + minLen := 26 + 1 + len(model.IncompleteUploadSuffix) + + // check if it's an incomplete upload and attempt to delete its session. + if len(filename) > minLen && filepath.Ext(filename) == model.IncompleteUploadSuffix { + uploadID := filename[:26] + if storeErr := s.UploadSession().Delete(uploadID); storeErr != nil { + mlog.Debug("Worker: Failed to delete UploadSession", + mlog.Err(storeErr), mlog.String("upload_id", uploadID)) + multipleErrors.Append(storeErr) + continue + } + } else { + // check if fileinfo exists and if so delete it. + filePath := filepath.Join(imports[i]) + info, storeErr := s.FileInfo().GetByPath(filePath) + var nfErr *store.ErrNotFound + if storeErr != nil && !errors.As(storeErr, &nfErr) { + mlog.Debug("Worker: Failed to get FileInfo", + mlog.Err(storeErr), mlog.String("path", filePath)) + multipleErrors.Append(storeErr) + continue + } else if storeErr == nil { + if storeErr = s.FileInfo().PermanentDelete(info.Id); storeErr != nil { + mlog.Debug("Worker: Failed to delete FileInfo", + mlog.Err(storeErr), mlog.String("file_id", info.Id)) + multipleErrors.Append(storeErr) + continue + } + } + } + + // remove file data from storage. + if appErr := app.RemoveFile(imports[i]); appErr != nil { + mlog.Debug("Worker: Failed to remove file", + mlog.Err(appErr), mlog.String("import", imports[i])) + multipleErrors.Append(appErr) + continue + } + } } - } - if hasErrs { - mlog.Warn("Worker: errors occurred") - } - - mlog.Info("Worker: Job is complete", mlog.String("worker", w.name), mlog.String("job_id", job.Id)) - w.setJobSuccess(job) -} - -func (w *ImportDeleteWorker) setJobSuccess(job *model.Job) { - if err := w.app.Srv().Jobs.SetJobSuccess(job); err != nil { - mlog.Error("Worker: Failed to set success for job", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) - w.setJobError(job, err) - } -} - -func (w *ImportDeleteWorker) setJobError(job *model.Job, appError *model.AppError) { - if err := w.app.Srv().Jobs.SetJobError(job, appError); err != nil { - mlog.Error("Worker: Failed to set job error", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) + if err := multipleErrors.ErrorOrNil(); err != nil { + mlog.Warn("Worker: errors occurred", mlog.String("job-name", jobName), mlog.Err(err)) + } + return nil } + worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled) + return worker } diff --git a/jobs/import_process/worker.go b/jobs/import_process/worker.go index a7c284af66..77c0fabe5f 100644 --- a/jobs/import_process/worker.go +++ b/jobs/import_process/worker.go @@ -12,182 +12,95 @@ import ( "strconv" "strings" - "github.com/mattermost/mattermost-server/v6/app" "github.com/mattermost/mattermost-server/v6/app/request" "github.com/mattermost/mattermost-server/v6/jobs" - tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces" "github.com/mattermost/mattermost-server/v6/model" - "github.com/mattermost/mattermost-server/v6/shared/mlog" + "github.com/mattermost/mattermost-server/v6/services/configservice" + "github.com/mattermost/mattermost-server/v6/shared/filestore" ) -func init() { - app.RegisterJobsImportProcessInterface(func(s *app.Server) tjobs.ImportProcessInterface { - a := app.New(app.ServerConnector(s.Channels())) - return &ImportProcessInterfaceImpl{a} - }) +const jobName = "ImportProcess" + +type AppIface interface { + configservice.ConfigService + RemoveFile(path string) *model.AppError + FileExists(path string) (bool, *model.AppError) + FileSize(path string) (int64, *model.AppError) + FileReader(path string) (filestore.ReadCloseSeeker, *model.AppError) + BulkImportWithPath(c *request.Context, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun bool, workers int, importPath string) (*model.AppError, int) } -type ImportProcessInterfaceImpl struct { - app *app.App -} - -type ImportProcessWorker struct { - name string - stopChan chan struct{} - stoppedChan chan struct{} - jobsChan chan model.Job - jobServer *jobs.JobServer - app *app.App - appContext *request.Context -} - -func (i *ImportProcessInterfaceImpl) MakeWorker() model.Worker { - return &ImportProcessWorker{ - name: "ImportProcess", - stopChan: make(chan struct{}), - stoppedChan: make(chan struct{}), - jobsChan: make(chan model.Job), - jobServer: i.app.Srv().Jobs, - app: i.app, - appContext: &request.Context{}, +func MakeWorker(jobServer *jobs.JobServer, app AppIface) model.Worker { + appContext := &request.Context{} + isEnabled := func(cfg *model.Config) bool { + return true } -} - -func (w *ImportProcessWorker) JobChannel() chan<- model.Job { - return w.jobsChan -} - -func (w *ImportProcessWorker) Run() { - mlog.Debug("Worker started", mlog.String("worker", w.name)) - - defer func() { - mlog.Debug("Worker finished", mlog.String("worker", w.name)) - close(w.stoppedChan) - }() - - for { - select { - case <-w.stopChan: - mlog.Debug("Worker received stop signal", mlog.String("worker", w.name)) - return - case job := <-w.jobsChan: - mlog.Debug("Worker received a new candidate job.", mlog.String("worker", w.name)) - w.doJob(&job) - } - } -} - -func (w *ImportProcessWorker) Stop() { - mlog.Debug("Worker stopping", mlog.String("worker", w.name)) - close(w.stopChan) - <-w.stoppedChan -} - -func (w *ImportProcessWorker) doJob(job *model.Job) { - if claimed, err := w.jobServer.ClaimJob(job); err != nil { - mlog.Warn("Worker experienced an error while trying to claim job", - mlog.String("worker", w.name), - mlog.String("job_id", job.Id), - mlog.String("error", err.Error())) - return - } else if !claimed { - return - } - - importFileName, ok := job.Data["import_file"] - if !ok { - appError := model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.missing_file", nil, "", http.StatusBadRequest) - w.setJobError(job, appError) - return - } - - importFilePath := filepath.Join(*w.app.Config().ImportSettings.Directory, importFileName) - if ok, err := w.app.FileExists(importFilePath); err != nil { - w.setJobError(job, err) - return - } else if !ok { - appError := model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.file_exists", nil, "", http.StatusBadRequest) - w.setJobError(job, appError) - return - } - - importFileSize, appErr := w.app.FileSize(importFilePath) - if appErr != nil { - w.setJobError(job, appErr) - return - } - - importFile, appErr := w.app.FileReader(importFilePath) - if appErr != nil { - w.setJobError(job, appErr) - return - } - defer importFile.Close() - - importZipReader, err := zip.NewReader(importFile.(io.ReaderAt), importFileSize) - if err != nil { - appError := model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, err.Error(), http.StatusInternalServerError) - w.setJobError(job, appError) - return - } - - // find JSONL import file. - var jsonFile io.ReadCloser - for _, f := range importZipReader.File { - if filepath.Ext(f.Name) != ".jsonl" { - continue - } - // avoid "zip slip" - if strings.Contains(f.Name, "..") { - appError := model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, "jsonFilePath contains path traversal", http.StatusForbidden) - w.setJobError(job, appError) - return + execute := func(job *model.Job) error { + importFileName, ok := job.Data["import_file"] + if !ok { + return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.missing_file", nil, "", http.StatusBadRequest) } - jsonFile, err = f.Open() + importFilePath := filepath.Join(*app.Config().ImportSettings.Directory, importFileName) + if ok, err := app.FileExists(importFilePath); err != nil { + return err + } else if !ok { + return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.file_exists", nil, "", http.StatusBadRequest) + } + + importFileSize, appErr := app.FileSize(importFilePath) + if appErr != nil { + return appErr + } + + importFile, appErr := app.FileReader(importFilePath) + if appErr != nil { + return appErr + } + defer importFile.Close() + + importZipReader, err := zip.NewReader(importFile.(io.ReaderAt), importFileSize) if err != nil { - appError := model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, err.Error(), http.StatusInternalServerError) - w.setJobError(job, appError) - return + return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, err.Error(), http.StatusInternalServerError) } - defer jsonFile.Close() - break - } + // find JSONL import file. + var jsonFile io.ReadCloser + for _, f := range importZipReader.File { + if filepath.Ext(f.Name) != ".jsonl" { + continue + } + // avoid "zip slip" + if strings.Contains(f.Name, "..") { + return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, "jsonFilePath contains path traversal", http.StatusForbidden) + } - if jsonFile == nil { - appError := model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.missing_jsonl", nil, "jsonFile was nil", http.StatusBadRequest) - w.setJobError(job, appError) - return - } + jsonFile, err = f.Open() + if err != nil { + return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, err.Error(), http.StatusInternalServerError) + } - // do the actual import. - appErr, lineNumber := w.app.BulkImportWithPath(w.appContext, jsonFile, importZipReader, false, runtime.NumCPU(), app.ExportDataDir) - if appErr != nil { - job.Data["line_number"] = strconv.Itoa(lineNumber) - w.setJobError(job, appErr) - return - } + defer jsonFile.Close() + break + } - // remove import file when done. - if appErr := w.app.RemoveFile(importFilePath); appErr != nil { - w.setJobError(job, appErr) - return - } + if jsonFile == nil { + return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.missing_jsonl", nil, "jsonFile was nil", http.StatusBadRequest) + } - mlog.Info("Worker: Job is complete", mlog.String("worker", w.name), mlog.String("job_id", job.Id)) - w.setJobSuccess(job) -} - -func (w *ImportProcessWorker) setJobSuccess(job *model.Job) { - if err := w.app.Srv().Jobs.SetJobSuccess(job); err != nil { - mlog.Error("Worker: Failed to set success for job", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) - w.setJobError(job, err) - } -} - -func (w *ImportProcessWorker) setJobError(job *model.Job, appError *model.AppError) { - if err := w.app.Srv().Jobs.SetJobError(job, appError); err != nil { - mlog.Error("Worker: Failed to set job error", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) - } + // do the actual import. + appErr, lineNumber := app.BulkImportWithPath(appContext, jsonFile, importZipReader, false, runtime.NumCPU(), model.ExportDataDir) + if appErr != nil { + job.Data["line_number"] = strconv.Itoa(lineNumber) + return appErr + } + + // remove import file when done. + if appErr := app.RemoveFile(importFilePath); appErr != nil { + return appErr + } + return nil + } + worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled) + return worker } diff --git a/jobs/interfaces/active_users_interface.go b/jobs/interfaces/active_users_interface.go deleted file mode 100644 index 8ed82b1ade..0000000000 --- a/jobs/interfaces/active_users_interface.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package interfaces - -import ( - "github.com/mattermost/mattermost-server/v6/model" -) - -type ActiveUsersJobInterface interface { - MakeWorker() model.Worker - MakeScheduler() model.Scheduler -} diff --git a/jobs/interfaces/expirynotify_interface.go b/jobs/interfaces/expirynotify_interface.go deleted file mode 100644 index 53bb072fae..0000000000 --- a/jobs/interfaces/expirynotify_interface.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package interfaces - -import ( - "github.com/mattermost/mattermost-server/v6/model" -) - -type ExpiryNotifyJobInterface interface { - MakeWorker() model.Worker - MakeScheduler() model.Scheduler -} diff --git a/jobs/interfaces/export_delete_interface.go b/jobs/interfaces/export_delete_interface.go deleted file mode 100644 index e5107d6a35..0000000000 --- a/jobs/interfaces/export_delete_interface.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package interfaces - -import "github.com/mattermost/mattermost-server/v6/model" - -type ExportDeleteInterface interface { - MakeWorker() model.Worker - MakeScheduler() model.Scheduler -} diff --git a/jobs/interfaces/export_process_interface.go b/jobs/interfaces/export_process_interface.go deleted file mode 100644 index 837781e2a3..0000000000 --- a/jobs/interfaces/export_process_interface.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package interfaces - -import ( - "github.com/mattermost/mattermost-server/v6/model" -) - -type ExportProcessInterface interface { - MakeWorker() model.Worker -} diff --git a/jobs/interfaces/extract_content_interface.go b/jobs/interfaces/extract_content_interface.go deleted file mode 100644 index 643df0c6bb..0000000000 --- a/jobs/interfaces/extract_content_interface.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package interfaces - -import ( - "github.com/mattermost/mattermost-server/v6/model" -) - -type ExtractContentInterface interface { - MakeWorker() model.Worker -} diff --git a/jobs/interfaces/import_delete_interface.go b/jobs/interfaces/import_delete_interface.go deleted file mode 100644 index cedf8a623a..0000000000 --- a/jobs/interfaces/import_delete_interface.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package interfaces - -import "github.com/mattermost/mattermost-server/v6/model" - -type ImportDeleteInterface interface { - MakeWorker() model.Worker - MakeScheduler() model.Scheduler -} diff --git a/jobs/interfaces/import_process_interface.go b/jobs/interfaces/import_process_interface.go deleted file mode 100644 index 85b7cda03a..0000000000 --- a/jobs/interfaces/import_process_interface.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package interfaces - -import ( - "github.com/mattermost/mattermost-server/v6/model" -) - -type ImportProcessInterface interface { - MakeWorker() model.Worker -} diff --git a/jobs/interfaces/migrations_interface.go b/jobs/interfaces/migrations_interface.go deleted file mode 100644 index 8772db7c31..0000000000 --- a/jobs/interfaces/migrations_interface.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package interfaces - -import ( - "github.com/mattermost/mattermost-server/v6/model" -) - -type MigrationsJobInterface interface { - MakeWorker() model.Worker - MakeScheduler() model.Scheduler -} diff --git a/jobs/interfaces/plugins_interface.go b/jobs/interfaces/plugins_interface.go deleted file mode 100644 index 0dab1b2542..0000000000 --- a/jobs/interfaces/plugins_interface.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package interfaces - -import ( - "github.com/mattermost/mattermost-server/v6/model" -) - -type PluginsJobInterface interface { - MakeWorker() model.Worker - MakeScheduler() model.Scheduler -} diff --git a/jobs/interfaces/product_notices_interface.go b/jobs/interfaces/product_notices_interface.go deleted file mode 100644 index 126696e460..0000000000 --- a/jobs/interfaces/product_notices_interface.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package interfaces - -import ( - "github.com/mattermost/mattermost-server/v6/model" -) - -type ProductNoticesJobInterface interface { - MakeWorker() model.Worker - MakeScheduler() model.Scheduler -} diff --git a/jobs/jobs.go b/jobs/jobs.go index 22779a69d4..bf0dc4a59f 100644 --- a/jobs/jobs.go +++ b/jobs/jobs.go @@ -31,6 +31,10 @@ func (srv *JobServer) CreateJob(jobType string, jobData map[string]string) (*mod return nil, err } + if srv.workers.Get(job.Type) == nil { + return nil, model.NewAppError("Job.IsValid", "model.job.is_valid.type.app_error", nil, "id="+job.Id, http.StatusBadRequest) + } + if _, err := srv.Store.Job().Save(&job); err != nil { return nil, model.NewAppError("CreateJob", "app.job.save.app_error", nil, err.Error(), http.StatusInternalServerError) } diff --git a/jobs/jobs_watcher.go b/jobs/jobs_watcher.go index d519c428e9..d8fcf71d55 100644 --- a/jobs/jobs_watcher.go +++ b/jobs/jobs_watcher.go @@ -72,131 +72,11 @@ func (watcher *Watcher) PollAndNotify() { } for _, job := range jobs { - if job.Type == model.JobTypeDataRetention { - if watcher.workers.DataRetention != nil { - select { - case watcher.workers.DataRetention.JobChannel() <- *job: - default: - } - } - } else if job.Type == model.JobTypeMessageExport { - if watcher.workers.MessageExport != nil { - select { - case watcher.workers.MessageExport.JobChannel() <- *job: - default: - } - } - } else if job.Type == model.JobTypeElasticsearchPostIndexing { - if watcher.workers.ElasticsearchIndexing != nil { - select { - case watcher.workers.ElasticsearchIndexing.JobChannel() <- *job: - default: - } - } - } else if job.Type == model.JobTypeElasticsearchPostAggregation { - if watcher.workers.ElasticsearchAggregation != nil { - select { - case watcher.workers.ElasticsearchAggregation.JobChannel() <- *job: - default: - } - } - } else if job.Type == model.JobTypeBlevePostIndexing { - if watcher.workers.BleveIndexing != nil { - select { - case watcher.workers.BleveIndexing.JobChannel() <- *job: - default: - } - } - } else if job.Type == model.JobTypeLdapSync { - if watcher.workers.LdapSync != nil { - select { - case watcher.workers.LdapSync.JobChannel() <- *job: - default: - } - } - } else if job.Type == model.JobTypeMigrations { - if watcher.workers.Migrations != nil { - select { - case watcher.workers.Migrations.JobChannel() <- *job: - default: - } - } - } else if job.Type == model.JobTypePlugins { - if watcher.workers.Plugins != nil { - select { - case watcher.workers.Plugins.JobChannel() <- *job: - default: - } - } - } else if job.Type == model.JobTypeExpiryNotify { - if watcher.workers.ExpiryNotify != nil { - select { - case watcher.workers.ExpiryNotify.JobChannel() <- *job: - default: - } - } - } else if job.Type == model.JobTypeProductNotices { - if watcher.workers.ProductNotices != nil { - select { - case watcher.workers.ProductNotices.JobChannel() <- *job: - default: - } - } - } else if job.Type == model.JobTypeActiveUsers { - if watcher.workers.ActiveUsers != nil { - select { - case watcher.workers.ActiveUsers.JobChannel() <- *job: - default: - } - } - } else if job.Type == model.JobTypeImportProcess { - if watcher.workers.ImportProcess != nil { - select { - case watcher.workers.ImportProcess.JobChannel() <- *job: - default: - } - } - } else if job.Type == model.JobTypeImportDelete { - if watcher.workers.ImportDelete != nil { - select { - case watcher.workers.ImportDelete.JobChannel() <- *job: - default: - } - } - } else if job.Type == model.JobTypeExportProcess { - if watcher.workers.ExportProcess != nil { - select { - case watcher.workers.ExportProcess.JobChannel() <- *job: - default: - } - } - } else if job.Type == model.JobTypeExportDelete { - if watcher.workers.ExportDelete != nil { - select { - case watcher.workers.ExportDelete.JobChannel() <- *job: - default: - } - } - } else if job.Type == model.JobTypeCloud { - if watcher.workers.Cloud != nil { - select { - case watcher.workers.Cloud.JobChannel() <- *job: - default: - } - } - } else if job.Type == model.JobTypeResendInvitationEmail { - if watcher.workers.ResendInvitationEmail != nil { - select { - case watcher.workers.ResendInvitationEmail.JobChannel() <- *job: - default: - } - } - } else if job.Type == model.JobTypeExtractContent { - if watcher.workers.ExtractContent != nil { - select { - case watcher.workers.ExtractContent.JobChannel() <- *job: - default: - } + worker := watcher.workers.Get(job.Type) + if worker != nil { + select { + case worker.JobChannel() <- *job: + default: } } } diff --git a/migrations/advanced_permissions_phase_2.go b/jobs/migrations/advanced_permissions_phase_2.go similarity index 93% rename from migrations/advanced_permissions_phase_2.go rename to jobs/migrations/advanced_permissions_phase_2.go index 037a138d45..caebb980a9 100644 --- a/migrations/advanced_permissions_phase_2.go +++ b/jobs/migrations/advanced_permissions_phase_2.go @@ -71,7 +71,7 @@ func (worker *Worker) runAdvancedPermissionsPhase2Migration(lastDone string) (bo if progress.CurrentTable == "TeamMembers" { // Run a TeamMembers migration batch. - result, err := worker.srv.Store.Team().MigrateTeamMembers(progress.LastTeamId, progress.LastUserId) + result, err := worker.store.Team().MigrateTeamMembers(progress.LastTeamId, progress.LastUserId) if err != nil { return false, progress.ToJSON(), model.NewAppError("MigrationsWorker.runAdvancedPermissionsPhase2Migration", "app.team.migrate_team_members.update.app_error", nil, err.Error(), http.StatusInternalServerError) } @@ -86,7 +86,7 @@ func (worker *Worker) runAdvancedPermissionsPhase2Migration(lastDone string) (bo progress.LastUserId = result["UserId"] } else if progress.CurrentTable == "ChannelMembers" { // Run a ChannelMembers migration batch. - data, err := worker.srv.Store.Channel().MigrateChannelMembers(progress.LastChannelId, progress.LastUserId) + data, err := worker.store.Channel().MigrateChannelMembers(progress.LastChannelId, progress.LastUserId) if err != nil { return false, progress.ToJSON(), model.NewAppError("MigrationsWorker.runAdvancedPermissionsPhase2Migration", "app.channel.migrate_channel_members.select.app_error", nil, err.Error(), http.StatusInternalServerError) } diff --git a/jobs/migrations/helper_test.go b/jobs/migrations/helper_test.go new file mode 100644 index 0000000000..4813fc0f69 --- /dev/null +++ b/jobs/migrations/helper_test.go @@ -0,0 +1,32 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package migrations + +import ( + "testing" + + "github.com/mattermost/mattermost-server/v6/model" + "github.com/mattermost/mattermost-server/v6/store" +) + +func Setup(tb testing.TB) store.Store { + store := mainHelper.GetStore() + store.DropAllTables() + return store +} + +func deleteAllJobsByTypeAndMigrationKey(store store.Store, jobType string, migrationKey string) { + jobs, err := store.Job().GetAllByType(model.JobTypeMigrations) + if err != nil { + panic(err) + } + + for _, job := range jobs { + if key, ok := job.Data[JobDataKeyMigration]; ok && key == migrationKey { + if _, err = store.Job().Delete(job.Id); err != nil { + panic(err) + } + } + } +} diff --git a/migrations/main_test.go b/jobs/migrations/main_test.go similarity index 100% rename from migrations/main_test.go rename to jobs/migrations/main_test.go diff --git a/migrations/migrations.go b/jobs/migrations/migrations.go similarity index 80% rename from migrations/migrations.go rename to jobs/migrations/migrations.go index df49a28dca..7327d953eb 100644 --- a/migrations/migrations.go +++ b/jobs/migrations/migrations.go @@ -6,8 +6,6 @@ package migrations import ( "net/http" - "github.com/mattermost/mattermost-server/v6/app" - tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces" "github.com/mattermost/mattermost-server/v6/model" "github.com/mattermost/mattermost-server/v6/store" ) @@ -21,16 +19,6 @@ const ( JobDataKeyMigrationLastDone = "last_done" ) -type MigrationsJobInterfaceImpl struct { - srv *app.Server -} - -func init() { - app.RegisterJobsMigrationsJobInterface(func(s *app.Server) tjobs.MigrationsJobInterface { - return &MigrationsJobInterfaceImpl{s} - }) -} - func MakeMigrationsList() []string { return []string{ model.MigrationKeyAdvancedPermissionsPhase2, diff --git a/migrations/migrations_test.go b/jobs/migrations/migrations_test.go similarity index 72% rename from migrations/migrations_test.go rename to jobs/migrations/migrations_test.go index 5adebfcdad..fee7830c0d 100644 --- a/migrations/migrations_test.go +++ b/jobs/migrations/migrations_test.go @@ -16,15 +16,14 @@ func TestGetMigrationState(t *testing.T) { if testing.Short() { t.SkipNow() } - th := Setup(t) - defer th.TearDown() + store := Setup(t) migrationKey := model.NewId() - th.DeleteAllJobsByTypeAndMigrationKey(model.JobTypeMigrations, migrationKey) + deleteAllJobsByTypeAndMigrationKey(store, model.JobTypeMigrations, migrationKey) // Test with no job yet. - state, job, err := GetMigrationState(migrationKey, th.App.Srv().Store) + state, job, err := GetMigrationState(migrationKey, store) assert.Nil(t, err) assert.Nil(t, job) assert.Equal(t, "unscheduled", state) @@ -34,15 +33,15 @@ func TestGetMigrationState(t *testing.T) { Name: migrationKey, Value: "true", } - nErr := th.App.Srv().Store.System().Save(&system) + nErr := store.System().Save(&system) assert.NoError(t, nErr) - state, job, err = GetMigrationState(migrationKey, th.App.Srv().Store) + state, job, err = GetMigrationState(migrationKey, store) assert.Nil(t, err) assert.Nil(t, job) assert.Equal(t, "completed", state) - _, nErr = th.App.Srv().Store.System().PermanentDeleteByName(migrationKey) + _, nErr = store.System().PermanentDeleteByName(migrationKey) assert.NoError(t, nErr) // Test with a job scheduled in "pending" state. @@ -56,10 +55,10 @@ func TestGetMigrationState(t *testing.T) { Type: model.JobTypeMigrations, } - j1, nErr = th.App.Srv().Store.Job().Save(j1) + j1, nErr = store.Job().Save(j1) require.NoError(t, nErr) - state, job, err = GetMigrationState(migrationKey, th.App.Srv().Store) + state, job, err = GetMigrationState(migrationKey, store) assert.Nil(t, err) assert.Equal(t, j1.Id, job.Id) assert.Equal(t, "in_progress", state) @@ -75,10 +74,10 @@ func TestGetMigrationState(t *testing.T) { Type: model.JobTypeMigrations, } - j2, nErr = th.App.Srv().Store.Job().Save(j2) + j2, nErr = store.Job().Save(j2) require.NoError(t, nErr) - state, job, err = GetMigrationState(migrationKey, th.App.Srv().Store) + state, job, err = GetMigrationState(migrationKey, store) assert.Nil(t, err) assert.Equal(t, j2.Id, job.Id) assert.Equal(t, "in_progress", state) @@ -94,10 +93,10 @@ func TestGetMigrationState(t *testing.T) { Type: model.JobTypeMigrations, } - j3, nErr = th.App.Srv().Store.Job().Save(j3) + j3, nErr = store.Job().Save(j3) require.NoError(t, nErr) - state, job, err = GetMigrationState(migrationKey, th.App.Srv().Store) + state, job, err = GetMigrationState(migrationKey, store) assert.Nil(t, err) assert.Equal(t, j3.Id, job.Id) assert.Equal(t, "unscheduled", state) diff --git a/migrations/scheduler.go b/jobs/migrations/scheduler.go similarity index 71% rename from migrations/scheduler.go rename to jobs/migrations/scheduler.go index 119ceafa85..928c5a51c0 100644 --- a/migrations/scheduler.go +++ b/jobs/migrations/scheduler.go @@ -6,9 +6,10 @@ package migrations import ( "time" - "github.com/mattermost/mattermost-server/v6/app" + "github.com/mattermost/mattermost-server/v6/jobs" "github.com/mattermost/mattermost-server/v6/model" "github.com/mattermost/mattermost-server/v6/shared/mlog" + "github.com/mattermost/mattermost-server/v6/store" ) const ( @@ -16,20 +17,13 @@ const ( ) type Scheduler struct { - srv *app.Server + jobServer *jobs.JobServer + store store.Store allMigrationsCompleted bool } -func (m *MigrationsJobInterfaceImpl) MakeScheduler() model.Scheduler { - return &Scheduler{m.srv, false} -} - -func (scheduler *Scheduler) Name() string { - return "MigrationsScheduler" -} - -func (scheduler *Scheduler) JobType() string { - return model.JobTypeMigrations +func MakeScheduler(jobServer *jobs.JobServer, store store.Store) model.Scheduler { + return &Scheduler{jobServer, store, false} } func (scheduler *Scheduler) Enabled(_ *model.Config) bool { @@ -48,22 +42,22 @@ func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, p //nolint:unparam func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) { - mlog.Debug("Scheduling Job", mlog.String("scheduler", scheduler.Name())) + mlog.Debug("Scheduling Job", mlog.String("scheduler", model.JobTypeMigrations)) // Work through the list of migrations in order. Schedule the first one that isn't done (assuming it isn't in progress already). for _, key := range MakeMigrationsList() { - state, job, err := GetMigrationState(key, scheduler.srv.Store) + state, job, err := GetMigrationState(key, scheduler.store) if err != nil { - mlog.Error("Failed to determine status of migration: ", mlog.String("scheduler", scheduler.Name()), mlog.String("migration_key", key), mlog.String("error", err.Error())) + mlog.Error("Failed to determine status of migration: ", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("migration_key", key), mlog.Err(err)) return nil, nil } if state == MigrationStateInProgress { // Check the migration job isn't wedged. if job != nil && job.LastActivityAt < model.GetMillis()-MigrationJobWedgedTimeoutMilliseconds && job.CreateAt < model.GetMillis()-MigrationJobWedgedTimeoutMilliseconds { - mlog.Warn("Job appears to be wedged. Rescheduling another instance.", mlog.String("scheduler", scheduler.Name()), mlog.String("wedged_job_id", job.Id), mlog.String("migration_key", key)) - if err := scheduler.srv.Jobs.SetJobError(job, nil); err != nil { - mlog.Error("Worker: Failed to set job error", mlog.String("scheduler", scheduler.Name()), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) + mlog.Warn("Job appears to be wedged. Rescheduling another instance.", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("wedged_job_id", job.Id), mlog.String("migration_key", key)) + if err := scheduler.jobServer.SetJobError(job, nil); err != nil { + mlog.Error("Worker: Failed to set job error", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("job_id", job.Id), mlog.Err(err)) } return scheduler.createJob(key, job) } @@ -77,7 +71,7 @@ func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, las } if state == MigrationStateUnscheduled { - mlog.Debug("Scheduling a new job for migration.", mlog.String("scheduler", scheduler.Name()), mlog.String("migration_key", key)) + mlog.Debug("Scheduling a new job for migration.", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("migration_key", key)) return scheduler.createJob(key, job) } @@ -87,7 +81,7 @@ func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, las // If we reached here, then there aren't any migrations left to run. scheduler.allMigrationsCompleted = true - mlog.Debug("All migrations are complete.", mlog.String("scheduler", scheduler.Name())) + mlog.Debug("All migrations are complete.", mlog.String("scheduler", model.JobTypeMigrations)) return nil, nil } @@ -103,7 +97,7 @@ func (scheduler *Scheduler) createJob(migrationKey string, lastJob *model.Job) ( JobDataKeyMigrationLastDone: lastDone, } - job, err := scheduler.srv.Jobs.CreateJob(model.JobTypeMigrations, data) + job, err := scheduler.jobServer.CreateJob(model.JobTypeMigrations, data) if err != nil { return nil, err } diff --git a/migrations/worker.go b/jobs/migrations/worker.go similarity index 87% rename from migrations/worker.go rename to jobs/migrations/worker.go index af8d81b4b8..1872b68811 100644 --- a/migrations/worker.go +++ b/jobs/migrations/worker.go @@ -8,10 +8,10 @@ import ( "net/http" "time" - "github.com/mattermost/mattermost-server/v6/app" "github.com/mattermost/mattermost-server/v6/jobs" "github.com/mattermost/mattermost-server/v6/model" "github.com/mattermost/mattermost-server/v6/shared/mlog" + "github.com/mattermost/mattermost-server/v6/store" ) const ( @@ -24,17 +24,17 @@ type Worker struct { stopped chan bool jobs chan model.Job jobServer *jobs.JobServer - srv *app.Server + store store.Store } -func (m *MigrationsJobInterfaceImpl) MakeWorker() model.Worker { +func MakeWorker(jobServer *jobs.JobServer, store store.Store) model.Worker { worker := Worker{ name: "Migrations", stop: make(chan bool, 1), stopped: make(chan bool, 1), jobs: make(chan model.Job), - jobServer: m.srv.Jobs, - srv: m.srv, + jobServer: jobServer, + store: store, } return &worker @@ -70,6 +70,10 @@ func (worker *Worker) JobChannel() chan<- model.Job { return worker.jobs } +func (worker *Worker) IsEnabled(_ *model.Config) bool { + return true +} + func (worker *Worker) DoJob(job *model.Job) { if claimed, err := worker.jobServer.ClaimJob(job); err != nil { mlog.Info("Worker experienced an error while trying to claim job", @@ -83,7 +87,7 @@ func (worker *Worker) DoJob(job *model.Job) { cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background()) cancelWatcherChan := make(chan interface{}, 1) - go worker.srv.Jobs.CancellationWatcher(cancelCtx, job.Id, cancelWatcherChan) + go worker.jobServer.CancellationWatcher(cancelCtx, job.Id, cancelWatcherChan) defer cancelCancelWatcher() @@ -111,7 +115,7 @@ func (worker *Worker) DoJob(job *model.Job) { return } else { job.Data[JobDataKeyMigrationLastDone] = progress - if err := worker.srv.Jobs.UpdateInProgressJobData(job); err != nil { + if err := worker.jobServer.UpdateInProgressJobData(job); err != nil { mlog.Error("Worker: Failed to update migration status data for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) worker.setJobError(job, err) return @@ -122,20 +126,20 @@ func (worker *Worker) DoJob(job *model.Job) { } func (worker *Worker) setJobSuccess(job *model.Job) { - if err := worker.srv.Jobs.SetJobSuccess(job); err != nil { + if err := worker.jobServer.SetJobSuccess(job); err != nil { mlog.Error("Worker: Failed to set success for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) worker.setJobError(job, err) } } func (worker *Worker) setJobError(job *model.Job, appError *model.AppError) { - if err := worker.srv.Jobs.SetJobError(job, appError); err != nil { + if err := worker.jobServer.SetJobError(job, appError); err != nil { mlog.Error("Worker: Failed to set job error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) } } func (worker *Worker) setJobCanceled(job *model.Job) { - if err := worker.srv.Jobs.SetJobCanceled(job); err != nil { + if err := worker.jobServer.SetJobCanceled(job); err != nil { mlog.Error("Worker: Failed to mark job as canceled", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) } } @@ -157,7 +161,7 @@ func (worker *Worker) runMigration(key string, lastDone string) (bool, string, * } if done { - if nErr := worker.srv.Store.System().Save(&model.System{Name: key, Value: "true"}); nErr != nil { + if nErr := worker.store.System().Save(&model.System{Name: key, Value: "true"}); nErr != nil { return false, "", model.NewAppError("runMigration", "migrations.system.save.app_error", nil, nErr.Error(), http.StatusInternalServerError) } } diff --git a/jobs/product_notices/product_notices.go b/jobs/product_notices/product_notices.go deleted file mode 100644 index 69e0271d02..0000000000 --- a/jobs/product_notices/product_notices.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package product_notices - -import ( - "github.com/mattermost/mattermost-server/v6/app" - tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces" -) - -type ProductNoticesJobInterfaceImpl struct { - App *app.App -} - -func init() { - app.RegisterProductNoticesJobInterface(func(s *app.Server) tjobs.ProductNoticesJobInterface { - a := app.New(app.ServerConnector(s.Channels())) - return &ProductNoticesJobInterfaceImpl{a} - }) -} diff --git a/jobs/product_notices/scheduler.go b/jobs/product_notices/scheduler.go index e85d30e8ef..e222e94fde 100644 --- a/jobs/product_notices/scheduler.go +++ b/jobs/product_notices/scheduler.go @@ -6,29 +6,12 @@ package product_notices import ( "time" - "github.com/mattermost/mattermost-server/v6/app" + "github.com/mattermost/mattermost-server/v6/jobs" "github.com/mattermost/mattermost-server/v6/model" ) type Scheduler struct { - App *app.App -} - -func (m *ProductNoticesJobInterfaceImpl) MakeScheduler() model.Scheduler { - return &Scheduler{m.App} -} - -func (scheduler *Scheduler) Name() string { - return JobName + "Scheduler" -} - -func (scheduler *Scheduler) JobType() string { - return model.JobTypeProductNotices -} - -func (scheduler *Scheduler) Enabled(cfg *model.Config) bool { - // Only enabled when ExtendSessionLengthWithActivity is enabled. - return *cfg.AnnouncementSettings.AdminNoticesEnabled || *cfg.AnnouncementSettings.UserNoticesEnabled + *jobs.PeriodicScheduler } func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time { @@ -36,12 +19,9 @@ func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, p return &nextTime } -func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) { - data := map[string]string{} - - job, err := scheduler.App.Srv().Jobs.CreateJob(model.JobTypeProductNotices, data) - if err != nil { - return nil, err +func MakeScheduler(jobServer *jobs.JobServer) model.Scheduler { + isEnabled := func(cfg *model.Config) bool { + return *cfg.AnnouncementSettings.AdminNoticesEnabled || *cfg.AnnouncementSettings.UserNoticesEnabled } - return job, nil + return &Scheduler{PeriodicScheduler: jobs.NewPeriodicScheduler(jobServer, model.JobTypeProductNotices, 0, isEnabled)} } diff --git a/jobs/product_notices/worker.go b/jobs/product_notices/worker.go index e8a9d91534..20935dfc80 100644 --- a/jobs/product_notices/worker.go +++ b/jobs/product_notices/worker.go @@ -4,97 +4,28 @@ package product_notices import ( - "github.com/mattermost/mattermost-server/v6/app" "github.com/mattermost/mattermost-server/v6/jobs" "github.com/mattermost/mattermost-server/v6/model" "github.com/mattermost/mattermost-server/v6/shared/mlog" ) -const ( - JobName = "ProductNotices" -) +const jobName = "ProductNotices" -type Worker struct { - name string - stop chan bool - stopped chan bool - jobs chan model.Job - jobServer *jobs.JobServer - app *app.App +type AppIface interface { + UpdateProductNotices() *model.AppError } -func (m *ProductNoticesJobInterfaceImpl) MakeWorker() model.Worker { - worker := Worker{ - name: JobName, - stop: make(chan bool, 1), - stopped: make(chan bool, 1), - jobs: make(chan model.Job), - jobServer: m.App.Srv().Jobs, - app: m.App, +func MakeWorker(jobServer *jobs.JobServer, app AppIface) model.Worker { + isEnabled := func(cfg *model.Config) bool { + return *cfg.AnnouncementSettings.AdminNoticesEnabled || *cfg.AnnouncementSettings.UserNoticesEnabled } - return &worker -} - -func (worker *Worker) Run() { - mlog.Debug("Worker started", mlog.String("worker", worker.name)) - - defer func() { - mlog.Debug("Worker finished", mlog.String("worker", worker.name)) - worker.stopped <- true - }() - - for { - select { - case <-worker.stop: - mlog.Debug("Worker received stop signal", mlog.String("worker", worker.name)) - return - case job := <-worker.jobs: - mlog.Debug("Worker received a new candidate job.", mlog.String("worker", worker.name)) - worker.DoJob(&job) + execute := func(job *model.Job) error { + if err := app.UpdateProductNotices(); err != nil { + mlog.Error("Worker: Failed to fetch product notices", mlog.String("worker", model.JobTypeProductNotices), mlog.String("job_id", job.Id), mlog.Err(err)) + return err } + return nil } -} - -func (worker *Worker) Stop() { - mlog.Debug("Worker stopping", mlog.String("worker", worker.name)) - worker.stop <- true - <-worker.stopped -} - -func (worker *Worker) JobChannel() chan<- model.Job { - return worker.jobs -} - -func (worker *Worker) DoJob(job *model.Job) { - if claimed, err := worker.jobServer.ClaimJob(job); err != nil { - mlog.Warn("Worker experienced an error while trying to claim job", - mlog.String("worker", worker.name), - mlog.String("job_id", job.Id), - mlog.String("error", err.Error())) - return - } else if !claimed { - return - } - - if err := worker.app.UpdateProductNotices(); err != nil { - mlog.Error("Worker: Failed to fetch product notices", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) - worker.setJobError(job, err) - return - } - - mlog.Info("Worker: Job is complete", mlog.String("worker", worker.name), mlog.String("job_id", job.Id)) - worker.setJobSuccess(job) -} - -func (worker *Worker) setJobSuccess(job *model.Job) { - if err := worker.app.Srv().Jobs.SetJobSuccess(job); err != nil { - mlog.Error("Worker: Failed to set success for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) - worker.setJobError(job, err) - } -} - -func (worker *Worker) setJobError(job *model.Job, appError *model.AppError) { - if err := worker.app.Srv().Jobs.SetJobError(job, appError); err != nil { - mlog.Error("Worker: Failed to set job error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) - } + worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled) + return worker } diff --git a/jobs/resend_invitation_email/resend_invitation_email.go b/jobs/resend_invitation_email/resend_invitation_email.go deleted file mode 100644 index ea2f038f90..0000000000 --- a/jobs/resend_invitation_email/resend_invitation_email.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. -package resend_invitation_email - -import ( - "github.com/mattermost/mattermost-server/v6/app" - ejobs "github.com/mattermost/mattermost-server/v6/einterfaces/jobs" -) - -type ResendInvitationEmailJobInterfaceImpl struct { - App *app.App -} - -func init() { - app.RegisterJobsResendInvitationEmailInterface(func(s *app.Server) ejobs.ResendInvitationEmailJobInterface { - a := app.New(app.ServerConnector(s.Channels())) - return &ResendInvitationEmailJobInterfaceImpl{a} - }) -} diff --git a/jobs/resend_invitation_email/scheduler.go b/jobs/resend_invitation_email/scheduler.go deleted file mode 100644 index d69bbc3e75..0000000000 --- a/jobs/resend_invitation_email/scheduler.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. -package resend_invitation_email - -import ( - "time" - - "github.com/mattermost/mattermost-server/v6/app" - "github.com/mattermost/mattermost-server/v6/model" -) - -const ResendInvitationEmailJob = "ResendInvitationEmailJob" - -type ResendInvitationEmailScheduler struct { - App *app.App -} - -func (rse *ResendInvitationEmailJobInterfaceImpl) MakeScheduler() model.Scheduler { - return &ResendInvitationEmailScheduler{rse.App} -} - -func (s *ResendInvitationEmailScheduler) Name() string { - return ResendInvitationEmailJob + "Scheduler" -} - -func (s *ResendInvitationEmailScheduler) JobType() string { - return model.JobTypeResendInvitationEmail -} - -func (s *ResendInvitationEmailScheduler) Enabled(cfg *model.Config) bool { - return *cfg.ServiceSettings.EnableEmailInvitations -} - -func (s *ResendInvitationEmailScheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time { - t := time.Now().Add(5 * time.Second) - return &t -} - -func (s *ResendInvitationEmailScheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) { - // noop because we manually schedule the job in api4.inviteUsersToTeam handler - return nil, nil -} diff --git a/jobs/resend_invitation_email/worker.go b/jobs/resend_invitation_email/worker.go index d3d3711d99..8f1232ece4 100644 --- a/jobs/resend_invitation_email/worker.go +++ b/jobs/resend_invitation_email/worker.go @@ -8,8 +8,10 @@ import ( "os" "strconv" - "github.com/mattermost/mattermost-server/v6/app" + "github.com/mattermost/mattermost-server/v6/jobs" "github.com/mattermost/mattermost-server/v6/model" + "github.com/mattermost/mattermost-server/v6/services/configservice" + "github.com/mattermost/mattermost-server/v6/services/telemetry" "github.com/mattermost/mattermost-server/v6/shared/mlog" "github.com/mattermost/mattermost-server/v6/store" ) @@ -18,21 +20,34 @@ const TwentyFourHoursInMillis int64 = 86400000 const FourtyEightHoursInMillis int64 = 172800000 const SeventyTwoHoursInMillis int64 = 259200000 -type ResendInvitationEmailWorker struct { - name string - stop chan bool - stopped chan bool - jobs chan model.Job - App *app.App +type AppIface interface { + configservice.ConfigService + GetUserByEmail(email string) (*model.User, *model.AppError) + GetTeamMembersByIds(teamID string, userIDs []string, restrictions *model.ViewUsersRestrictions) ([]*model.TeamMember, *model.AppError) + InviteNewUsersToTeamGracefully(emailList []string, teamID, senderId string, reminderInterval string) ([]*model.EmailInviteWithError, *model.AppError) } -func (rse *ResendInvitationEmailJobInterfaceImpl) MakeWorker() model.Worker { +type ResendInvitationEmailWorker struct { + name string + stop chan bool + stopped chan bool + jobs chan model.Job + jobServer *jobs.JobServer + app AppIface + store store.Store + telemetryService *telemetry.TelemetryService +} + +func MakeWorker(jobServer *jobs.JobServer, app AppIface, store store.Store, telemetryService *telemetry.TelemetryService) model.Worker { worker := ResendInvitationEmailWorker{ - name: ResendInvitationEmailJob, - stop: make(chan bool, 1), - stopped: make(chan bool, 1), - jobs: make(chan model.Job), - App: rse.App, + name: model.JobTypeResendInvitationEmail, + stop: make(chan bool, 1), + stopped: make(chan bool, 1), + jobs: make(chan model.Job), + jobServer: jobServer, + app: app, + store: store, + telemetryService: telemetryService, } return &worker } @@ -57,6 +72,10 @@ func (rseworker *ResendInvitationEmailWorker) Run() { } } +func (rseworker *ResendInvitationEmailWorker) IsEnabled(cfg *model.Config) bool { + return *cfg.ServiceSettings.EnableEmailInvitations +} + func (rseworker *ResendInvitationEmailWorker) Stop() { mlog.Debug("Worker stopping", mlog.String("worker", rseworker.name)) rseworker.stop <- true @@ -68,7 +87,7 @@ func (rseworker *ResendInvitationEmailWorker) JobChannel() chan<- model.Job { } func (rseworker *ResendInvitationEmailWorker) DoJob(job *model.Job) { - resendInviteEmailIntervalFlag := rseworker.App.Config().FeatureFlags.ResendInviteEmailInterval + resendInviteEmailIntervalFlag := rseworker.app.Config().FeatureFlags.ResendInviteEmailInterval switch resendInviteEmailIntervalFlag { case "48": @@ -99,7 +118,7 @@ func (rseworker *ResendInvitationEmailWorker) DoJob_24_72(job *model.Job) { } func (rseworker *ResendInvitationEmailWorker) Execute(job *model.Job, elapsedTimeSinceSchedule, firstDuration, secondDuration int64, firstDurationTelemetryValue, secondDurationTelemetryValue string) { - systemValue, sysValErr := rseworker.App.Srv().Store.System().GetByName(job.Id) + systemValue, sysValErr := rseworker.store.System().GetByName(job.Id) if sysValErr != nil { if _, ok := sysValErr.(*store.ErrNotFound); !ok { mlog.Error("An error occurred while getting NUMBER_OF_INVITE_EMAILS_SENT from system store", mlog.String("worker", rseworker.name), mlog.Err(sysValErr)) @@ -119,21 +138,21 @@ func (rseworker *ResendInvitationEmailWorker) Execute(job *model.Job, elapsedTim } func (rseworker *ResendInvitationEmailWorker) setJobSuccess(job *model.Job) { - if err := rseworker.App.Srv().Jobs.SetJobSuccess(job); err != nil { + if err := rseworker.jobServer.SetJobSuccess(job); err != nil { mlog.Error("Worker: Failed to set success for job", mlog.String("worker", rseworker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) rseworker.setJobError(job, err) } } func (rseworker *ResendInvitationEmailWorker) setJobCancelled(job *model.Job) { - if err := rseworker.App.Srv().Jobs.SetJobCanceled(job); err != nil { + if err := rseworker.jobServer.SetJobCanceled(job); err != nil { mlog.Error("Worker: Failed to cancel job", mlog.String("worker", rseworker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) rseworker.setJobError(job, err) } } func (rseworker *ResendInvitationEmailWorker) setJobError(job *model.Job, appError *model.AppError) { - if err := rseworker.App.Srv().Jobs.SetJobError(job, appError); err != nil { + if err := rseworker.jobServer.SetJobError(job, appError); err != nil { mlog.Error("Worker: Failed to set job error", mlog.String("worker", rseworker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) } } @@ -153,14 +172,14 @@ func (rseworker *ResendInvitationEmailWorker) removeAlreadyJoined(teamID string, var notJoinedYet []string for _, email := range emailList { // check if the user with this email is on the system already - user, appErr := rseworker.App.GetUserByEmail(email) + user, appErr := rseworker.app.GetUserByEmail(email) if appErr != nil { notJoinedYet = append(notJoinedYet, email) continue } // now we check if they are part of the team already userID := []string{user.Id} - members, appErr := rseworker.App.GetTeamMembersByIds(teamID, userID, nil) + members, appErr := rseworker.app.GetTeamMembersByIds(teamID, userID, nil) if len(members) == 0 || appErr != nil { notJoinedYet = append(notJoinedYet, email) } @@ -171,7 +190,7 @@ func (rseworker *ResendInvitationEmailWorker) removeAlreadyJoined(teamID string, func (rseworker *ResendInvitationEmailWorker) setNumResendEmailSent(job *model.Job, num string) { sysVar := &model.System{Name: job.Id, Value: num} - if err := rseworker.App.Srv().Store.System().SaveOrUpdate(sysVar); err != nil { + if err := rseworker.store.System().SaveOrUpdate(sysVar); err != nil { mlog.Error("Unable to save NUMBER_OF_INVITE_EMAIL_SENT", mlog.String("worker", rseworker.name), mlog.Err(err)) } } @@ -208,7 +227,7 @@ func (rseworker *ResendInvitationEmailWorker) GetDurations(job *model.Job) (int6 } func (rseworker *ResendInvitationEmailWorker) TearDown(job *model.Job) { - rseworker.App.Srv().Store.System().PermanentDeleteByName(job.Id) + rseworker.store.System().PermanentDeleteByName(job.Id) rseworker.setJobSuccess(job) } @@ -225,10 +244,10 @@ func (rseworker *ResendInvitationEmailWorker) ResendEmails(job *model.Job, inter emailList = rseworker.removeAlreadyJoined(teamID, emailList) - _, appErr := rseworker.App.InviteNewUsersToTeamGracefully(emailList, teamID, job.Data["senderID"], interval) + _, appErr := rseworker.app.InviteNewUsersToTeamGracefully(emailList, teamID, job.Data["senderID"], interval) if appErr != nil { mlog.Error("Worker: Failed to send emails", mlog.String("worker", rseworker.name), mlog.String("job_id", job.Id), mlog.String("error", appErr.Error())) rseworker.setJobError(job, appErr) } - rseworker.App.Srv().GetTelemetryService().SendTelemetry("track_invite_email_resend", map[string]interface{}{interval: interval}) + rseworker.telemetryService.SendTelemetry("track_invite_email_resend", map[string]interface{}{interval: interval}) } diff --git a/jobs/schedulers.go b/jobs/schedulers.go index 774f4c6fc9..01aca21baa 100644 --- a/jobs/schedulers.go +++ b/jobs/schedulers.go @@ -22,8 +22,8 @@ type Schedulers struct { isLeader bool running bool - schedulers []model.Scheduler - nextRunTimes []*time.Time + schedulers map[string]model.Scheduler + nextRunTimes map[string]*time.Time } var ( @@ -47,66 +47,19 @@ func (srv *JobServer) InitSchedulers() error { clusterLeaderChanged: make(chan bool, 1), jobs: srv, isLeader: true, + schedulers: make(map[string]model.Scheduler), + nextRunTimes: make(map[string]*time.Time), } - if srv.DataRetentionJob != nil { - schedulers.schedulers = append(schedulers.schedulers, srv.DataRetentionJob.MakeScheduler()) - } - - if srv.MessageExportJob != nil { - schedulers.schedulers = append(schedulers.schedulers, srv.MessageExportJob.MakeScheduler()) - } - - if elasticsearchAggregatorInterface := srv.ElasticsearchAggregator; elasticsearchAggregatorInterface != nil { - schedulers.schedulers = append(schedulers.schedulers, elasticsearchAggregatorInterface.MakeScheduler()) - } - - if ldapSyncInterface := srv.LdapSync; ldapSyncInterface != nil { - schedulers.schedulers = append(schedulers.schedulers, ldapSyncInterface.MakeScheduler()) - } - - if migrationsInterface := srv.Migrations; migrationsInterface != nil { - schedulers.schedulers = append(schedulers.schedulers, migrationsInterface.MakeScheduler()) - } - - if pluginsInterface := srv.Plugins; pluginsInterface != nil { - schedulers.schedulers = append(schedulers.schedulers, pluginsInterface.MakeScheduler()) - } - - if expiryNotifyInterface := srv.ExpiryNotify; expiryNotifyInterface != nil { - schedulers.schedulers = append(schedulers.schedulers, expiryNotifyInterface.MakeScheduler()) - } - - if activeUsersInterface := srv.ActiveUsers; activeUsersInterface != nil { - schedulers.schedulers = append(schedulers.schedulers, activeUsersInterface.MakeScheduler()) - } - - if productNoticesInterface := srv.ProductNotices; productNoticesInterface != nil { - schedulers.schedulers = append(schedulers.schedulers, productNoticesInterface.MakeScheduler()) - } - - if cloudInterface := srv.Cloud; cloudInterface != nil { - schedulers.schedulers = append(schedulers.schedulers, cloudInterface.MakeScheduler()) - } - - if resendInvitationEmailInterface := srv.ResendInvitationEmails; resendInvitationEmailInterface != nil { - schedulers.schedulers = append(schedulers.schedulers, resendInvitationEmailInterface.MakeScheduler()) - } - - if importDeleteInterface := srv.ImportDelete; importDeleteInterface != nil { - schedulers.schedulers = append(schedulers.schedulers, importDeleteInterface.MakeScheduler()) - } - - if exportDeleteInterface := srv.ExportDelete; exportDeleteInterface != nil { - schedulers.schedulers = append(schedulers.schedulers, exportDeleteInterface.MakeScheduler()) - } - - schedulers.nextRunTimes = make([]*time.Time, len(schedulers.schedulers)) srv.schedulers = schedulers return nil } +func (schedulers *Schedulers) AddScheduler(name string, scheduler model.Scheduler) { + schedulers.schedulers[name] = scheduler +} + // Start starts the schedulers. This call is not safe for concurrent use. // Synchronization should be implemented by the caller. func (schedulers *Schedulers) Start() { @@ -121,11 +74,11 @@ func (schedulers *Schedulers) Start() { }() now := time.Now() - for idx, scheduler := range schedulers.schedulers { + for name, scheduler := range schedulers.schedulers { if !scheduler.Enabled(schedulers.jobs.Config()) { - schedulers.nextRunTimes[idx] = nil + schedulers.nextRunTimes[name] = nil } else { - schedulers.setNextRunTime(schedulers.jobs.Config(), idx, now, false) + schedulers.setNextRunTime(schedulers.jobs.Config(), name, now, false) } } @@ -139,38 +92,38 @@ func (schedulers *Schedulers) Start() { case now = <-timer.C: cfg := schedulers.jobs.Config() - for idx, nextTime := range schedulers.nextRunTimes { + for name, nextTime := range schedulers.nextRunTimes { if nextTime == nil { continue } if time.Now().After(*nextTime) { - scheduler := schedulers.schedulers[idx] + scheduler := schedulers.schedulers[name] if scheduler == nil || !schedulers.isLeader || !scheduler.Enabled(cfg) { continue } - if _, err := schedulers.scheduleJob(cfg, scheduler); err != nil { - mlog.Error("Failed to schedule job", mlog.String("scheduler", scheduler.Name()), mlog.Err(err)) + if _, err := schedulers.scheduleJob(cfg, name, scheduler); err != nil { + mlog.Error("Failed to schedule job", mlog.String("scheduler", name), mlog.Err(err)) continue } - schedulers.setNextRunTime(cfg, idx, now, true) + schedulers.setNextRunTime(cfg, name, now, true) } } case newCfg := <-schedulers.configChanged: - for idx, scheduler := range schedulers.schedulers { + for name, scheduler := range schedulers.schedulers { if !schedulers.isLeader || !scheduler.Enabled(newCfg) { - schedulers.nextRunTimes[idx] = nil + schedulers.nextRunTimes[name] = nil } else { - schedulers.setNextRunTime(newCfg, idx, now, false) + schedulers.setNextRunTime(newCfg, name, now, false) } } case isLeader := <-schedulers.clusterLeaderChanged: - for idx := range schedulers.schedulers { + for name := range schedulers.schedulers { schedulers.isLeader = isLeader if !isLeader { - schedulers.nextRunTimes[idx] = nil + schedulers.nextRunTimes[name] = nil } else { - schedulers.setNextRunTime(schedulers.jobs.Config(), idx, now, false) + schedulers.setNextRunTime(schedulers.jobs.Config(), name, now, false) } } } @@ -192,37 +145,37 @@ func (schedulers *Schedulers) Stop() { schedulers.running = false } -func (schedulers *Schedulers) setNextRunTime(cfg *model.Config, idx int, now time.Time, pendingJobs bool) { - scheduler := schedulers.schedulers[idx] +func (schedulers *Schedulers) setNextRunTime(cfg *model.Config, name string, now time.Time, pendingJobs bool) { + scheduler := schedulers.schedulers[name] if !pendingJobs { - pj, err := schedulers.jobs.CheckForPendingJobsByType(scheduler.JobType()) + pj, err := schedulers.jobs.CheckForPendingJobsByType(name) if err != nil { mlog.Error("Failed to set next job run time", mlog.Err(err)) - schedulers.nextRunTimes[idx] = nil + schedulers.nextRunTimes[name] = nil return } pendingJobs = pj } - lastSuccessfulJob, err := schedulers.jobs.GetLastSuccessfulJobByType(scheduler.JobType()) + lastSuccessfulJob, err := schedulers.jobs.GetLastSuccessfulJobByType(name) if err != nil { mlog.Error("Failed to set next job run time", mlog.Err(err)) - schedulers.nextRunTimes[idx] = nil + schedulers.nextRunTimes[name] = nil return } - schedulers.nextRunTimes[idx] = scheduler.NextScheduleTime(cfg, now, pendingJobs, lastSuccessfulJob) - mlog.Debug("Next run time for scheduler", mlog.String("scheduler_name", scheduler.Name()), mlog.String("next_runtime", fmt.Sprintf("%v", schedulers.nextRunTimes[idx]))) + schedulers.nextRunTimes[name] = scheduler.NextScheduleTime(cfg, now, pendingJobs, lastSuccessfulJob) + mlog.Debug("Next run time for scheduler", mlog.String("scheduler_name", name), mlog.String("next_runtime", fmt.Sprintf("%v", schedulers.nextRunTimes[name]))) } -func (schedulers *Schedulers) scheduleJob(cfg *model.Config, scheduler model.Scheduler) (*model.Job, *model.AppError) { - pendingJobs, err := schedulers.jobs.CheckForPendingJobsByType(scheduler.JobType()) +func (schedulers *Schedulers) scheduleJob(cfg *model.Config, name string, scheduler model.Scheduler) (*model.Job, *model.AppError) { + pendingJobs, err := schedulers.jobs.CheckForPendingJobsByType(name) if err != nil { return nil, err } - lastSuccessfulJob, err2 := schedulers.jobs.GetLastSuccessfulJobByType(scheduler.JobType()) + lastSuccessfulJob, err2 := schedulers.jobs.GetLastSuccessfulJobByType(name) if err2 != nil { return nil, err } diff --git a/jobs/schedulers_test.go b/jobs/schedulers_test.go index 66b001d765..12f68a81b3 100644 --- a/jobs/schedulers_test.go +++ b/jobs/schedulers_test.go @@ -9,7 +9,6 @@ import ( "github.com/stretchr/testify/assert" - "github.com/mattermost/mattermost-server/v6/einterfaces/mocks" "github.com/mattermost/mattermost-server/v6/model" "github.com/mattermost/mattermost-server/v6/plugin/plugintest/mock" "github.com/mattermost/mattermost-server/v6/store/storetest" @@ -24,14 +23,6 @@ func (scheduler *MockScheduler) Enabled(cfg *model.Config) bool { return true } -func (scheduler *MockScheduler) Name() string { - return "MockScheduler" -} - -func (scheduler *MockScheduler) JobType() string { - return model.JobTypeDataRetention -} - func (scheduler *MockScheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time { nextTime := time.Now().Add(60 * time.Second) return &nextTime @@ -70,16 +61,11 @@ func TestScheduler(t *testing.T) { }, } - jobInterface := new(mocks.DataRetentionJobInterface) - jobInterface.On("MakeScheduler").Return(new(MockScheduler)) - jobServer.DataRetentionJob = jobInterface - - exportInterface := new(mocks.MessageExportJobInterface) - exportInterface.On("MakeScheduler").Return(new(MockScheduler)) - jobServer.MessageExportJob = exportInterface + jobServer.InitSchedulers() + jobServer.RegisterJobType(model.JobTypeDataRetention, nil, new(MockScheduler)) + jobServer.RegisterJobType(model.JobTypeMessageExport, nil, new(MockScheduler)) t.Run("Base", func(t *testing.T) { - jobServer.InitSchedulers() jobServer.StartSchedulers() time.Sleep(time.Second) diff --git a/jobs/server.go b/jobs/server.go index b34b073927..b7e7dbe341 100644 --- a/jobs/server.go +++ b/jobs/server.go @@ -7,8 +7,6 @@ import ( "sync" "github.com/mattermost/mattermost-server/v6/einterfaces" - ejobs "github.com/mattermost/mattermost-server/v6/einterfaces/jobs" - tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces" "github.com/mattermost/mattermost-server/v6/model" "github.com/mattermost/mattermost-server/v6/services/configservice" "github.com/mattermost/mattermost-server/v6/store" @@ -19,25 +17,6 @@ type JobServer struct { Store store.Store metrics einterfaces.MetricsInterface - DataRetentionJob ejobs.DataRetentionJobInterface - MessageExportJob ejobs.MessageExportJobInterface - ElasticsearchAggregator ejobs.ElasticsearchAggregatorInterface - ElasticsearchIndexer tjobs.IndexerJobInterface - LdapSync ejobs.LdapSyncInterface - Migrations tjobs.MigrationsJobInterface - Plugins tjobs.PluginsJobInterface - BleveIndexer tjobs.IndexerJobInterface - ExpiryNotify tjobs.ExpiryNotifyJobInterface - ProductNotices tjobs.ProductNoticesJobInterface - ActiveUsers tjobs.ActiveUsersJobInterface - ImportProcess tjobs.ImportProcessInterface - ImportDelete tjobs.ImportDeleteInterface - ExportProcess tjobs.ExportProcessInterface - ExportDelete tjobs.ExportDeleteInterface - Cloud ejobs.CloudJobInterface - ResendInvitationEmails ejobs.ResendInvitationEmailJobInterface - ExtractContent tjobs.ExtractContentInterface - // mut is used to protect the following fields from concurrent access. mut sync.Mutex workers *Workers @@ -56,6 +35,17 @@ func (srv *JobServer) Config() *model.Config { return srv.ConfigService.Config() } +func (srv *JobServer) RegisterJobType(name string, worker model.Worker, scheduler model.Scheduler) { + srv.mut.Lock() + defer srv.mut.Unlock() + if worker != nil { + srv.workers.AddWorker(name, worker) + } + if scheduler != nil { + srv.schedulers.AddScheduler(name, scheduler) + } +} + func (srv *JobServer) StartWorkers() error { srv.mut.Lock() defer srv.mut.Unlock() diff --git a/jobs/workers.go b/jobs/workers.go index ec2b94f71f..aa6251ec72 100644 --- a/jobs/workers.go +++ b/jobs/workers.go @@ -15,24 +15,7 @@ type Workers struct { ConfigService configservice.ConfigService Watcher *Watcher - DataRetention model.Worker - MessageExport model.Worker - ElasticsearchIndexing model.Worker - ElasticsearchAggregation model.Worker - LdapSync model.Worker - Migrations model.Worker - Plugins model.Worker - BleveIndexing model.Worker - ExpiryNotify model.Worker - ProductNotices model.Worker - ActiveUsers model.Worker - ImportProcess model.Worker - ImportDelete model.Worker - ExportProcess model.Worker - ExportDelete model.Worker - Cloud model.Worker - ResendInvitationEmail model.Worker - ExtractContent model.Worker + workers map[string]model.Worker listenerId string running bool @@ -52,163 +35,39 @@ func (srv *JobServer) InitWorkers() error { return ErrWorkersRunning } - workers := &Workers{ - ConfigService: srv.ConfigService, - } + workers := NewWorkers(srv.ConfigService) + workers.Watcher = srv.MakeWatcher(workers, DefaultWatcherPollingInterval) - if srv.DataRetentionJob != nil { - workers.DataRetention = srv.DataRetentionJob.MakeWorker() - } - - if srv.MessageExportJob != nil { - workers.MessageExport = srv.MessageExportJob.MakeWorker() - } - - if elasticsearchIndexerInterface := srv.ElasticsearchIndexer; elasticsearchIndexerInterface != nil { - workers.ElasticsearchIndexing = elasticsearchIndexerInterface.MakeWorker() - } - - if elasticsearchAggregatorInterface := srv.ElasticsearchAggregator; elasticsearchAggregatorInterface != nil { - workers.ElasticsearchAggregation = elasticsearchAggregatorInterface.MakeWorker() - } - - if ldapSyncInterface := srv.LdapSync; ldapSyncInterface != nil { - workers.LdapSync = ldapSyncInterface.MakeWorker() - } - - if migrationsInterface := srv.Migrations; migrationsInterface != nil { - workers.Migrations = migrationsInterface.MakeWorker() - } - - if pluginsInterface := srv.Plugins; pluginsInterface != nil { - workers.Plugins = pluginsInterface.MakeWorker() - } - - if bleveIndexerInterface := srv.BleveIndexer; bleveIndexerInterface != nil { - workers.BleveIndexing = bleveIndexerInterface.MakeWorker() - } - - if expiryNotifyInterface := srv.ExpiryNotify; expiryNotifyInterface != nil { - workers.ExpiryNotify = expiryNotifyInterface.MakeWorker() - } - - if activeUsersInterface := srv.ActiveUsers; activeUsersInterface != nil { - workers.ActiveUsers = activeUsersInterface.MakeWorker() - } - - if productNoticesInterface := srv.ProductNotices; productNoticesInterface != nil { - workers.ProductNotices = productNoticesInterface.MakeWorker() - } - - if importProcessInterface := srv.ImportProcess; importProcessInterface != nil { - workers.ImportProcess = importProcessInterface.MakeWorker() - } - - if importDeleteInterface := srv.ImportDelete; importDeleteInterface != nil { - workers.ImportDelete = importDeleteInterface.MakeWorker() - } - - if exportProcessInterface := srv.ExportProcess; exportProcessInterface != nil { - workers.ExportProcess = exportProcessInterface.MakeWorker() - } - - if exportDeleteInterface := srv.ExportDelete; exportDeleteInterface != nil { - workers.ExportDelete = exportDeleteInterface.MakeWorker() - } - - if cloudInterface := srv.Cloud; cloudInterface != nil { - workers.Cloud = cloudInterface.MakeWorker() - } - - if resendInvitationEmailInterface := srv.ResendInvitationEmails; resendInvitationEmailInterface != nil { - workers.ResendInvitationEmail = resendInvitationEmailInterface.MakeWorker() - } - - if extractContentInterface := srv.ExtractContent; extractContentInterface != nil { - workers.ExtractContent = extractContentInterface.MakeWorker() - } - srv.workers = workers return nil } +func NewWorkers(configService configservice.ConfigService) *Workers { + return &Workers{ + ConfigService: configService, + workers: make(map[string]model.Worker), + } +} + +func (workers *Workers) AddWorker(name string, worker model.Worker) { + workers.workers[name] = worker +} + +func (workers *Workers) Get(name string) model.Worker { + return workers.workers[name] +} + // Start starts the workers. This call is not safe for concurrent use. // Synchronization should be implemented by the caller. func (workers *Workers) Start() { mlog.Info("Starting workers") - if workers.DataRetention != nil { - go workers.DataRetention.Run() - } - - if workers.MessageExport != nil && *workers.ConfigService.Config().MessageExportSettings.EnableExport { - go workers.MessageExport.Run() - } - - if workers.ElasticsearchIndexing != nil && *workers.ConfigService.Config().ElasticsearchSettings.EnableIndexing { - go workers.ElasticsearchIndexing.Run() - } - - if workers.ElasticsearchAggregation != nil && *workers.ConfigService.Config().ElasticsearchSettings.EnableIndexing { - go workers.ElasticsearchAggregation.Run() - } - - if workers.LdapSync != nil && *workers.ConfigService.Config().LdapSettings.EnableSync { - go workers.LdapSync.Run() - } - - if workers.Migrations != nil { - go workers.Migrations.Run() - } - - if workers.Plugins != nil { - go workers.Plugins.Run() - } - - if workers.BleveIndexing != nil && *workers.ConfigService.Config().BleveSettings.EnableIndexing && *workers.ConfigService.Config().BleveSettings.IndexDir != "" { - go workers.BleveIndexing.Run() - } - - if workers.ExpiryNotify != nil { - go workers.ExpiryNotify.Run() - } - - if workers.ActiveUsers != nil { - go workers.ActiveUsers.Run() - } - - if workers.ProductNotices != nil { - go workers.ProductNotices.Run() - } - - if workers.ImportProcess != nil { - go workers.ImportProcess.Run() - } - - if workers.ImportDelete != nil { - go workers.ImportDelete.Run() - } - - if workers.ExportProcess != nil { - go workers.ExportProcess.Run() - } - - if workers.ExportDelete != nil { - go workers.ExportDelete.Run() - } - - if workers.Cloud != nil { - go workers.Cloud.Run() - } - - if workers.ResendInvitationEmail != nil { - go workers.ResendInvitationEmail.Run() - } - - if workers.ExtractContent != nil { - go workers.ExtractContent.Run() + for _, w := range workers.workers { + if w.IsEnabled(workers.ConfigService.Config()) { + go w.Run() + } } go workers.Watcher.Start() @@ -220,51 +79,12 @@ func (workers *Workers) Start() { func (workers *Workers) handleConfigChange(oldConfig *model.Config, newConfig *model.Config) { mlog.Debug("Workers received config change.") - if workers.DataRetention != nil { - if (!*oldConfig.DataRetentionSettings.EnableMessageDeletion && !*oldConfig.DataRetentionSettings.EnableFileDeletion && !*oldConfig.DataRetentionSettings.EnableBoardsDeletion) && (*newConfig.DataRetentionSettings.EnableMessageDeletion || *newConfig.DataRetentionSettings.EnableFileDeletion || *newConfig.DataRetentionSettings.EnableBoardsDeletion) { - go workers.DataRetention.Run() - } else if (*oldConfig.DataRetentionSettings.EnableMessageDeletion || *oldConfig.DataRetentionSettings.EnableFileDeletion || *oldConfig.DataRetentionSettings.EnableBoardsDeletion) && (!*newConfig.DataRetentionSettings.EnableMessageDeletion && !*newConfig.DataRetentionSettings.EnableFileDeletion && !*newConfig.DataRetentionSettings.EnableBoardsDeletion) { - workers.DataRetention.Stop() + for _, w := range workers.workers { + if w.IsEnabled(oldConfig) && !w.IsEnabled(newConfig) { + w.Stop() } - } - - if workers.MessageExport != nil { - if !*oldConfig.MessageExportSettings.EnableExport && *newConfig.MessageExportSettings.EnableExport { - go workers.MessageExport.Run() - } else if *oldConfig.MessageExportSettings.EnableExport && !*newConfig.MessageExportSettings.EnableExport { - workers.MessageExport.Stop() - } - } - - if workers.ElasticsearchIndexing != nil { - if !*oldConfig.ElasticsearchSettings.EnableIndexing && *newConfig.ElasticsearchSettings.EnableIndexing { - go workers.ElasticsearchIndexing.Run() - } else if *oldConfig.ElasticsearchSettings.EnableIndexing && !*newConfig.ElasticsearchSettings.EnableIndexing { - workers.ElasticsearchIndexing.Stop() - } - } - - if workers.ElasticsearchAggregation != nil { - if !*oldConfig.ElasticsearchSettings.EnableIndexing && *newConfig.ElasticsearchSettings.EnableIndexing { - go workers.ElasticsearchAggregation.Run() - } else if *oldConfig.ElasticsearchSettings.EnableIndexing && !*newConfig.ElasticsearchSettings.EnableIndexing { - workers.ElasticsearchAggregation.Stop() - } - } - - if workers.LdapSync != nil { - if !*oldConfig.LdapSettings.EnableSync && *newConfig.LdapSettings.EnableSync { - go workers.LdapSync.Run() - } else if *oldConfig.LdapSettings.EnableSync && !*newConfig.LdapSettings.EnableSync { - workers.LdapSync.Stop() - } - } - - if workers.BleveIndexing != nil { - if !*oldConfig.BleveSettings.EnableIndexing && *newConfig.BleveSettings.EnableIndexing { - go workers.BleveIndexing.Run() - } else if *oldConfig.BleveSettings.EnableIndexing && !*newConfig.BleveSettings.EnableIndexing { - workers.BleveIndexing.Stop() + if !w.IsEnabled(oldConfig) && w.IsEnabled(newConfig) { + go w.Run() } } } @@ -276,76 +96,10 @@ func (workers *Workers) Stop() { workers.Watcher.Stop() - if workers.DataRetention != nil && (*workers.ConfigService.Config().DataRetentionSettings.EnableMessageDeletion || *workers.ConfigService.Config().DataRetentionSettings.EnableFileDeletion) { - workers.DataRetention.Stop() - } - - if workers.MessageExport != nil && *workers.ConfigService.Config().MessageExportSettings.EnableExport { - workers.MessageExport.Stop() - } - - if workers.ElasticsearchIndexing != nil && *workers.ConfigService.Config().ElasticsearchSettings.EnableIndexing { - workers.ElasticsearchIndexing.Stop() - } - - if workers.ElasticsearchAggregation != nil && *workers.ConfigService.Config().ElasticsearchSettings.EnableIndexing { - workers.ElasticsearchAggregation.Stop() - } - - if workers.LdapSync != nil && *workers.ConfigService.Config().LdapSettings.EnableSync { - workers.LdapSync.Stop() - } - - if workers.Migrations != nil { - workers.Migrations.Stop() - } - - if workers.Plugins != nil { - workers.Plugins.Stop() - } - - if workers.BleveIndexing != nil && *workers.ConfigService.Config().BleveSettings.EnableIndexing { - workers.BleveIndexing.Stop() - } - - if workers.ExpiryNotify != nil { - workers.ExpiryNotify.Stop() - } - - if workers.ActiveUsers != nil { - workers.ActiveUsers.Stop() - } - - if workers.ProductNotices != nil { - workers.ProductNotices.Stop() - } - - if workers.ImportProcess != nil { - workers.ImportProcess.Stop() - } - - if workers.ImportDelete != nil { - workers.ImportDelete.Stop() - } - - if workers.ExportProcess != nil { - workers.ExportProcess.Stop() - } - - if workers.ExportDelete != nil { - workers.ExportDelete.Stop() - } - - if workers.Cloud != nil { - workers.Cloud.Stop() - } - - if workers.ResendInvitationEmail != nil { - workers.ResendInvitationEmail.Stop() - } - - if workers.ExtractContent != nil { - workers.ExtractContent.Stop() + for _, w := range workers.workers { + if w.IsEnabled(workers.ConfigService.Config()) { + w.Stop() + } } workers.running = false diff --git a/migrations/helper_test.go b/migrations/helper_test.go deleted file mode 100644 index 4f98abe881..0000000000 --- a/migrations/helper_test.go +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package migrations - -import ( - "os" - "testing" - - "github.com/mattermost/mattermost-server/v6/app" - "github.com/mattermost/mattermost-server/v6/app/request" - "github.com/mattermost/mattermost-server/v6/config" - "github.com/mattermost/mattermost-server/v6/model" - "github.com/mattermost/mattermost-server/v6/shared/mlog" - "github.com/mattermost/mattermost-server/v6/store/localcachelayer" -) - -type TestHelper struct { - App *app.App - Context *request.Context - Server *app.Server - BasicTeam *model.Team - BasicUser *model.User - BasicUser2 *model.User - BasicChannel *model.Channel - BasicPost *model.Post - - SystemAdminUser *model.User - - tempWorkspace string - - TestLogger *mlog.Logger -} - -func setupTestHelper(enterprise bool) *TestHelper { - store := mainHelper.GetStore() - store.DropAllTables() - - memoryStore := config.NewTestMemoryStore() - newConfig := memoryStore.Get().Clone() - *newConfig.AnnouncementSettings.AdminNoticesEnabled = false - *newConfig.AnnouncementSettings.UserNoticesEnabled = false - memoryStore.Set(newConfig) - - var options []app.Option - options = append(options, app.ConfigStore(memoryStore)) - options = append(options, app.StoreOverride(mainHelper.Store)) - options = append(options, app.SkipPostInitializiation()) - - testLogger, _ := mlog.NewLogger() - logCfg, _ := config.MloggerConfigFromLoggerConfig(&newConfig.LogSettings, nil, config.GetLogFileLocation) - if errCfg := testLogger.ConfigureTargets(logCfg, nil); errCfg != nil { - panic("failed to configure test logger: " + errCfg.Error()) - } - // lock logger config so server init cannot override it during testing. - testLogger.LockConfiguration() - options = append(options, app.SetLogger(testLogger)) - - s, err := app.NewServer(options...) - if err != nil { - panic(err) - } - // Adds the cache layer to the test store - s.Store, err = localcachelayer.NewLocalCacheLayer(s.Store, s.Metrics, s.Cluster, s.CacheProvider) - if err != nil { - panic(err) - } - - th := &TestHelper{ - App: app.New(app.ServerConnector(s.Channels())), - Context: &request.Context{}, - Server: s, - TestLogger: testLogger, - } - - th.App.UpdateConfig(func(cfg *model.Config) { *cfg.TeamSettings.MaxUsersPerTeam = 50 }) - th.App.UpdateConfig(func(cfg *model.Config) { *cfg.RateLimitSettings.Enable = false }) - prevListenAddress := *th.App.Config().ServiceSettings.ListenAddress - th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = ":0" }) - - serverErr := th.Server.Start() - if serverErr != nil { - panic(serverErr) - } - - th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = prevListenAddress }) - - th.App.DoAppMigrations() - - th.App.Srv().Store.MarkSystemRanUnitTests() - - th.App.UpdateConfig(func(cfg *model.Config) { *cfg.TeamSettings.EnableOpenServer = true }) - - if enterprise { - th.App.Srv().SetLicense(model.NewTestLicense()) - th.App.Srv().Jobs.InitWorkers() - th.App.Srv().Jobs.InitSchedulers() - } else { - th.App.Srv().SetLicense(nil) - } - - return th -} - -func SetupEnterprise(tb testing.TB) *TestHelper { - return setupTestHelper(true) -} - -func Setup(tb testing.TB) *TestHelper { - return setupTestHelper(false) -} - -func (th *TestHelper) InitBasic() *TestHelper { - th.SystemAdminUser = th.CreateUser() - th.App.UpdateUserRoles(th.SystemAdminUser.Id, model.SystemUserRoleId+" "+model.SystemAdminRoleId, false) - th.SystemAdminUser, _ = th.App.GetUser(th.SystemAdminUser.Id) - - th.BasicTeam = th.CreateTeam() - th.BasicUser = th.CreateUser() - th.LinkUserToTeam(th.BasicUser, th.BasicTeam) - th.BasicUser2 = th.CreateUser() - th.LinkUserToTeam(th.BasicUser2, th.BasicTeam) - th.BasicChannel = th.CreateChannel(th.BasicTeam) - th.BasicPost = th.CreatePost(th.BasicChannel) - - return th -} - -func (*TestHelper) MakeEmail() string { - return "success_" + model.NewId() + "@simulator.amazonses.com" -} - -func (th *TestHelper) CreateTeam() *model.Team { - id := model.NewId() - team := &model.Team{ - DisplayName: "dn_" + id, - Name: "name" + id, - Email: "success+" + id + "@simulator.amazonses.com", - Type: model.TeamOpen, - } - - var err *model.AppError - if team, err = th.App.CreateTeam(th.Context, team); err != nil { - panic(err) - } - return team -} - -func (th *TestHelper) CreateUser() *model.User { - id := model.NewId() - - user := &model.User{ - Email: "success+" + id + "@simulator.amazonses.com", - Username: "un_" + id, - Nickname: "nn_" + id, - Password: "Password1", - EmailVerified: true, - } - - var err *model.AppError - if user, err = th.App.CreateUser(th.Context, user); err != nil { - panic(err) - } - return user -} - -func (th *TestHelper) CreateChannel(team *model.Team) *model.Channel { - return th.createChannel(team, model.ChannelTypeOpen) -} - -func (th *TestHelper) createChannel(team *model.Team, channelType model.ChannelType) *model.Channel { - id := model.NewId() - - channel := &model.Channel{ - DisplayName: "dn_" + id, - Name: "name_" + id, - Type: channelType, - TeamId: team.Id, - CreatorId: th.BasicUser.Id, - } - - var err *model.AppError - if channel, err = th.App.CreateChannel(th.Context, channel, true); err != nil { - panic(err) - } - return channel -} - -func (th *TestHelper) CreateDmChannel(user *model.User) *model.Channel { - var err *model.AppError - var channel *model.Channel - if channel, err = th.App.GetOrCreateDirectChannel(th.Context, th.BasicUser.Id, user.Id); err != nil { - panic(err) - } - return channel -} - -func (th *TestHelper) CreatePost(channel *model.Channel) *model.Post { - id := model.NewId() - - post := &model.Post{ - UserId: th.BasicUser.Id, - ChannelId: channel.Id, - Message: "message_" + id, - CreateAt: model.GetMillis() - 10000, - } - - var err *model.AppError - if post, err = th.App.CreatePost(th.Context, post, channel, false, true); err != nil { - panic(err) - } - return post -} - -func (th *TestHelper) LinkUserToTeam(user *model.User, team *model.Team) { - _, err := th.App.JoinUserToTeam(th.Context, team, user, "") - if err != nil { - panic(err) - } -} - -func (th *TestHelper) AddUserToChannel(user *model.User, channel *model.Channel) *model.ChannelMember { - member, err := th.App.AddUserToChannel(user, channel, false) - if err != nil { - panic(err) - } - return member -} - -func (th *TestHelper) TearDown() { - // Clean all the caches - th.App.Srv().InvalidateAllCaches() - th.Server.Shutdown() - if th.tempWorkspace != "" { - os.RemoveAll(th.tempWorkspace) - } -} - -func (*TestHelper) ResetRoleMigration() { - sqlStore := mainHelper.GetSQLStore() - if _, err := sqlStore.GetMaster().Exec("DELETE from Roles"); err != nil { - panic(err) - } - - mainHelper.GetClusterInterface().SendClearRoleCacheMessage() - - if _, err := sqlStore.GetMaster().Exec("DELETE from Systems where Name = :Name", map[string]interface{}{"Name": model.AdvancedPermissionsMigrationKey}); err != nil { - panic(err) - } -} - -func (th *TestHelper) DeleteAllJobsByTypeAndMigrationKey(jobType string, migrationKey string) { - jobs, err := th.App.Srv().Store.Job().GetAllByType(model.JobTypeMigrations) - if err != nil { - panic(err) - } - - for _, job := range jobs { - if key, ok := job.Data[JobDataKeyMigration]; ok && key == migrationKey { - if _, err = th.App.Srv().Store.Job().Delete(job.Id); err != nil { - panic(err) - } - } - } -} diff --git a/model/bulk_export.go b/model/bulk_export.go new file mode 100644 index 0000000000..b18c32a3ab --- /dev/null +++ b/model/bulk_export.go @@ -0,0 +1,13 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package model + +// ExportDataDir is the name of the directory were to store additional data +// included with the export (e.g. file attachments). +const ExportDataDir = "data" + +type BulkExportOpts struct { + IncludeAttachments bool + CreateArchive bool +} diff --git a/model/job.go b/model/job.go index e892b051b4..8b6272e8af 100644 --- a/model/job.go +++ b/model/job.go @@ -78,29 +78,6 @@ func (j *Job) IsValid() *AppError { return NewAppError("Job.IsValid", "model.job.is_valid.create_at.app_error", nil, "id="+j.Id, http.StatusBadRequest) } - switch j.Type { - case JobTypeDataRetention: - case JobTypeElasticsearchPostIndexing: - case JobTypeElasticsearchPostAggregation: - case JobTypeBlevePostIndexing: - case JobTypeLdapSync: - case JobTypeMessageExport: - case JobTypeMigrations: - case JobTypePlugins: - case JobTypeProductNotices: - case JobTypeExpiryNotify: - case JobTypeActiveUsers: - case JobTypeImportProcess: - case JobTypeImportDelete: - case JobTypeExportProcess: - case JobTypeExportDelete: - case JobTypeCloud: - case JobTypeResendInvitationEmail: - case JobTypeExtractContent: - default: - return NewAppError("Job.IsValid", "model.job.is_valid.type.app_error", nil, "id="+j.Id, http.StatusBadRequest) - } - switch j.Status { case JobStatusPending: case JobStatusInProgress: @@ -119,11 +96,10 @@ type Worker interface { Run() Stop() JobChannel() chan<- Job + IsEnabled(cfg *Config) bool } type Scheduler interface { - Name() string - JobType() string Enabled(cfg *Config) bool NextScheduleTime(cfg *Config, now time.Time, pendingJobs bool, lastSuccessfulJob *Job) *time.Time ScheduleJob(cfg *Config, pendingJobs bool, lastSuccessfulJob *Job) (*Job, *AppError) diff --git a/model/upload_session.go b/model/upload_session.go index 994e7fb32c..0fb54ce6b5 100644 --- a/model/upload_session.go +++ b/model/upload_session.go @@ -12,8 +12,9 @@ import ( type UploadType string const ( - UploadTypeAttachment UploadType = "attachment" - UploadTypeImport UploadType = "import" + UploadTypeAttachment UploadType = "attachment" + UploadTypeImport UploadType = "import" + IncompleteUploadSuffix = ".tmp" ) // UploadNoUserID is a "fake" user id used by the API layer when in local mode. diff --git a/plugin/scheduler/plugin.go b/plugin/scheduler/plugin.go deleted file mode 100644 index 63dcc56835..0000000000 --- a/plugin/scheduler/plugin.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package scheduler - -import ( - "github.com/mattermost/mattermost-server/v6/app" - tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces" -) - -type PluginsJobInterfaceImpl struct { - App *app.App -} - -func init() { - app.RegisterJobsPluginsJobInterface(func(s *app.Server) tjobs.PluginsJobInterface { - a := app.New(app.ServerConnector(s.Channels())) - return &PluginsJobInterfaceImpl{a} - }) -} diff --git a/plugin/scheduler/scheduler.go b/plugin/scheduler/scheduler.go index 3ad4cb465b..85589fd60c 100644 --- a/plugin/scheduler/scheduler.go +++ b/plugin/scheduler/scheduler.go @@ -6,45 +6,15 @@ package scheduler import ( "time" - "github.com/mattermost/mattermost-server/v6/app" + "github.com/mattermost/mattermost-server/v6/jobs" "github.com/mattermost/mattermost-server/v6/model" - "github.com/mattermost/mattermost-server/v6/shared/mlog" ) -const pluginsJobInterval = 24 * 60 * 60 * time.Second +const schedFreq = 24 * time.Hour -type Scheduler struct { - App *app.App -} - -func (m *PluginsJobInterfaceImpl) MakeScheduler() model.Scheduler { - return &Scheduler{m.App} -} - -func (scheduler *Scheduler) Name() string { - return "PluginsScheduler" -} - -func (scheduler *Scheduler) JobType() string { - return model.JobTypePlugins -} - -func (scheduler *Scheduler) Enabled(cfg *model.Config) bool { - return true -} - -func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time { - nextTime := time.Now().Add(pluginsJobInterval) - return &nextTime -} - -func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) { - mlog.Debug("Scheduling Job", mlog.String("scheduler", scheduler.Name())) - - job, err := scheduler.App.Srv().Jobs.CreateJob(model.JobTypePlugins, nil) - if err != nil { - return nil, err +func MakeScheduler(jobServer *jobs.JobServer) model.Scheduler { + isEnabled := func(cfg *model.Config) bool { + return true } - - return job, nil + return jobs.NewPeriodicScheduler(jobServer, model.JobTypePlugins, schedFreq, isEnabled) } diff --git a/plugin/scheduler/worker.go b/plugin/scheduler/worker.go index 9c9b8f6794..cd141a4e18 100644 --- a/plugin/scheduler/worker.go +++ b/plugin/scheduler/worker.go @@ -4,29 +4,32 @@ package scheduler import ( - "github.com/mattermost/mattermost-server/v6/app" "github.com/mattermost/mattermost-server/v6/jobs" "github.com/mattermost/mattermost-server/v6/model" "github.com/mattermost/mattermost-server/v6/shared/mlog" ) +type AppIface interface { + DeleteAllExpiredPluginKeys() *model.AppError +} + type Worker struct { name string stop chan bool stopped chan bool jobs chan model.Job jobServer *jobs.JobServer - app *app.App + app AppIface } -func (m *PluginsJobInterfaceImpl) MakeWorker() model.Worker { +func MakeWorker(jobServer *jobs.JobServer, app AppIface) model.Worker { worker := Worker{ name: "Plugins", stop: make(chan bool, 1), stopped: make(chan bool, 1), jobs: make(chan model.Job), - jobServer: m.App.Srv().Jobs, - app: m.App, + jobServer: jobServer, + app: app, } return &worker @@ -62,6 +65,10 @@ func (worker *Worker) JobChannel() chan<- model.Job { return worker.jobs } +func (worker *Worker) IsEnabled(cfg *model.Config) bool { + return true +} + func (worker *Worker) DoJob(job *model.Job) { if claimed, err := worker.jobServer.ClaimJob(job); err != nil { mlog.Info("Worker experienced an error while trying to claim job", @@ -84,14 +91,14 @@ func (worker *Worker) DoJob(job *model.Job) { } func (worker *Worker) setJobSuccess(job *model.Job) { - if err := worker.app.Srv().Jobs.SetJobSuccess(job); err != nil { + if err := worker.jobServer.SetJobSuccess(job); err != nil { mlog.Error("Worker: Failed to set success for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) worker.setJobError(job, err) } } func (worker *Worker) setJobError(job *model.Job, appError *model.AppError) { - if err := worker.app.Srv().Jobs.SetJobError(job, appError); err != nil { + if err := worker.jobServer.SetJobError(job, appError); err != nil { mlog.Error("Worker: Failed to set job error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) } } diff --git a/services/searchengine/bleveengine/indexer/indexing_job.go b/services/searchengine/bleveengine/indexer/indexing_job.go index 2baa996dcc..510be5ed57 100644 --- a/services/searchengine/bleveengine/indexer/indexing_job.go +++ b/services/searchengine/bleveengine/indexer/indexing_job.go @@ -1,7 +1,7 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -package ebleveengine +package indexer import ( "context" @@ -9,9 +9,7 @@ import ( "strconv" "time" - "github.com/mattermost/mattermost-server/v6/app" "github.com/mattermost/mattermost-server/v6/jobs" - tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces" "github.com/mattermost/mattermost-server/v6/model" "github.com/mattermost/mattermost-server/v6/services/searchengine/bleveengine" "github.com/mattermost/mattermost-server/v6/shared/mlog" @@ -26,28 +24,17 @@ const ( EstimatedUserCount = 10000 ) -func init() { - app.RegisterJobsBleveIndexerInterface(func(s *app.Server) tjobs.IndexerJobInterface { - return &BleveIndexerInterfaceImpl{s} - }) -} - -type BleveIndexerInterfaceImpl struct { - Server *app.Server -} - type BleveIndexerWorker struct { name string stop chan bool stopped chan bool jobs chan model.Job jobServer *jobs.JobServer - - engine *bleveengine.BleveEngine + engine *bleveengine.BleveEngine } -func (bi *BleveIndexerInterfaceImpl) MakeWorker() model.Worker { - if bi.Server.SearchEngine.BleveEngine == nil { +func MakeWorker(jobServer *jobs.JobServer, engine *bleveengine.BleveEngine) model.Worker { + if engine == nil { return nil } return &BleveIndexerWorker{ @@ -55,9 +42,8 @@ func (bi *BleveIndexerInterfaceImpl) MakeWorker() model.Worker { stop: make(chan bool, 1), stopped: make(chan bool, 1), jobs: make(chan model.Job), - jobServer: bi.Server.Jobs, - - engine: bi.Server.SearchEngine.BleveEngine.(*bleveengine.BleveEngine), + jobServer: jobServer, + engine: engine, } } @@ -92,6 +78,10 @@ func (worker *BleveIndexerWorker) JobChannel() chan<- model.Job { return worker.jobs } +func (worker *BleveIndexerWorker) IsEnabled(cfg *model.Config) bool { + return true +} + func (worker *BleveIndexerWorker) Run() { mlog.Debug("Worker Started", mlog.String("workername", worker.name)) diff --git a/services/searchengine/bleveengine/indexer/indexing_job_test.go b/services/searchengine/bleveengine/indexer/indexing_job_test.go index 55f93d2f08..16de60201e 100644 --- a/services/searchengine/bleveengine/indexer/indexing_job_test.go +++ b/services/searchengine/bleveengine/indexer/indexing_job_test.go @@ -1,7 +1,7 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -package ebleveengine +package indexer import ( "errors"