mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

[MM-54132] Use annotated logger for log messages from jobs (#24275)

Просмотр исходного кода
Этот коммит содержится в:
Ben Schumacher
2023-09-07 08:50:22 +02:00
коммит произвёл GitHub
родитель fcfcbd9909
Коммит 30b12f199b
120 изменённых файлов: 1064 добавлений и 1111 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

17
server/channels/jobs/migrations/helper_test.go
Просмотреть файл

@@ -7,7 +7,10 @@ import (
"testing"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/stretchr/testify/require"
)
func Setup(tb testing.TB) store.Store {
@@ -16,17 +19,15 @@ func Setup(tb testing.TB) store.Store {
return store
}
func deleteAllJobsByTypeAndMigrationKey(store store.Store, jobType string, migrationKey string) {
jobs, err := store.Job().GetAllByType(model.JobTypeMigrations)
if err != nil {
panic(err)
}
func deleteAllJobsByTypeAndMigrationKey(t *testing.T, store store.Store, jobType string, migrationKey string) {
ctx := request.EmptyContext(mlog.CreateConsoleTestLogger(t))
jobs, err := store.Job().GetAllByType(ctx, model.JobTypeMigrations)
require.NoError(t, err)
for _, job := range jobs {
if key, ok := job.Data[JobDataKeyMigration]; ok && key == migrationKey {
if _, err = store.Job().Delete(job.Id); err != nil {
panic(err)
}
_, err = store.Job().Delete(job.Id)
require.NoError(t, err)
}
}
}

5
server/channels/jobs/migrations/migrations.go
Просмотреть файл

@@ -7,6 +7,7 @@ import (
"net/http"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
@@ -25,12 +26,12 @@ func MakeMigrationsList() []string {
}
}
func GetMigrationState(migration string, store store.Store) (string, *model.Job, *model.AppError) {
func GetMigrationState(c *request.Context, migration string, store store.Store) (string, *model.Job, *model.AppError) {
if _, err := store.System().GetByName(migration); err == nil {
return MigrationStateCompleted, nil, nil
}
jobs, err := store.Job().GetAllByType(model.JobTypeMigrations)
jobs, err := store.Job().GetAllByType(c, model.JobTypeMigrations)
if err != nil {
return "", nil, model.NewAppError("GetMigrationState", "app.job.get_all.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}

15
server/channels/jobs/migrations/migrations_test.go
Просмотреть файл

@@ -10,6 +10,8 @@ import (
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
)
func TestGetMigrationState(t *testing.T) {
@@ -17,13 +19,14 @@ func TestGetMigrationState(t *testing.T) {
t.SkipNow()
}
store := Setup(t)
ctx := request.EmptyContext(mlog.CreateConsoleTestLogger(t))
migrationKey := model.NewId()
deleteAllJobsByTypeAndMigrationKey(store, model.JobTypeMigrations, migrationKey)
deleteAllJobsByTypeAndMigrationKey(t, store, model.JobTypeMigrations, migrationKey)
// Test with no job yet.
state, job, err := GetMigrationState(migrationKey, store)
state, job, err := GetMigrationState(ctx, migrationKey, store)
assert.Nil(t, err)
assert.Nil(t, job)
assert.Equal(t, "unscheduled", state)
@@ -36,7 +39,7 @@ func TestGetMigrationState(t *testing.T) {
nErr := store.System().Save(&system)
assert.NoError(t, nErr)
state, job, err = GetMigrationState(migrationKey, store)
state, job, err = GetMigrationState(ctx, migrationKey, store)
assert.Nil(t, err)
assert.Nil(t, job)
assert.Equal(t, "completed", state)
@@ -58,7 +61,7 @@ func TestGetMigrationState(t *testing.T) {
j1, nErr = store.Job().Save(j1)
require.NoError(t, nErr)
state, job, err = GetMigrationState(migrationKey, store)
state, job, err = GetMigrationState(ctx, migrationKey, store)
assert.Nil(t, err)
assert.Equal(t, j1.Id, job.Id)
assert.Equal(t, "in_progress", state)
@@ -77,7 +80,7 @@ func TestGetMigrationState(t *testing.T) {
j2, nErr = store.Job().Save(j2)
require.NoError(t, nErr)
state, job, err = GetMigrationState(migrationKey, store)
state, job, err = GetMigrationState(ctx, migrationKey, store)
assert.Nil(t, err)
assert.Equal(t, j2.Id, job.Id)
assert.Equal(t, "in_progress", state)
@@ -96,7 +99,7 @@ func TestGetMigrationState(t *testing.T) {
j3, nErr = store.Job().Save(j3)
require.NoError(t, nErr)
state, job, err = GetMigrationState(migrationKey, store)
state, job, err = GetMigrationState(ctx, migrationKey, store)
assert.Nil(t, err)
assert.Equal(t, j3.Id, job.Id)
assert.Equal(t, "unscheduled", state)

53
server/channels/jobs/migrations/scheduler.go
Просмотреть файл

@@ -8,6 +8,7 @@ import (
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/jobs"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
@@ -22,7 +23,9 @@ type Scheduler struct {
allMigrationsCompleted bool
}
func MakeScheduler(jobServer *jobs.JobServer, store store.Store) model.Scheduler {
var _ jobs.Scheduler = (*Scheduler)(nil)
func MakeScheduler(jobServer *jobs.JobServer, store store.Store) *Scheduler {
return &Scheduler{jobServer, store, false}
}
@@ -41,27 +44,14 @@ func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, p
}
//nolint:unparam
func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) {
mlog.Debug("Scheduling Job", mlog.String("scheduler", model.JobTypeMigrations))
func (scheduler *Scheduler) ScheduleJob(c *request.Context, cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) {
c.Logger().Debug("Scheduling Job", mlog.String("scheduler", model.JobTypeMigrations))
// Work through the list of migrations in order. Schedule the first one that isn't done (assuming it isn't in progress already).
for _, key := range MakeMigrationsList() {
state, job, err := GetMigrationState(key, scheduler.store)
state, job, err := GetMigrationState(c, key, scheduler.store)
if err != nil {
mlog.Error("Failed to determine status of migration: ", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("migration_key", key), mlog.Err(err))
return nil, nil
}
if state == MigrationStateInProgress {
// Check the migration job isn't wedged.
if job != nil && job.LastActivityAt < model.GetMillis()-MigrationJobWedgedTimeoutMilliseconds && job.CreateAt < model.GetMillis()-MigrationJobWedgedTimeoutMilliseconds {
mlog.Warn("Job appears to be wedged. Rescheduling another instance.", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("wedged_job_id", job.Id), mlog.String("migration_key", key))
if err := scheduler.jobServer.SetJobError(job, nil); err != nil {
mlog.Error("Worker: Failed to set job error", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("job_id", job.Id), mlog.Err(err))
}
return scheduler.createJob(key, job)
}
c.Logger().Error("Failed to determine status of migration: ", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("migration_key", key), mlog.Err(err))
return nil, nil
}
@@ -70,23 +60,36 @@ func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, las
continue
}
if state == MigrationStateUnscheduled {
mlog.Debug("Scheduling a new job for migration.", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("migration_key", key))
return scheduler.createJob(key, job)
if state == MigrationStateInProgress {
// Check the migration job isn't wedged.
if job != nil && job.LastActivityAt < model.GetMillis()-MigrationJobWedgedTimeoutMilliseconds && job.CreateAt < model.GetMillis()-MigrationJobWedgedTimeoutMilliseconds {
job.Logger.Warn("Job appears to be wedged. Rescheduling another instance.", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("wedged_job_id", job.Id), mlog.String("migration_key", key))
if err := scheduler.jobServer.SetJobError(job, nil); err != nil {
job.Logger.Error("Worker: Failed to set job error", mlog.String("scheduler", model.JobTypeMigrations), mlog.Err(err))
}
return scheduler.createJob(c, key, job)
}
return nil, nil
}
mlog.Error("Unknown migration state. Not doing anything.", mlog.String("migration_state", state))
if state == MigrationStateUnscheduled {
job.Logger.Debug("Scheduling a new job for migration.", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("migration_key", key))
return scheduler.createJob(c, key, job)
}
job.Logger.Error("Unknown migration state. Not doing anything.", mlog.String("migration_state", state))
return nil, nil
}
// If we reached here, then there aren't any migrations left to run.
scheduler.allMigrationsCompleted = true
mlog.Debug("All migrations are complete.", mlog.String("scheduler", model.JobTypeMigrations))
c.Logger().Debug("All migrations are complete.", mlog.String("scheduler", model.JobTypeMigrations))
return nil, nil
}
func (scheduler *Scheduler) createJob(migrationKey string, lastJob *model.Job) (*model.Job, *model.AppError) {
func (scheduler *Scheduler) createJob(c *request.Context, migrationKey string, lastJob *model.Job) (*model.Job, *model.AppError) {
var lastDone string
if lastJob != nil {
lastDone = lastJob.Data[JobDataKeyMigrationLastDone]
@@ -97,7 +100,7 @@ func (scheduler *Scheduler) createJob(migrationKey string, lastJob *model.Job) (
JobDataKeyMigrationLastDone: lastDone,
}
job, err := scheduler.jobServer.CreateJob(model.JobTypeMigrations, data)
job, err := scheduler.jobServer.CreateJob(c, model.JobTypeMigrations, data)
if err != nil {
return nil, err
}

46
server/channels/jobs/migrations/worker.go
Просмотреть файл

@@ -11,6 +11,7 @@ import (
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/jobs"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
@@ -25,17 +26,20 @@ type Worker struct {
stopped chan bool
jobs chan model.Job
jobServer *jobs.JobServer
logger mlog.LoggerIFace
store store.Store
closed int32
}
func MakeWorker(jobServer *jobs.JobServer, store store.Store) model.Worker {
func MakeWorker(jobServer *jobs.JobServer, store store.Store) *Worker {
const workerName = "Migrations"
worker := Worker{
name: "Migrations",
name: workerName,
stop: make(chan struct{}),
stopped: make(chan bool, 1),
jobs: make(chan model.Job),
jobServer: jobServer,
logger: jobServer.Logger().With(mlog.String("workername", workerName)),
store: store,
}
@@ -47,20 +51,22 @@ func (worker *Worker) Run() {
if atomic.CompareAndSwapInt32(&worker.closed, 1, 0) {
worker.stop = make(chan struct{})
}
mlog.Debug("Worker started", mlog.String("worker", worker.name))
worker.logger.Debug("Worker started")
defer func() {
mlog.Debug("Worker finished", mlog.String("worker", worker.name))
worker.logger.Debug("Worker finished")
worker.stopped <- true
}()
for {
select {
case <-worker.stop:
mlog.Debug("Worker received stop signal", mlog.String("worker", worker.name))
worker.logger.Debug("Worker received stop signal")
return
case job := <-worker.jobs:
mlog.Debug("Worker received a new candidate job.", mlog.String("worker", worker.name))
job.Logger = job.Logger.With(mlog.String("workername", worker.name))
job.Logger.Debug("Worker received a new candidate job")
worker.DoJob(&job)
}
}
@@ -71,7 +77,7 @@ func (worker *Worker) Stop() {
if !atomic.CompareAndSwapInt32(&worker.closed, 0, 1) {
return
}
mlog.Debug("Worker stopping", mlog.String("worker", worker.name))
worker.logger.Debug("Worker stopping")
close(worker.stop)
<-worker.stopped
}
@@ -88,47 +94,45 @@ func (worker *Worker) DoJob(job *model.Job) {
defer worker.jobServer.HandleJobPanic(job)
if claimed, err := worker.jobServer.ClaimJob(job); err != nil {
mlog.Info("Worker experienced an error while trying to claim job",
mlog.String("worker", worker.name),
mlog.String("job_id", job.Id),
mlog.String("error", err.Error()))
job.Logger.Info("Worker experienced an error while trying to claim job", mlog.Err(err))
return
} else if !claimed {
return
}
cancelContext := request.EmptyContext(worker.logger)
cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background())
cancelWatcherChan := make(chan struct{}, 1)
go worker.jobServer.CancellationWatcher(cancelCtx, job.Id, cancelWatcherChan)
cancelContext.SetContext(cancelCtx)
go worker.jobServer.CancellationWatcher(cancelContext, job.Id, cancelWatcherChan)
defer cancelCancelWatcher()
for {
select {
case <-cancelWatcherChan:
mlog.Debug("Worker: Job has been canceled via CancellationWatcher", mlog.String("worker", worker.name), mlog.String("job_id", job.Id))
job.Logger.Debug("Worker: Job has been canceled via CancellationWatcher")
worker.setJobCanceled(job)
return
case <-worker.stop:
mlog.Debug("Worker: Job has been canceled via Worker Stop", mlog.String("worker", worker.name), mlog.String("job_id", job.Id))
job.Logger.Debug("Worker: Job has been canceled via Worker Stop")
worker.setJobCanceled(job)
return
case <-time.After(TimeBetweenBatches * time.Millisecond):
done, progress, err := worker.runMigration(job.Data[JobDataKeyMigration], job.Data[JobDataKeyMigrationLastDone])
if err != nil {
mlog.Error("Worker: Failed to run migration", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
job.Logger.Error("Worker: Failed to run migration", mlog.Err(err))
worker.setJobError(job, err)
return
} else if done {
mlog.Info("Worker: Job is complete", mlog.String("worker", worker.name), mlog.String("job_id", job.Id))
job.Logger.Info("Worker: Job is complete")
worker.setJobSuccess(job)
return
} else {
job.Data[JobDataKeyMigrationLastDone] = progress
if err := worker.jobServer.UpdateInProgressJobData(job); err != nil {
mlog.Error("Worker: Failed to update migration status data for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
job.Logger.Error("Worker: Failed to update migration status data for job", mlog.Err(err))
worker.setJobError(job, err)
return
}
@@ -139,20 +143,20 @@ func (worker *Worker) DoJob(job *model.Job) {
func (worker *Worker) setJobSuccess(job *model.Job) {
if err := worker.jobServer.SetJobSuccess(job); err != nil {
mlog.Error("Worker: Failed to set success for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
job.Logger.Error("Worker: Failed to set success for job", mlog.Err(err))
worker.setJobError(job, err)
}
}
func (worker *Worker) setJobError(job *model.Job, appError *model.AppError) {
if err := worker.jobServer.SetJobError(job, appError); err != nil {
mlog.Error("Worker: Failed to set job error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
job.Logger.Error("Worker: Failed to set job error", mlog.Err(err))
}
}
func (worker *Worker) setJobCanceled(job *model.Job) {
if err := worker.jobServer.SetJobCanceled(job); err != nil {
mlog.Error("Worker: Failed to mark job as canceled", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
job.Logger.Error("Worker: Failed to mark job as canceled", mlog.Err(err))
}
}