Mono repo -> Master (#22553)

Combines the following repositories into one:

https://github.com/mattermost/mattermost-server
https://github.com/mattermost/mattermost-webapp
https://github.com/mattermost/focalboard
https://github.com/mattermost/mattermost-plugin-playbooks

server/channels/jobs/migrations/advanced_permissions_phase_2.go

@@ -0,0 +1,112 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+type AdvancedPermissionsPhase2Progress struct {
+	CurrentTable  string `json:"current_table"`
+	LastTeamId    string `json:"last_team_id"`
+	LastChannelId string `json:"last_channel_id"`
+	LastUserId    string `json:"last_user"`
+}
+
+func (p *AdvancedPermissionsPhase2Progress) ToJSON() string {
+	b, _ := json.Marshal(p)
+	return string(b)
+}
+
+func AdvancedPermissionsPhase2ProgressFromJSON(data io.Reader) *AdvancedPermissionsPhase2Progress {
+	var o *AdvancedPermissionsPhase2Progress
+	err := json.NewDecoder(data).Decode(&o)
+	if err != nil {
+		mlog.Warn("Error decoding advanced permissions phase 2 progress", mlog.Err(err))
+	}
+	return o
+}
+
+func (p *AdvancedPermissionsPhase2Progress) IsValid() bool {
+	if !model.IsValidId(p.LastChannelId) {
+		return false
+	}
+
+	if !model.IsValidId(p.LastTeamId) {
+		return false
+	}
+
+	if !model.IsValidId(p.LastUserId) {
+		return false
+	}
+
+	switch p.CurrentTable {
+	case "TeamMembers":
+	case "ChannelMembers":
+	default:
+		return false
+	}
+
+	return true
+}
+
+func (worker *Worker) runAdvancedPermissionsPhase2Migration(lastDone string) (bool, string, *model.AppError) {
+	var progress *AdvancedPermissionsPhase2Progress
+	if lastDone == "" {
+		// Haven't started the migration yet.
+		progress = &AdvancedPermissionsPhase2Progress{
+			CurrentTable:  "TeamMembers",
+			LastChannelId: strings.Repeat("0", 26),
+			LastTeamId:    strings.Repeat("0", 26),
+			LastUserId:    strings.Repeat("0", 26),
+		}
+	} else {
+		err := json.NewDecoder(strings.NewReader(lastDone)).Decode(&progress)
+		if err != nil {
+			return false, "", model.NewAppError("MigrationsWorker.runAdvancedPermissionsPhase2Migration", "migrations.worker.run_advanced_permissions_phase_2_migration.invalid_progress", map[string]any{"lastDone": lastDone}, "", http.StatusInternalServerError).Wrap(err)
+		}
+		if !progress.IsValid() {
+			return false, "", model.NewAppError("MigrationsWorker.runAdvancedPermissionsPhase2Migration", "migrations.worker.run_advanced_permissions_phase_2_migration.invalid_progress", map[string]any{"progress": progress.ToJSON()}, "", http.StatusInternalServerError)
+		}
+	}
+
+	if progress.CurrentTable == "TeamMembers" {
+		// Run a TeamMembers migration batch.
+		result, err := worker.store.Team().MigrateTeamMembers(progress.LastTeamId, progress.LastUserId)
+		if err != nil {
+			return false, progress.ToJSON(), model.NewAppError("MigrationsWorker.runAdvancedPermissionsPhase2Migration", "app.team.migrate_team_members.update.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+		if result == nil {
+			// We haven't progressed. That means that we've reached the end of this stage of the migration, and should now advance to the next stage.
+			progress.LastUserId = strings.Repeat("0", 26)
+			progress.CurrentTable = "ChannelMembers"
+			return false, progress.ToJSON(), nil
+		}
+
+		progress.LastTeamId = result["TeamId"]
+		progress.LastUserId = result["UserId"]
+	} else if progress.CurrentTable == "ChannelMembers" {
+		// Run a ChannelMembers migration batch.
+		data, err := worker.store.Channel().MigrateChannelMembers(progress.LastChannelId, progress.LastUserId)
+		if err != nil {
+			return false, progress.ToJSON(), model.NewAppError("MigrationsWorker.runAdvancedPermissionsPhase2Migration", "app.channel.migrate_channel_members.select.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+		if data == nil {
+			// We haven't progressed. That means we've reached the end of this final stage of the migration.
+
+			return true, progress.ToJSON(), nil
+		}
+
+		progress.LastChannelId = data["ChannelId"]
+		progress.LastUserId = data["UserId"]
+	}
+
+	return false, progress.ToJSON(), nil
+}

server/channels/jobs/migrations/helper_test.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+)
+
+func Setup(tb testing.TB) store.Store {
+	store := mainHelper.GetStore()
+	store.DropAllTables()
+	return store
+}
+
+func deleteAllJobsByTypeAndMigrationKey(store store.Store, jobType string, migrationKey string) {
+	jobs, err := store.Job().GetAllByType(model.JobTypeMigrations)
+	if err != nil {
+		panic(err)
+	}
+
+	for _, job := range jobs {
+		if key, ok := job.Data[JobDataKeyMigration]; ok && key == migrationKey {
+			if _, err = store.Job().Delete(job.Id); err != nil {
+				panic(err)
+			}
+		}
+	}
+}

server/channels/jobs/migrations/main_test.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost-server/v6/server/channels/testlib"
+)
+
+var mainHelper *testlib.MainHelper
+
+func TestMain(m *testing.M) {
+	var options = testlib.HelperOptions{
+		EnableStore:     true,
+		EnableResources: true,
+	}
+
+	mainHelper = testlib.NewMainHelperWithOptions(&options)
+	defer mainHelper.Close()
+
+	mainHelper.Main(m)
+}

server/channels/jobs/migrations/migrations.go

@@ -0,0 +1,54 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"net/http"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+)
+
+const (
+	MigrationStateUnscheduled = "unscheduled"
+	MigrationStateInProgress  = "in_progress"
+	MigrationStateCompleted   = "completed"
+
+	JobDataKeyMigration         = "migration_key"
+	JobDataKeyMigrationLastDone = "last_done"
+)
+
+func MakeMigrationsList() []string {
+	return []string{
+		model.MigrationKeyAdvancedPermissionsPhase2,
+	}
+}
+
+func GetMigrationState(migration string, store store.Store) (string, *model.Job, *model.AppError) {
+	if _, err := store.System().GetByName(migration); err == nil {
+		return MigrationStateCompleted, nil, nil
+	}
+
+	jobs, err := store.Job().GetAllByType(model.JobTypeMigrations)
+	if err != nil {
+		return "", nil, model.NewAppError("GetMigrationState", "app.job.get_all.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	for _, job := range jobs {
+		if key, ok := job.Data[JobDataKeyMigration]; ok {
+			if key != migration {
+				continue
+			}
+
+			switch job.Status {
+			case model.JobStatusInProgress, model.JobStatusPending:
+				return MigrationStateInProgress, job, nil
+			default:
+				return MigrationStateUnscheduled, job, nil
+			}
+		}
+	}
+
+	return MigrationStateUnscheduled, nil, nil
+}

server/channels/jobs/migrations/migrations_test.go

@@ -0,0 +1,103 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+func TestGetMigrationState(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+	}
+	store := Setup(t)
+
+	migrationKey := model.NewId()
+
+	deleteAllJobsByTypeAndMigrationKey(store, model.JobTypeMigrations, migrationKey)
+
+	// Test with no job yet.
+	state, job, err := GetMigrationState(migrationKey, store)
+	assert.Nil(t, err)
+	assert.Nil(t, job)
+	assert.Equal(t, "unscheduled", state)
+
+	// Test with the system table showing the migration as done.
+	system := model.System{
+		Name:  migrationKey,
+		Value: "true",
+	}
+	nErr := store.System().Save(&system)
+	assert.NoError(t, nErr)
+
+	state, job, err = GetMigrationState(migrationKey, store)
+	assert.Nil(t, err)
+	assert.Nil(t, job)
+	assert.Equal(t, "completed", state)
+
+	_, nErr = store.System().PermanentDeleteByName(migrationKey)
+	assert.NoError(t, nErr)
+
+	// Test with a job scheduled in "pending" state.
+	j1 := &model.Job{
+		Id:       model.NewId(),
+		CreateAt: model.GetMillis(),
+		Data: map[string]string{
+			JobDataKeyMigration: migrationKey,
+		},
+		Status: model.JobStatusPending,
+		Type:   model.JobTypeMigrations,
+	}
+
+	j1, nErr = store.Job().Save(j1)
+	require.NoError(t, nErr)
+
+	state, job, err = GetMigrationState(migrationKey, store)
+	assert.Nil(t, err)
+	assert.Equal(t, j1.Id, job.Id)
+	assert.Equal(t, "in_progress", state)
+
+	// Test with a job scheduled in "in progress" state.
+	j2 := &model.Job{
+		Id:       model.NewId(),
+		CreateAt: j1.CreateAt + 1,
+		Data: map[string]string{
+			JobDataKeyMigration: migrationKey,
+		},
+		Status: model.JobStatusInProgress,
+		Type:   model.JobTypeMigrations,
+	}
+
+	j2, nErr = store.Job().Save(j2)
+	require.NoError(t, nErr)
+
+	state, job, err = GetMigrationState(migrationKey, store)
+	assert.Nil(t, err)
+	assert.Equal(t, j2.Id, job.Id)
+	assert.Equal(t, "in_progress", state)
+
+	// Test with a job scheduled in "error" state.
+	j3 := &model.Job{
+		Id:       model.NewId(),
+		CreateAt: j2.CreateAt + 1,
+		Data: map[string]string{
+			JobDataKeyMigration: migrationKey,
+		},
+		Status: model.JobStatusError,
+		Type:   model.JobTypeMigrations,
+	}
+
+	j3, nErr = store.Job().Save(j3)
+	require.NoError(t, nErr)
+
+	state, job, err = GetMigrationState(migrationKey, store)
+	assert.Nil(t, err)
+	assert.Equal(t, j3.Id, job.Id)
+	assert.Equal(t, "unscheduled", state)
+}

server/channels/jobs/migrations/scheduler.go

@@ -0,0 +1,105 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const (
+	MigrationJobWedgedTimeoutMilliseconds = 3600000 // 1 hour
+)
+
+type Scheduler struct {
+	jobServer              *jobs.JobServer
+	store                  store.Store
+	allMigrationsCompleted bool
+}
+
+func MakeScheduler(jobServer *jobs.JobServer, store store.Store) model.Scheduler {
+	return &Scheduler{jobServer, store, false}
+}
+
+func (scheduler *Scheduler) Enabled(_ *model.Config) bool {
+	return true
+}
+
+//nolint:unparam
+func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
+	if scheduler.allMigrationsCompleted {
+		return nil
+	}
+
+	nextTime := time.Now().Add(60 * time.Second)
+	return &nextTime
+}
+
+//nolint:unparam
+func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) {
+	mlog.Debug("Scheduling Job", mlog.String("scheduler", model.JobTypeMigrations))
+
+	// Work through the list of migrations in order. Schedule the first one that isn't done (assuming it isn't in progress already).
+	for _, key := range MakeMigrationsList() {
+		state, job, err := GetMigrationState(key, scheduler.store)
+		if err != nil {
+			mlog.Error("Failed to determine status of migration: ", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("migration_key", key), mlog.Err(err))
+			return nil, nil
+		}
+
+		if state == MigrationStateInProgress {
+			// Check the migration job isn't wedged.
+			if job != nil && job.LastActivityAt < model.GetMillis()-MigrationJobWedgedTimeoutMilliseconds && job.CreateAt < model.GetMillis()-MigrationJobWedgedTimeoutMilliseconds {
+				mlog.Warn("Job appears to be wedged. Rescheduling another instance.", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("wedged_job_id", job.Id), mlog.String("migration_key", key))
+				if err := scheduler.jobServer.SetJobError(job, nil); err != nil {
+					mlog.Error("Worker: Failed to set job error", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("job_id", job.Id), mlog.Err(err))
+				}
+				return scheduler.createJob(key, job)
+			}
+
+			return nil, nil
+		}
+
+		if state == MigrationStateCompleted {
+			// This migration is done. Continue to check the next.
+			continue
+		}
+
+		if state == MigrationStateUnscheduled {
+			mlog.Debug("Scheduling a new job for migration.", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("migration_key", key))
+			return scheduler.createJob(key, job)
+		}
+
+		mlog.Error("Unknown migration state. Not doing anything.", mlog.String("migration_state", state))
+		return nil, nil
+	}
+
+	// If we reached here, then there aren't any migrations left to run.
+	scheduler.allMigrationsCompleted = true
+	mlog.Debug("All migrations are complete.", mlog.String("scheduler", model.JobTypeMigrations))
+
+	return nil, nil
+}
+
+func (scheduler *Scheduler) createJob(migrationKey string, lastJob *model.Job) (*model.Job, *model.AppError) {
+	var lastDone string
+	if lastJob != nil {
+		lastDone = lastJob.Data[JobDataKeyMigrationLastDone]
+	}
+
+	data := map[string]string{
+		JobDataKeyMigration:         migrationKey,
+		JobDataKeyMigrationLastDone: lastDone,
+	}
+
+	job, err := scheduler.jobServer.CreateJob(model.JobTypeMigrations, data)
+	if err != nil {
+		return nil, err
+	}
+	return job, nil
+}

server/channels/jobs/migrations/worker.go

@@ -0,0 +1,182 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package migrations
+
+import (
+	"context"
+	"net/http"
+	"sync/atomic"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const (
+	TimeBetweenBatches = 100
+)
+
+type Worker struct {
+	name      string
+	stop      chan struct{}
+	stopped   chan bool
+	jobs      chan model.Job
+	jobServer *jobs.JobServer
+	store     store.Store
+	closed    int32
+}
+
+func MakeWorker(jobServer *jobs.JobServer, store store.Store) model.Worker {
+	worker := Worker{
+		name:      "Migrations",
+		stop:      make(chan struct{}),
+		stopped:   make(chan bool, 1),
+		jobs:      make(chan model.Job),
+		jobServer: jobServer,
+		store:     store,
+	}
+
+	return &worker
+}
+
+func (worker *Worker) Run() {
+	// Set to open if closed before. We are not bothered about multiple opens.
+	if atomic.CompareAndSwapInt32(&worker.closed, 1, 0) {
+		worker.stop = make(chan struct{})
+	}
+	mlog.Debug("Worker started", mlog.String("worker", worker.name))
+
+	defer func() {
+		mlog.Debug("Worker finished", mlog.String("worker", worker.name))
+		worker.stopped <- true
+	}()
+
+	for {
+		select {
+		case <-worker.stop:
+			mlog.Debug("Worker received stop signal", mlog.String("worker", worker.name))
+			return
+		case job := <-worker.jobs:
+			mlog.Debug("Worker received a new candidate job.", mlog.String("worker", worker.name))
+			worker.DoJob(&job)
+		}
+	}
+}
+
+func (worker *Worker) Stop() {
+	// Set to close, and if already closed before, then return.
+	if !atomic.CompareAndSwapInt32(&worker.closed, 0, 1) {
+		return
+	}
+	mlog.Debug("Worker stopping", mlog.String("worker", worker.name))
+	close(worker.stop)
+	<-worker.stopped
+}
+
+func (worker *Worker) JobChannel() chan<- model.Job {
+	return worker.jobs
+}
+
+func (worker *Worker) IsEnabled(_ *model.Config) bool {
+	return true
+}
+
+func (worker *Worker) DoJob(job *model.Job) {
+	defer worker.jobServer.HandleJobPanic(job)
+
+	if claimed, err := worker.jobServer.ClaimJob(job); err != nil {
+		mlog.Info("Worker experienced an error while trying to claim job",
+			mlog.String("worker", worker.name),
+			mlog.String("job_id", job.Id),
+			mlog.String("error", err.Error()))
+		return
+	} else if !claimed {
+		return
+	}
+
+	cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background())
+	cancelWatcherChan := make(chan struct{}, 1)
+	go worker.jobServer.CancellationWatcher(cancelCtx, job.Id, cancelWatcherChan)
+
+	defer cancelCancelWatcher()
+
+	for {
+		select {
+		case <-cancelWatcherChan:
+			mlog.Debug("Worker: Job has been canceled via CancellationWatcher", mlog.String("worker", worker.name), mlog.String("job_id", job.Id))
+			worker.setJobCanceled(job)
+			return
+
+		case <-worker.stop:
+			mlog.Debug("Worker: Job has been canceled via Worker Stop", mlog.String("worker", worker.name), mlog.String("job_id", job.Id))
+			worker.setJobCanceled(job)
+			return
+
+		case <-time.After(TimeBetweenBatches * time.Millisecond):
+			done, progress, err := worker.runMigration(job.Data[JobDataKeyMigration], job.Data[JobDataKeyMigrationLastDone])
+			if err != nil {
+				mlog.Error("Worker: Failed to run migration", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
+				worker.setJobError(job, err)
+				return
+			} else if done {
+				mlog.Info("Worker: Job is complete", mlog.String("worker", worker.name), mlog.String("job_id", job.Id))
+				worker.setJobSuccess(job)
+				return
+			} else {
+				job.Data[JobDataKeyMigrationLastDone] = progress
+				if err := worker.jobServer.UpdateInProgressJobData(job); err != nil {
+					mlog.Error("Worker: Failed to update migration status data for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
+					worker.setJobError(job, err)
+					return
+				}
+			}
+		}
+	}
+}
+
+func (worker *Worker) setJobSuccess(job *model.Job) {
+	if err := worker.jobServer.SetJobSuccess(job); err != nil {
+		mlog.Error("Worker: Failed to set success for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
+		worker.setJobError(job, err)
+	}
+}
+
+func (worker *Worker) setJobError(job *model.Job, appError *model.AppError) {
+	if err := worker.jobServer.SetJobError(job, appError); err != nil {
+		mlog.Error("Worker: Failed to set job error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
+	}
+}
+
+func (worker *Worker) setJobCanceled(job *model.Job) {
+	if err := worker.jobServer.SetJobCanceled(job); err != nil {
+		mlog.Error("Worker: Failed to mark job as canceled", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
+	}
+}
+
+// Return parameters:
+// - whether the migration is completed on this run (true) or still incomplete (false).
+// - the updated lastDone string for the migration.
+// - any error which may have occurred while running the migration.
+func (worker *Worker) runMigration(key string, lastDone string) (bool, string, *model.AppError) {
+	var done bool
+	var progress string
+	var err *model.AppError
+
+	switch key {
+	case model.MigrationKeyAdvancedPermissionsPhase2:
+		done, progress, err = worker.runAdvancedPermissionsPhase2Migration(lastDone)
+	default:
+		return false, "", model.NewAppError("MigrationsWorker.runMigration", "migrations.worker.run_migration.unknown_key", map[string]any{"key": key}, "", http.StatusInternalServerError)
+	}
+
+	if done {
+		if nErr := worker.store.System().Save(&model.System{Name: key, Value: "true"}); nErr != nil {
+			return false, "", model.NewAppError("runMigration", "migrations.system.save.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
+		}
+	}
+
+	return done, progress, err
+}