Mono repo -> Master (#22553)

Combines the following repositories into one:

https://github.com/mattermost/mattermost-server
https://github.com/mattermost/mattermost-webapp
https://github.com/mattermost/focalboard
https://github.com/mattermost/mattermost-plugin-playbooks

server/playbooks/product/pluginapi/cluster/job.go

@@ -0,0 +1,232 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package cluster
+
+import (
+	"encoding/json"
+	"sync"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	// cronPrefix is used to namespace key values created for a job from other key values
+	// created by a plugin.
+	cronPrefix = "cron_"
+)
+
+// JobPluginAPI is the plugin API interface required to schedule jobs.
+type JobPluginAPI interface {
+	MutexPluginAPI
+	KVGet(key string) ([]byte, error)
+	KVDelete(key string) error
+	KVList(page, count int) ([]string, error)
+}
+
+// JobConfig defines the configuration of a scheduled job.
+type JobConfig struct {
+	// Interval is the period of execution for the job.
+	Interval time.Duration
+}
+
+// NextWaitInterval is a callback computing the next wait interval for a job.
+type NextWaitInterval func(now time.Time, metadata JobMetadata) time.Duration
+
+// MakeWaitForInterval creates a function to scheduling a job to run on the given interval relative
+// to the last finished timestamp.
+//
+// For example, if the job first starts at 12:01 PM, and is configured with interval 5 minutes,
+// it will next run at:
+//
+//	12:06, 12:11, 12:16, ...
+//
+// If the job has not previously started, it will run immediately.
+func MakeWaitForInterval(interval time.Duration) NextWaitInterval {
+	if interval == 0 {
+		panic("must specify non-zero ready interval")
+	}
+
+	return func(now time.Time, metadata JobMetadata) time.Duration {
+		sinceLastFinished := now.Sub(metadata.LastFinished)
+		if sinceLastFinished < interval {
+			return interval - sinceLastFinished
+		}
+
+		return 0
+	}
+}
+
+// MakeWaitForRoundedInterval creates a function, scheduling a job to run on the nearest rounded
+// interval relative to the last finished timestamp.
+//
+// For example, if the job first starts at 12:04 PM, and is configured with interval 5 minutes,
+// and is configured to round to 5 minute intervals, it will next run at:
+//
+//	12:05 PM, 12:10 PM, 12:15 PM, ...
+//
+// If the job has not previously started, it will run immediately. Note that this wait interval
+// strategy does not guarantee a minimum interval between runs, only that subsequent runs will be
+// scheduled on the rounded interval.
+func MakeWaitForRoundedInterval(interval time.Duration) NextWaitInterval {
+	if interval == 0 {
+		panic("must specify non-zero ready interval")
+	}
+
+	return func(now time.Time, metadata JobMetadata) time.Duration {
+		if metadata.LastFinished.IsZero() {
+			return 0
+		}
+
+		target := metadata.LastFinished.Add(interval).Truncate(interval)
+		untilTarget := target.Sub(now)
+		if untilTarget > 0 {
+			return untilTarget
+		}
+
+		return 0
+	}
+}
+
+// Job is a scheduled job whose callback function is executed on a configured interval by at most
+// one plugin instance at a time.
+//
+// Use scheduled jobs to perform background activity on a regular interval without having to
+// explicitly coordinate with other instances of the same plugin that might repeat that effort.
+type Job struct {
+	pluginAPI        JobPluginAPI
+	key              string
+	mutex            *Mutex
+	nextWaitInterval NextWaitInterval
+	callback         func()
+
+	stopOnce sync.Once
+	stop     chan bool
+	done     chan bool
+}
+
+// JobMetadata persists metadata about job execution.
+type JobMetadata struct {
+	// LastFinished is the last time the job finished anywhere in the cluster.
+	LastFinished time.Time
+}
+
+// Schedule creates a scheduled job.
+func Schedule(pluginAPI JobPluginAPI, key string, nextWaitInterval NextWaitInterval, callback func()) (*Job, error) {
+	key = cronPrefix + key
+
+	mutex, err := NewMutex(pluginAPI, key)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create job mutex")
+	}
+
+	job := &Job{
+		pluginAPI:        pluginAPI,
+		key:              key,
+		mutex:            mutex,
+		nextWaitInterval: nextWaitInterval,
+		callback:         callback,
+		stop:             make(chan bool),
+		done:             make(chan bool),
+	}
+
+	go job.run()
+
+	return job, nil
+}
+
+// readMetadata reads the job execution metadata from the kv store.
+func (j *Job) readMetadata() (JobMetadata, error) {
+	data, appErr := j.pluginAPI.KVGet(j.key)
+	if appErr != nil {
+		return JobMetadata{}, errors.Wrap(appErr, "failed to read data")
+	}
+
+	if data == nil {
+		return JobMetadata{}, nil
+	}
+
+	var metadata JobMetadata
+	err := json.Unmarshal(data, &metadata)
+	if err != nil {
+		return JobMetadata{}, errors.Wrap(err, "failed to decode data")
+	}
+
+	return metadata, nil
+}
+
+// saveMetadata writes updated job execution metadata from the kv store.
+//
+// It is assumed that the job mutex is held, negating the need to require an atomic write.
+func (j *Job) saveMetadata(metadata JobMetadata) error {
+	data, err := json.Marshal(metadata)
+	if err != nil {
+		return errors.Wrap(err, "failed to marshal data")
+	}
+
+	ok, appErr := j.pluginAPI.KVSetWithOptions(j.key, data, model.PluginKVSetOptions{})
+	if appErr != nil || !ok {
+		return errors.Wrap(appErr, "failed to set data")
+	}
+
+	return nil
+}
+
+// run attempts to run the scheduled job, guaranteeing only one instance is executing concurrently.
+func (j *Job) run() {
+	defer close(j.done)
+
+	var waitInterval time.Duration
+
+	for {
+		select {
+		case <-j.stop:
+			return
+		case <-time.After(waitInterval):
+		}
+
+		func() {
+			// Acquire the corresponding job lock and hold it throughout execution.
+			j.mutex.Lock()
+			defer j.mutex.Unlock()
+
+			metadata, err := j.readMetadata()
+			if err != nil {
+				logrus.WithError(err).WithField("key", j.key).Error("failed to read job metadata")
+				waitInterval = nextWaitInterval(waitInterval, err)
+				return
+			}
+
+			// Is it time to run the job?
+			waitInterval = j.nextWaitInterval(time.Now(), metadata)
+			if waitInterval > 0 {
+				return
+			}
+
+			// Run the job
+			j.callback()
+
+			metadata.LastFinished = time.Now()
+
+			err = j.saveMetadata(metadata)
+			if err != nil {
+				logrus.WithError(err).WithField("key", j.key).Error("failed to write job data")
+			}
+
+			waitInterval = j.nextWaitInterval(time.Now(), metadata)
+		}()
+	}
+}
+
+// Close terminates a scheduled job, preventing it from being scheduled on this plugin instance.
+func (j *Job) Close() error {
+	j.stopOnce.Do(func() {
+		close(j.stop)
+	})
+	<-j.done
+
+	return nil
+}

server/playbooks/product/pluginapi/cluster/job_once.go

@@ -0,0 +1,212 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package cluster
+
+import (
+	"encoding/json"
+	"math/rand"
+	"sync"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/pkg/errors"
+)
+
+const (
+	// oncePrefix is used to namespace key values created for a scheduleOnce job
+	oncePrefix = "once_"
+
+	// keysPerPage is the maximum number of keys to retrieve from the db per call
+	keysPerPage = 1000
+
+	// maxNumFails is the maximum number of KVStore read fails or failed attempts to run the
+	// callback until the scheduler cancels a job.
+	maxNumFails = 3
+
+	// waitAfterFail is the amount of time to wait after a failure
+	waitAfterFail = 1 * time.Second
+
+	// pollNewJobsInterval is the amount of time to wait between polling the db for new scheduled jobs
+	pollNewJobsInterval = 5 * time.Minute
+
+	// scheduleOnceJitter is the range of jitter to add to intervals to avoid contention issues
+	scheduleOnceJitter = 100 * time.Millisecond
+)
+
+type JobOnceMetadata struct {
+	Key   string
+	RunAt time.Time
+}
+
+type JobOnce struct {
+	pluginAPI    JobPluginAPI
+	clusterMutex *Mutex
+
+	// key is the original key. It is prefixed with oncePrefix when used as a key in the KVStore
+	key      string
+	runAt    time.Time
+	numFails int
+
+	// done signals the job.run go routine to exit
+	done     chan bool
+	doneOnce sync.Once
+
+	// join is a join point for the job.run() goroutine to join the calling goroutine (in this case,
+	// the one calling job.Cancel)
+	join     chan bool
+	joinOnce sync.Once
+
+	storedCallback *syncedCallback
+	activeJobs     *syncedJobs
+}
+
+// Cancel terminates a scheduled job, preventing it from being scheduled on this plugin instance.
+// It also removes the job from the db, preventing it from being run in the future.
+func (j *JobOnce) Cancel() {
+	j.clusterMutex.Lock()
+	defer j.clusterMutex.Unlock()
+
+	j.cancelWhileHoldingMutex()
+
+	// join the running goroutine
+	j.joinOnce.Do(func() {
+		<-j.join
+	})
+}
+
+func newJobOnce(pluginAPI JobPluginAPI, key string, runAt time.Time, callback *syncedCallback, jobs *syncedJobs) (*JobOnce, error) {
+	mutex, err := NewMutex(pluginAPI, key)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create job mutex")
+	}
+
+	return &JobOnce{
+		pluginAPI:      pluginAPI,
+		clusterMutex:   mutex,
+		key:            key,
+		runAt:          runAt,
+		done:           make(chan bool),
+		join:           make(chan bool),
+		storedCallback: callback,
+		activeJobs:     jobs,
+	}, nil
+}
+
+func (j *JobOnce) run() {
+	defer close(j.join)
+
+	wait := time.Until(j.runAt)
+
+	for {
+		select {
+		case <-j.done:
+			return
+		case <-time.After(wait + addJitter()):
+		}
+
+		func() {
+			// Acquire the cluster mutex while we're trying to do the job
+			j.clusterMutex.Lock()
+			defer j.clusterMutex.Unlock()
+
+			// Check that the job has not been completed
+			metadata, err := readMetadata(j.pluginAPI, j.key)
+			if err != nil {
+				j.numFails++
+				if j.numFails > maxNumFails {
+					j.cancelWhileHoldingMutex()
+					return
+				}
+
+				// wait a bit of time and try again
+				wait = waitAfterFail
+				return
+			}
+
+			// If key doesn't exist, or if the runAt has changed, the original job has been completed already
+			if metadata == nil || !j.runAt.Equal(metadata.RunAt) {
+				j.cancelWhileHoldingMutex()
+				return
+			}
+
+			j.executeJob()
+
+			j.cancelWhileHoldingMutex()
+		}()
+	}
+}
+
+func (j *JobOnce) executeJob() {
+	j.storedCallback.mu.Lock()
+	defer j.storedCallback.mu.Unlock()
+
+	j.storedCallback.callback(j.key)
+}
+
+// readMetadata reads the job's stored metadata. If the caller wishes to make an atomic
+// read/write, the cluster mutex for job's key should be held.
+func readMetadata(pluginAPI JobPluginAPI, key string) (*JobOnceMetadata, error) {
+	data, err := pluginAPI.KVGet(oncePrefix + key)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read data")
+	}
+
+	if data == nil {
+		return nil, nil
+	}
+
+	var metadata JobOnceMetadata
+	if err := json.Unmarshal(data, &metadata); err != nil {
+		return nil, errors.Wrap(err, "failed to decode data")
+	}
+
+	return &metadata, nil
+}
+
+// saveMetadata writes the job's metadata to the kvstore. saveMetadata acquires the job's cluster lock.
+// saveMetadata will not overwrite an existing key.
+func (j *JobOnce) saveMetadata() error {
+	j.clusterMutex.Lock()
+	defer j.clusterMutex.Unlock()
+
+	metadata := JobOnceMetadata{
+		Key:   j.key,
+		RunAt: j.runAt,
+	}
+	data, err := json.Marshal(metadata)
+	if err != nil {
+		return errors.Wrap(err, "failed to marshal data")
+	}
+
+	ok, err := j.pluginAPI.KVSetWithOptions(oncePrefix+j.key, data, model.PluginKVSetOptions{
+		Atomic:   true,
+		OldValue: nil,
+	})
+	if err != nil {
+		return err
+	}
+	if !ok {
+		return errors.New("failed to set data")
+	}
+
+	return nil
+}
+
+// cancelWhileHoldingMutex assumes the caller holds the job's mutex.
+func (j *JobOnce) cancelWhileHoldingMutex() {
+	// remove the job from the kv store, if it exists
+	_ = j.pluginAPI.KVDelete(oncePrefix + j.key)
+
+	j.activeJobs.mu.Lock()
+	defer j.activeJobs.mu.Unlock()
+	delete(j.activeJobs.jobs, j.key)
+
+	j.doneOnce.Do(func() {
+		close(j.done)
+	})
+}
+
+func addJitter() time.Duration {
+	return time.Duration(rand.Int63n(int64(scheduleOnceJitter)))
+}

server/playbooks/product/pluginapi/cluster/job_once_scheduler.go

@@ -0,0 +1,231 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package cluster
+
+import (
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// syncedCallback uses the mutex to make things predictable for the client: the callback will be
+// called once at a time (the client does not need to worry about concurrency within the callback)
+type syncedCallback struct {
+	mu       sync.Mutex
+	callback func(string)
+}
+
+type syncedJobs struct {
+	mu   sync.RWMutex
+	jobs map[string]*JobOnce
+}
+
+type JobOnceScheduler struct {
+	pluginAPI JobPluginAPI
+
+	startedMu sync.RWMutex
+	started   bool
+
+	activeJobs     *syncedJobs
+	storedCallback *syncedCallback
+}
+
+// GetJobOnceScheduler returns a scheduler which is ready to have its callback set. Repeated
+// calls will return the same scheduler.
+func GetJobOnceScheduler(pluginAPI JobPluginAPI) *JobOnceScheduler {
+	return &JobOnceScheduler{
+		pluginAPI: pluginAPI,
+		activeJobs: &syncedJobs{
+			jobs: make(map[string]*JobOnce),
+		},
+		storedCallback: &syncedCallback{},
+	}
+}
+
+// Start starts the Scheduler. It finds all previous ScheduleOnce jobs and starts them running, and
+// fires any jobs that have reached or exceeded their runAt time. Thus, even if a cluster goes down
+// and is restarted, Start will restart previously scheduled jobs.
+func (s *JobOnceScheduler) Start() error {
+	s.startedMu.Lock()
+	defer s.startedMu.Unlock()
+	if s.started {
+		return errors.New("scheduler has already been started")
+	}
+
+	if err := s.verifyCallbackExists(); err != nil {
+		return errors.Wrap(err, "callback not found; cannot start scheduler")
+	}
+
+	if err := s.scheduleNewJobsFromDB(); err != nil {
+		return errors.Wrap(err, "could not start JobOnceScheduler due to error")
+	}
+
+	go s.pollForNewScheduledJobs()
+
+	s.started = true
+
+	return nil
+}
+
+// SetCallback sets the scheduler's callback. When a job fires, the callback will be called with
+// the job's id.
+func (s *JobOnceScheduler) SetCallback(callback func(string)) error {
+	if callback == nil {
+		return errors.New("callback cannot be nil")
+	}
+
+	s.storedCallback.mu.Lock()
+	defer s.storedCallback.mu.Unlock()
+
+	s.storedCallback.callback = callback
+	return nil
+}
+
+// ListScheduledJobs returns a list of the jobs in the db that have been scheduled. There is no
+// guarantee that list is accurate by the time the caller reads the list. E.g., the jobs in the list
+// may have been run, canceled, or new jobs may have scheduled.
+func (s *JobOnceScheduler) ListScheduledJobs() ([]JobOnceMetadata, error) {
+	var ret []JobOnceMetadata
+	for i := 0; ; i++ {
+		keys, err := s.pluginAPI.KVList(i, keysPerPage)
+		if err != nil {
+			return nil, errors.Wrap(err, "error getting KVList")
+		}
+		for _, k := range keys {
+			if strings.HasPrefix(k, oncePrefix) {
+				metadata, err := readMetadata(s.pluginAPI, k[len(oncePrefix):])
+				if err != nil {
+					logrus.WithError(err).WithField("key", k).Error("could not retrieve data from plugin kvstore")
+					continue
+				}
+				if metadata == nil {
+					continue
+				}
+
+				ret = append(ret, *metadata)
+			}
+		}
+
+		if len(keys) < keysPerPage {
+			break
+		}
+	}
+
+	return ret, nil
+}
+
+// ScheduleOnce creates a scheduled job that will run once. When the clock reaches runAt, the
+// callback will be called with key as the argument.
+//
+// If the job key already exists in the db, this will return an error. To reschedule a job, first
+// cancel the original then schedule it again.
+func (s *JobOnceScheduler) ScheduleOnce(key string, runAt time.Time) (*JobOnce, error) {
+	s.startedMu.RLock()
+	defer s.startedMu.RUnlock()
+	if !s.started {
+		return nil, errors.New("start the scheduler before adding jobs")
+	}
+
+	job, err := newJobOnce(s.pluginAPI, key, runAt, s.storedCallback, s.activeJobs)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create new job")
+	}
+
+	if err = job.saveMetadata(); err != nil {
+		return nil, errors.Wrap(err, "could not save job metadata")
+	}
+
+	s.runAndTrack(job)
+
+	return job, nil
+}
+
+// Cancel cancels a job by its key. This is useful if the plugin lost the original *JobOnce, or
+// is stopping a job found in ListScheduledJobs().
+func (s *JobOnceScheduler) Cancel(key string) {
+	// using an anonymous function because job.Close() below needs access to the activeJobs mutex
+	job := func() *JobOnce {
+		s.activeJobs.mu.RLock()
+		defer s.activeJobs.mu.RUnlock()
+		j, ok := s.activeJobs.jobs[key]
+		if ok {
+			return j
+		}
+
+		// Job wasn't active, so no need to call CancelWhileHoldingMutex (which shuts down the
+		// goroutine). There's a condition where another server in the cluster started the job, and
+		// the current server hasn't polled for it yet. To solve that case, delete it from the db.
+		mutex, err := NewMutex(s.pluginAPI, key)
+		if err != nil {
+			logrus.WithError(err).WithField("key", key).Error("failed to create job mutex in Cancel")
+		}
+		mutex.Lock()
+		defer mutex.Unlock()
+
+		_ = s.pluginAPI.KVDelete(oncePrefix + key)
+
+		return nil
+	}()
+
+	if job != nil {
+		job.Cancel()
+	}
+}
+
+func (s *JobOnceScheduler) scheduleNewJobsFromDB() error {
+	scheduled, err := s.ListScheduledJobs()
+	if err != nil {
+		return errors.Wrap(err, "could not read scheduled jobs from db")
+	}
+
+	for _, m := range scheduled {
+		job, err := newJobOnce(s.pluginAPI, m.Key, m.RunAt, s.storedCallback, s.activeJobs)
+		if err != nil {
+			logrus.WithError(err).WithField("key", m.Key).Error("could not create new job")
+			continue
+		}
+
+		s.runAndTrack(job)
+	}
+
+	return nil
+}
+
+func (s *JobOnceScheduler) runAndTrack(job *JobOnce) {
+	s.activeJobs.mu.Lock()
+	defer s.activeJobs.mu.Unlock()
+
+	// has this been scheduled already on this server?
+	if _, ok := s.activeJobs.jobs[job.key]; ok {
+		return
+	}
+
+	go job.run()
+
+	s.activeJobs.jobs[job.key] = job
+}
+
+// pollForNewScheduledJobs will only be started once per plugin. It doesn't need to be stopped.
+func (s *JobOnceScheduler) pollForNewScheduledJobs() {
+	for {
+		<-time.After(pollNewJobsInterval + addJitter())
+
+		if err := s.scheduleNewJobsFromDB(); err != nil {
+			logrus.WithError(err).Error("scheduleOnce poller encountered an error but is still polling")
+		}
+	}
+}
+
+func (s *JobOnceScheduler) verifyCallbackExists() error {
+	s.storedCallback.mu.Lock()
+	defer s.storedCallback.mu.Unlock()
+
+	if s.storedCallback.callback == nil {
+		return errors.New("set callback before starting the scheduler")
+	}
+	return nil
+}

server/playbooks/product/pluginapi/cluster/mutex.go

@@ -0,0 +1,188 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package cluster
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	// mutexPrefix is used to namespace key values created for a mutex from other key values
+	// created by a plugin.
+	mutexPrefix = "mutex_"
+)
+
+const (
+	// ttl is the interval after which a locked mutex will expire unless refreshed
+	ttl = time.Second * 15
+
+	// refreshInterval is the interval on which the mutex will be refreshed when locked
+	refreshInterval = ttl / 2
+)
+
+// MutexPluginAPI is the plugin API interface required to manage mutexes.
+type MutexPluginAPI interface {
+	KVSetWithOptions(key string, value []byte, options model.PluginKVSetOptions) (bool, error)
+}
+
+// Mutex is similar to sync.Mutex, except usable by multiple plugin instances across a cluster.
+//
+// Internally, a mutex relies on an atomic key-value set operation as exposed by the Mattermost
+// plugin API.
+//
+// Mutexes with different names are unrelated. Mutexes with the same name from different plugins
+// are unrelated. Pick a unique name for each mutex your plugin requires.
+//
+// A Mutex must not be copied after first use.
+type Mutex struct {
+	pluginAPI MutexPluginAPI
+	key       string
+
+	// lock guards the variables used to manage the refresh task, and is not itself related to
+	// the cluster-wide lock.
+	lock        sync.Mutex
+	stopRefresh chan bool
+	refreshDone chan bool
+}
+
+// NewMutex creates a mutex with the given key name.
+//
+// Panics if key is empty.
+func NewMutex(pluginAPI MutexPluginAPI, key string) (*Mutex, error) {
+	key, err := makeLockKey(key)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Mutex{
+		pluginAPI: pluginAPI,
+		key:       key,
+	}, nil
+}
+
+// makeLockKey returns the prefixed key used to namespace mutex keys.
+func makeLockKey(key string) (string, error) {
+	if key == "" {
+		return "", errors.New("must specify valid mutex key")
+	}
+
+	return mutexPrefix + key, nil
+}
+
+// lock makes a single attempt to atomically lock the mutex, returning true only if successful.
+func (m *Mutex) tryLock() (bool, error) {
+	ok, err := m.pluginAPI.KVSetWithOptions(m.key, []byte{1}, model.PluginKVSetOptions{
+		Atomic:          true,
+		OldValue:        nil, // No existing key value.
+		ExpireInSeconds: int64(ttl / time.Second),
+	})
+	if err != nil {
+		return false, errors.Wrap(err, "failed to set mutex kv")
+	}
+
+	return ok, nil
+}
+
+// refreshLock rewrites the lock key value with a new expiry, returning true only if successful.
+func (m *Mutex) refreshLock() error {
+	ok, err := m.pluginAPI.KVSetWithOptions(m.key, []byte{1}, model.PluginKVSetOptions{
+		Atomic:          true,
+		OldValue:        []byte{1},
+		ExpireInSeconds: int64(ttl / time.Second),
+	})
+	if err != nil {
+		return errors.Wrap(err, "failed to refresh mutex kv")
+	} else if !ok {
+		return errors.New("unexpectedly failed to refresh mutex kv")
+	}
+
+	return nil
+}
+
+// Lock locks m. If the mutex is already locked by any plugin instance, including the current one,
+// the calling goroutine blocks until the mutex can be locked.
+func (m *Mutex) Lock() {
+	_ = m.LockWithContext(context.Background())
+}
+
+// LockWithContext locks m unless the context is canceled. If the mutex is already locked by any plugin
+// instance, including the current one, the calling goroutine blocks until the mutex can be locked,
+// or the context is canceled.
+//
+// The mutex is locked only if a nil error is returned.
+func (m *Mutex) LockWithContext(ctx context.Context) error {
+	var waitInterval time.Duration
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(waitInterval):
+		}
+
+		locked, err := m.tryLock()
+		if err != nil {
+			logrus.WithError(err).WithField("lock_key", m.key).Error("failed to lock mutex")
+			waitInterval = nextWaitInterval(waitInterval, err)
+			continue
+		} else if !locked {
+			waitInterval = nextWaitInterval(waitInterval, err)
+			continue
+		}
+
+		stop := make(chan bool)
+		done := make(chan bool)
+		go func() {
+			defer close(done)
+			t := time.NewTicker(refreshInterval)
+			for {
+				select {
+				case <-t.C:
+					err := m.refreshLock()
+					if err != nil {
+						logrus.WithError(err).WithField("lock_key", m.key).Error("failed to refresh mutex")
+						return
+					}
+				case <-stop:
+					return
+				}
+			}
+		}()
+
+		m.lock.Lock()
+		m.stopRefresh = stop
+		m.refreshDone = done
+		m.lock.Unlock()
+
+		return nil
+	}
+}
+
+// Unlock unlocks m. It is a run-time error if m is not locked on entry to Unlock.
+//
+// Just like sync.Mutex, a locked Lock is not associated with a particular goroutine or plugin
+// instance. It is allowed for one goroutine or plugin instance to lock a Lock and then arrange
+// for another goroutine or plugin instance to unlock it. In practice, ownership of the lock should
+// remain within a single plugin instance.
+func (m *Mutex) Unlock() {
+	m.lock.Lock()
+	if m.stopRefresh == nil {
+		m.lock.Unlock()
+		panic("mutex has not been acquired")
+	}
+
+	close(m.stopRefresh)
+	m.stopRefresh = nil
+	<-m.refreshDone
+	m.lock.Unlock()
+
+	// If an error occurs deleting, the mutex kv will still expire, allowing later retry.
+	_, _ = m.pluginAPI.KVSetWithOptions(m.key, nil, model.PluginKVSetOptions{})
+}

server/playbooks/product/pluginapi/cluster/wait.go

@@ -0,0 +1,46 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package cluster
+
+import (
+	"math/rand"
+	"time"
+)
+
+const (
+	// minWaitInterval is the minimum amount of time to wait between locking attempts
+	minWaitInterval = 1 * time.Second
+
+	// maxWaitInterval is the maximum amount of time to wait between locking attempts
+	maxWaitInterval = 5 * time.Minute
+
+	// pollWaitInterval is the usual time to wait between unsuccessful locking attempts
+	pollWaitInterval = 1 * time.Second
+
+	// jitterWaitInterval is the amount of jitter to add when waiting to avoid thundering herds
+	jitterWaitInterval = minWaitInterval / 2
+)
+
+// nextWaitInterval determines how long to wait until the next lock retry.
+func nextWaitInterval(lastWaitInterval time.Duration, err error) time.Duration {
+	nextWaitInterval := lastWaitInterval
+
+	if nextWaitInterval <= 0 {
+		nextWaitInterval = minWaitInterval
+	}
+
+	if err != nil {
+		nextWaitInterval *= 2
+		if nextWaitInterval > maxWaitInterval {
+			nextWaitInterval = maxWaitInterval
+		}
+	} else {
+		nextWaitInterval = pollWaitInterval
+	}
+
+	// Add some jitter to avoid unnecessary collision between competing plugin instances.
+	nextWaitInterval += time.Duration(rand.Int63n(int64(jitterWaitInterval)) - int64(jitterWaitInterval)/2)
+
+	return nextWaitInterval
+}

server/playbooks/product/pluginapi/license.go

@@ -0,0 +1,110 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package pluginapi
+
+import (
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+const (
+	e10          = "E10"
+	e20          = "E20"
+	professional = "professional"
+	enterprise   = "enterprise"
+)
+
+// IsEnterpriseLicensedOrDevelopment returns true when the server is licensed with any Mattermost
+// Enterprise License, or has `EnableDeveloper` and `EnableTesting` configuration settings
+// enabled signaling a non-production, developer mode.
+func IsEnterpriseLicensedOrDevelopment(config *model.Config, license *model.License) bool {
+	if license != nil {
+		return true
+	}
+
+	return IsConfiguredForDevelopment(config)
+}
+
+// isValidSkuShortName returns whether the SKU short name is one of the known strings;
+// namely: E10 or professional, or E20 or enterprise
+func isValidSkuShortName(license *model.License) bool {
+	if license == nil {
+		return false
+	}
+
+	switch license.SkuShortName {
+	case e10, e20, professional, enterprise:
+		return true
+	default:
+		return false
+	}
+}
+
+// IsE10LicensedOrDevelopment returns true when the server is at least licensed with a legacy Mattermost
+// Enterprise E10 License or a Mattermost Professional License, or has `EnableDeveloper` and
+// `EnableTesting` configuration settings enabled, signaling a non-production, developer mode.
+func IsE10LicensedOrDevelopment(config *model.Config, license *model.License) bool {
+	if license != nil &&
+		(license.SkuShortName == e10 || license.SkuShortName == professional ||
+			license.SkuShortName == e20 || license.SkuShortName == enterprise) {
+		return true
+	}
+
+	if !isValidSkuShortName(license) {
+		// As a fallback for licenses whose SKU short name is unknown, make a best effort to try
+		// and use the presence of a known E10/Professional feature as a check to determine licensing.
+		if license != nil &&
+			license.Features != nil &&
+			license.Features.LDAP != nil &&
+			*license.Features.LDAP {
+			return true
+		}
+	}
+
+	return IsConfiguredForDevelopment(config)
+}
+
+// IsE20LicensedOrDevelopment returns true when the server is licensed with a legacy Mattermost
+// Enterprise E20 License or a Mattermost Enterprise License, or has `EnableDeveloper` and
+// `EnableTesting` configuration settings enabled, signaling a non-production, developer mode.
+func IsE20LicensedOrDevelopment(config *model.Config, license *model.License) bool {
+	if license != nil && (license.SkuShortName == e20 || license.SkuShortName == enterprise) {
+		return true
+	}
+
+	if !isValidSkuShortName(license) {
+		// As a fallback for licenses whose SKU short name is unknown, make a best effort to try
+		// and use the presence of a known E20/Enterprise feature as a check to determine licensing.
+		if license != nil &&
+			license.Features != nil &&
+			license.Features.FutureFeatures != nil &&
+			*license.Features.FutureFeatures {
+			return true
+		}
+	}
+
+	return IsConfiguredForDevelopment(config)
+}
+
+// IsConfiguredForDevelopment returns true when the server has `EnableDeveloper` and `EnableTesting`
+// configuration settings enabled, signaling a non-production, developer mode.
+func IsConfiguredForDevelopment(config *model.Config) bool {
+	if config != nil &&
+		config.ServiceSettings.EnableTesting != nil &&
+		*config.ServiceSettings.EnableTesting &&
+		config.ServiceSettings.EnableDeveloper != nil &&
+		*config.ServiceSettings.EnableDeveloper {
+		return true
+	}
+
+	return false
+}
+
+// IsCloud returns true when the server is on cloud, and false otherwise.
+func IsCloud(license *model.License) bool {
+	if license == nil || license.Features == nil || license.Features.Cloud == nil {
+		return false
+	}
+
+	return *license.Features.Cloud
+}