MM-8607: Add ability to turn off non-critical services when under load; cluster support (#13267)

* MM-8607: Add cluster support for server busy status

MM-8607: Busy.Cluster should be non-exported

MM-8607: rename logging field seconds -> expires_sec

MM-8607: each node clears its own busy flag

MM-8607: ensure unit tests are not sensitive to test machine speed

* MM-8607: chg comment to force CI rebuild

app/busy.go

@@ -7,14 +7,31 @@ import (
 	"sync"
 	"sync/atomic"
 	"time"
+
+	"github.com/mattermost/mattermost-server/v5/einterfaces"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
-type Busy struct {
-	busy int32 // protected via atomic for fast IsBusy calls
+const (
+	TIMESTAMP_FORMAT = "Mon Jan 2 15:04:05 -0700 MST 2006"
+)
 
+// Busy represents the busy state of the server. A server marked busy
+// will have non-critical services disabled. If a Cluster is provided
+// any changes will be propagated to each node.
+type Busy struct {
+	busy    int32 // protected via atomic for fast IsBusy calls
 	mux     sync.RWMutex
 	timer   *time.Timer
 	expires time.Time
+
+	cluster einterfaces.ClusterInterface
+}
+
+// NewBusy creates a new Busy instance with optional cluster which will
+// be notified of busy state changes.
+func NewBusy(cluster einterfaces.ClusterInterface) *Busy {
+	return &Busy{cluster: cluster}
 }
 
 // IsBusy returns true if the server has been marked as busy.
@@ -25,27 +42,47 @@ func (b *Busy) IsBusy() bool {
 	return atomic.LoadInt32(&b.busy) != 0
 }
 
-// Set marks the server as busy for dur duration.
+// Set marks the server as busy for dur duration and notifies cluster nodes.
 func (b *Busy) Set(dur time.Duration) {
 	b.mux.Lock()
 	defer b.mux.Unlock()
 
-	b.clear()
-	atomic.StoreInt32(&b.busy, 1)
+	// minimum 1 second
+	if dur < (time.Second * 1) {
+		dur = time.Second * 1
+	}
 
-	b.timer = time.AfterFunc(dur, b.Clear)
-	b.expires = time.Now().Add(dur)
-}
+	b.setWithoutNotify(dur)
 
-// ClearBusy marks the server as not busy.
-func (b *Busy) Clear() {
-	b.mux.Lock()
-	defer b.mux.Unlock()
-	b.clear()
+	if b.cluster != nil {
+		sbs := &model.ServerBusyState{Busy: true, Expires: b.expires.Unix(), Expires_ts: b.expires.UTC().Format(TIMESTAMP_FORMAT)}
+		b.notifyServerBusyChange(sbs)
+	}
 }
 
 // must hold mutex
-func (b *Busy) clear() {
+func (b *Busy) setWithoutNotify(dur time.Duration) {
+	b.clearWithoutNotify()
+	atomic.StoreInt32(&b.busy, 1)
+	b.expires = time.Now().Add(dur)
+	b.timer = time.AfterFunc(dur, b.clearWithoutNotify)
+}
+
+// ClearBusy marks the server as not busy and notifies cluster nodes.
+func (b *Busy) Clear() {
+	b.mux.Lock()
+	defer b.mux.Unlock()
+
+	b.clearWithoutNotify()
+
+	if b.cluster != nil {
+		sbs := &model.ServerBusyState{Busy: false, Expires: time.Time{}.Unix(), Expires_ts: ""}
+		b.notifyServerBusyChange(sbs)
+	}
+}
+
+// must hold mutex
+func (b *Busy) clearWithoutNotify() {
 	if b.timer != nil {
 		b.timer.Stop() // don't drain timer.C channel for AfterFunc timers.
 	}
@@ -62,3 +99,45 @@ func (b *Busy) Expires() time.Time {
 	defer b.mux.RUnlock()
 	return b.expires
 }
+
+// notifyServerBusyChange informs all cluster members of a server busy state change.
+func (b *Busy) notifyServerBusyChange(sbs *model.ServerBusyState) {
+	if b.cluster == nil {
+		return
+	}
+	msg := &model.ClusterMessage{
+		Event:            model.CLUSTER_EVENT_BUSY_STATE_CHANGED,
+		SendType:         model.CLUSTER_SEND_RELIABLE,
+		WaitForAllToSend: true,
+		Data:             sbs.ToJson(),
+	}
+	b.cluster.SendClusterMessage(msg)
+}
+
+// ClusterEventChanged is called when a CLUSTER_EVENT_BUSY_STATE_CHANGED is received.
+func (b *Busy) ClusterEventChanged(sbs *model.ServerBusyState) {
+	b.mux.Lock()
+	defer b.mux.Unlock()
+
+	if sbs.Busy {
+		expires := time.Unix(sbs.Expires, 0)
+		dur := time.Until(expires)
+		if dur > 0 {
+			b.setWithoutNotify(dur)
+		}
+	} else {
+		b.clearWithoutNotify()
+	}
+}
+
+func (b *Busy) ToJson() string {
+	b.mux.RLock()
+	defer b.mux.RUnlock()
+
+	sbs := &model.ServerBusyState{
+		Busy:       atomic.LoadInt32(&b.busy) != 0,
+		Expires:    b.expires.Unix(),
+		Expires_ts: b.expires.UTC().Format(TIMESTAMP_FORMAT),
+	}
+	return sbs.ToJson()
+}