MM-8607 Add ability to turn off non-critical services when under load (#13212)

* MM-8607: add ability to turn off non-critical services under load

* server busy invalid param unit tests

* MM-8607: rename server busy endpoints

* MM-8607: handle case where App not initialized

* MM-8607: additional unit test cases per feedback.

* MM-8607: use decorator to check isbusy when adding endpoint route

* MM-8607: rename endpoints, use struct for json

* Update api4/system.go

Fix misspelled log output

Co-Authored-By: Saturnino Abril <saturnino.abril@gmail.com>

* MM-8607: fix i18n order; max seconds for server busy expiry

app/busy.go

@@ -0,0 +1,64 @@
+// Copyright (c) 2019-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package app
+
+import (
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+type Busy struct {
+	busy int32 // protected via atomic for fast IsBusy calls
+
+	mux     sync.RWMutex
+	timer   *time.Timer
+	expires time.Time
+}
+
+// IsBusy returns true if the server has been marked as busy.
+func (b *Busy) IsBusy() bool {
+	if b == nil {
+		return false
+	}
+	return atomic.LoadInt32(&b.busy) != 0
+}
+
+// Set marks the server as busy for dur duration.
+func (b *Busy) Set(dur time.Duration) {
+	b.mux.Lock()
+	defer b.mux.Unlock()
+
+	b.clear()
+	atomic.StoreInt32(&b.busy, 1)
+
+	b.timer = time.AfterFunc(dur, b.Clear)
+	b.expires = time.Now().Add(dur)
+}
+
+// ClearBusy marks the server as not busy.
+func (b *Busy) Clear() {
+	b.mux.Lock()
+	defer b.mux.Unlock()
+	b.clear()
+}
+
+// must hold mutex
+func (b *Busy) clear() {
+	if b.timer != nil {
+		b.timer.Stop() // don't drain timer.C channel for AfterFunc timers.
+	}
+	b.timer = nil
+	b.expires = time.Time{}
+	atomic.StoreInt32(&b.busy, 0)
+}
+
+// Expires returns the expected time that the server
+// will be marked not busy. This expiry can be extended
+// via additional calls to SetBusy.
+func (b *Busy) Expires() time.Time {
+	b.mux.RLock()
+	defer b.mux.RUnlock()
+	return b.expires
+}

app/busy_test.go

@@ -0,0 +1,72 @@
+// Copyright (c) 2019-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package app
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestBusySet(t *testing.T) {
+	busy := &Busy{}
+
+	isNotBusy := func() bool {
+		return !busy.IsBusy()
+	}
+
+	require.False(t, busy.IsBusy())
+
+	busy.Set(time.Millisecond * 100)
+	require.True(t, busy.IsBusy())
+	// should automatically expire after 100ms
+	require.Eventually(t, isNotBusy, time.Second*5, time.Millisecond*20)
+
+	// test set after auto expiry
+	busy.Set(time.Second * 30)
+	require.True(t, busy.IsBusy())
+	expire := busy.Expires()
+	require.Greater(t, expire.Unix(), time.Now().Add(time.Second*10).Unix())
+
+	// test extending existing expiry
+	busy.Set(time.Minute * 5)
+	require.True(t, busy.IsBusy())
+	expire = busy.Expires()
+	require.Greater(t, expire.Unix(), time.Now().Add(time.Minute*2).Unix())
+
+	busy.Clear()
+	require.False(t, busy.IsBusy())
+}
+
+func TestBusyExpires(t *testing.T) {
+	busy := &Busy{}
+
+	isNotBusy := func() bool {
+		return !busy.IsBusy()
+	}
+
+	// get expiry before it is set
+	expire := busy.Expires()
+	// should be time.Time zero value
+	require.Equal(t, time.Time{}.Unix(), expire.Unix())
+
+	// get expiry after it is set
+	busy.Set(time.Minute * 5)
+	expire = busy.Expires()
+	require.Greater(t, expire.Unix(), time.Now().Add(time.Minute*2).Unix())
+
+	// get expiry after clear
+	busy.Clear()
+	expire = busy.Expires()
+	// should be time.Time zero value
+	require.Equal(t, time.Time{}.Unix(), expire.Unix())
+
+	// get expiry after auto-expire
+	busy.Set(time.Millisecond * 100)
+	require.Eventually(t, isNotBusy, time.Second*5, time.Millisecond*20)
+	expire = busy.Expires()
+	// should be time.Time zero value
+	require.Equal(t, time.Time{}.Unix(), expire.Unix())
+}

app/server.go

@@ -53,6 +53,7 @@ type Server struct {
 	Server      *http.Server
 	ListenAddr  *net.TCPAddr
 	RateLimiter *RateLimiter
+	Busy        *Busy
 
 	didFinishListen chan struct{}
 
@@ -473,6 +474,7 @@ func (s *Server) Start() error {
 		s.RateLimiter = rateLimiter
 		handler = rateLimiter.RateLimitHandler(handler)
 	}
+	s.Busy = &Busy{}
 
 	// Creating a logger for logging errors from http.Server at error level
 	errStdLog, err := s.Log.StdLogAt(mlog.LevelError, mlog.String("source", "httpserver"))

app/status.go

@@ -228,6 +228,10 @@ func (a *App) SetStatusOnline(userId string, manual bool) {
 }
 
 func (a *App) BroadcastStatus(status *model.Status) {
+	if a.Srv.Busy.IsBusy() {
+		// this is considered a non-critical service and will be disabled when server busy.
+		return
+	}
 	event := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_STATUS_CHANGE, "", "", status.UserId, nil)
 	event.Add("status", status.Status)
 	event.Add("user_id", status.UserId)