MM-8607 Add ability to turn off non-critical services when under load (#13212)

* MM-8607: add ability to turn off non-critical services under load

* server busy invalid param unit tests

* MM-8607: rename server busy endpoints

* MM-8607: handle case where App not initialized

* MM-8607: additional unit test cases per feedback.

* MM-8607: use decorator to check isbusy when adding endpoint route

* MM-8607: rename endpoints, use struct for json

* Update api4/system.go

Fix misspelled log output

Co-Authored-By: Saturnino Abril <saturnino.abril@gmail.com>

* MM-8607: fix i18n order; max seconds for server busy expiry

app/busy.go

@@ -0,0 +1,64 @@
+// Copyright (c) 2019-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package app
+
+import (
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+type Busy struct {
+	busy int32 // protected via atomic for fast IsBusy calls
+
+	mux     sync.RWMutex
+	timer   *time.Timer
+	expires time.Time
+}
+
+// IsBusy returns true if the server has been marked as busy.
+func (b *Busy) IsBusy() bool {
+	if b == nil {
+		return false
+	}
+	return atomic.LoadInt32(&b.busy) != 0
+}
+
+// Set marks the server as busy for dur duration.
+func (b *Busy) Set(dur time.Duration) {
+	b.mux.Lock()
+	defer b.mux.Unlock()
+
+	b.clear()
+	atomic.StoreInt32(&b.busy, 1)
+
+	b.timer = time.AfterFunc(dur, b.Clear)
+	b.expires = time.Now().Add(dur)
+}
+
+// ClearBusy marks the server as not busy.
+func (b *Busy) Clear() {
+	b.mux.Lock()
+	defer b.mux.Unlock()
+	b.clear()
+}
+
+// must hold mutex
+func (b *Busy) clear() {
+	if b.timer != nil {
+		b.timer.Stop() // don't drain timer.C channel for AfterFunc timers.
+	}
+	b.timer = nil
+	b.expires = time.Time{}
+	atomic.StoreInt32(&b.busy, 0)
+}
+
+// Expires returns the expected time that the server
+// will be marked not busy. This expiry can be extended
+// via additional calls to SetBusy.
+func (b *Busy) Expires() time.Time {
+	b.mux.RLock()
+	defer b.mux.RUnlock()
+	return b.expires
+}