MM-8607 Add ability to turn off non-critical services when under load (#13212)

* MM-8607: add ability to turn off non-critical services under load

* server busy invalid param unit tests

* MM-8607: rename server busy endpoints

* MM-8607: handle case where App not initialized

* MM-8607: additional unit test cases per feedback.

* MM-8607: use decorator to check isbusy when adding endpoint route

* MM-8607: rename endpoints, use struct for json

* Update api4/system.go

Fix misspelled log output

Co-Authored-By: Saturnino Abril <saturnino.abril@gmail.com>

* MM-8607: fix i18n order; max seconds for server busy expiry

api4/system.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"net/http"
 	"runtime"
+	"strconv"
 	"time"
 
 	"github.com/mattermost/mattermost-server/mlog"
@@ -16,7 +17,11 @@ import (
 	"github.com/mattermost/mattermost-server/utils"
 )
 
-const REDIRECT_LOCATION_CACHE_SIZE = 10000
+const (
+	REDIRECT_LOCATION_CACHE_SIZE = 10000
+	DEFAULT_SERVER_BUSY_SECONDS  = 3600
+	MAX_SERVER_BUSY_SECONDS      = 86400
+)
 
 var redirectLocationDataCache = utils.NewLru(REDIRECT_LOCATION_CACHE_SIZE)
 
@@ -40,6 +45,10 @@ func (api *API) InitSystem() {
 	api.BaseRoutes.ApiRoot.Handle("/redirect_location", api.ApiSessionRequiredTrustRequester(getRedirectLocation)).Methods("GET")
 
 	api.BaseRoutes.ApiRoot.Handle("/notifications/ack", api.ApiSessionRequired(pushNotificationAck)).Methods("POST")
+
+	api.BaseRoutes.ApiRoot.Handle("/server_busy", api.ApiSessionRequired(setServerBusy)).Methods("POST")
+	api.BaseRoutes.ApiRoot.Handle("/server_busy", api.ApiSessionRequired(getServerBusyExpires)).Methods("GET")
+	api.BaseRoutes.ApiRoot.Handle("/server_busy/clear", api.ApiSessionRequired(clearServerBusy)).Methods("POST")
 }
 
 func getSystemPing(c *Context, w http.ResponseWriter, r *http.Request) {
@@ -444,3 +453,51 @@ func pushNotificationAck(c *Context, w http.ResponseWriter, r *http.Request) {
 
 	ReturnStatusOK(w)
 }
+
+func setServerBusy(c *Context, w http.ResponseWriter, r *http.Request) {
+	if !c.App.SessionHasPermissionTo(c.App.Session, model.PERMISSION_MANAGE_SYSTEM) {
+		c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM)
+		return
+	}
+
+	// number of seconds to keep server marked busy
+	secs := r.URL.Query().Get("seconds")
+	if secs == "" {
+		secs = strconv.FormatInt(DEFAULT_SERVER_BUSY_SECONDS, 10)
+	}
+
+	i, err := strconv.ParseInt(secs, 10, 64)
+	if err != nil || i <= 0 || i > MAX_SERVER_BUSY_SECONDS {
+		c.SetInvalidUrlParam(fmt.Sprintf("seconds must be 1 - %d", MAX_SERVER_BUSY_SECONDS))
+		return
+	}
+
+	c.App.Srv.Busy.Set(time.Second * time.Duration(i))
+	mlog.Warn("server busy state activated - non-critical services disabled", mlog.Int64("seconds", i))
+	ReturnStatusOK(w)
+}
+
+func clearServerBusy(c *Context, w http.ResponseWriter, r *http.Request) {
+	if !c.App.SessionHasPermissionTo(c.App.Session, model.PERMISSION_MANAGE_SYSTEM) {
+		c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM)
+		return
+	}
+	c.App.Srv.Busy.Clear()
+	mlog.Info("server busy state cleared - non-critical services enabled")
+	ReturnStatusOK(w)
+}
+
+func getServerBusyExpires(c *Context, w http.ResponseWriter, r *http.Request) {
+	if !c.App.SessionHasPermissionTo(c.App.Session, model.PERMISSION_MANAGE_SYSTEM) {
+		c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM)
+		return
+	}
+
+	busy := c.App.Srv.Busy
+	sbs := &model.ServerBusyState{
+		Busy:       busy.IsBusy(),
+		Expires:    busy.Expires().Unix(),
+		Expires_ts: busy.Expires().UTC().Format("Mon Jan 2 15:04:05 -0700 MST 2006"),
+	}
+	w.Write([]byte(sbs.ToJson()))
+}