diff --git a/api4/apitestlib.go b/api4/apitestlib.go index 6da7efe6f4..c2e6cdae98 100644 --- a/api4/apitestlib.go +++ b/api4/apitestlib.go @@ -636,6 +636,11 @@ func CheckInternalErrorStatus(t *testing.T, resp *model.Response) { checkHTTPStatus(t, resp, http.StatusInternalServerError, true) } +func CheckServiceUnavailableStatus(t *testing.T, resp *model.Response) { + t.Helper() + checkHTTPStatus(t, resp, http.StatusServiceUnavailable, true) +} + func CheckErrorMessage(t *testing.T, resp *model.Response, errorId string) { t.Helper() diff --git a/api4/channel.go b/api4/channel.go index e953d1b518..5396a83646 100644 --- a/api4/channel.go +++ b/api4/channel.go @@ -17,8 +17,8 @@ func (api *API) InitChannel() { api.BaseRoutes.Channels.Handle("", api.ApiSessionRequired(getAllChannels)).Methods("GET") api.BaseRoutes.Channels.Handle("", api.ApiSessionRequired(createChannel)).Methods("POST") api.BaseRoutes.Channels.Handle("/direct", api.ApiSessionRequired(createDirectChannel)).Methods("POST") - api.BaseRoutes.Channels.Handle("/search", api.ApiSessionRequired(searchAllChannels)).Methods("POST") - api.BaseRoutes.Channels.Handle("/group/search", api.ApiSessionRequired(searchGroupChannels)).Methods("POST") + api.BaseRoutes.Channels.Handle("/search", api.ApiSessionRequiredDisableWhenBusy(searchAllChannels)).Methods("POST") + api.BaseRoutes.Channels.Handle("/group/search", api.ApiSessionRequiredDisableWhenBusy(searchGroupChannels)).Methods("POST") api.BaseRoutes.Channels.Handle("/group", api.ApiSessionRequired(createGroupChannel)).Methods("POST") api.BaseRoutes.Channels.Handle("/members/{user_id:[A-Za-z0-9]+}/view", api.ApiSessionRequired(viewChannel)).Methods("POST") api.BaseRoutes.Channels.Handle("/{channel_id:[A-Za-z0-9]+}/scheme", api.ApiSessionRequired(updateChannelScheme)).Methods("PUT") @@ -26,8 +26,8 @@ func (api *API) InitChannel() { api.BaseRoutes.ChannelsForTeam.Handle("", api.ApiSessionRequired(getPublicChannelsForTeam)).Methods("GET") api.BaseRoutes.ChannelsForTeam.Handle("/deleted", api.ApiSessionRequired(getDeletedChannelsForTeam)).Methods("GET") api.BaseRoutes.ChannelsForTeam.Handle("/ids", api.ApiSessionRequired(getPublicChannelsByIdsForTeam)).Methods("POST") - api.BaseRoutes.ChannelsForTeam.Handle("/search", api.ApiSessionRequired(searchChannelsForTeam)).Methods("POST") - api.BaseRoutes.ChannelsForTeam.Handle("/search_archived", api.ApiSessionRequired(searchArchivedChannelsForTeam)).Methods("POST") + api.BaseRoutes.ChannelsForTeam.Handle("/search", api.ApiSessionRequiredDisableWhenBusy(searchChannelsForTeam)).Methods("POST") + api.BaseRoutes.ChannelsForTeam.Handle("/search_archived", api.ApiSessionRequiredDisableWhenBusy(searchArchivedChannelsForTeam)).Methods("POST") api.BaseRoutes.ChannelsForTeam.Handle("/autocomplete", api.ApiSessionRequired(autocompleteChannelsForTeam)).Methods("GET") api.BaseRoutes.ChannelsForTeam.Handle("/search_autocomplete", api.ApiSessionRequired(autocompleteChannelsForTeamForSearch)).Methods("GET") api.BaseRoutes.User.Handle("/teams/{team_id:[A-Za-z0-9]+}/channels", api.ApiSessionRequired(getChannelsForTeamForUser)).Methods("GET") diff --git a/api4/handlers.go b/api4/handlers.go index 41bf74f758..4bfdaa52ab 100644 --- a/api4/handlers.go +++ b/api4/handlers.go @@ -107,3 +107,23 @@ func (api *API) ApiSessionRequiredTrustRequester(h func(*Context, http.ResponseW return handler } + +// DisableWhenBusy provides a handler for API endpoints which should be disabled when the server is under load, +// responding with HTTP 503 (Service Unavailable). +func (api *API) ApiSessionRequiredDisableWhenBusy(h func(*Context, http.ResponseWriter, *http.Request)) http.Handler { + handler := &web.Handler{ + GetGlobalAppOptions: api.GetGlobalAppOptions, + HandleFunc: h, + HandlerName: web.GetHandlerName(h), + RequireSession: true, + TrustRequester: false, + RequireMfa: false, + IsStatic: false, + DisableWhenBusy: true, + } + if *api.ConfigService.Config().ServiceSettings.WebserverMode == "gzip" { + return gziphandler.GzipHandler(handler) + } + return handler + +} diff --git a/api4/post.go b/api4/post.go index 4ec2411469..e21ec66305 100644 --- a/api4/post.go +++ b/api4/post.go @@ -25,7 +25,7 @@ func (api *API) InitPost() { api.BaseRoutes.ChannelForUser.Handle("/posts/unread", api.ApiSessionRequired(getPostsForChannelAroundLastUnread)).Methods("GET") - api.BaseRoutes.Team.Handle("/posts/search", api.ApiSessionRequired(searchPosts)).Methods("POST") + api.BaseRoutes.Team.Handle("/posts/search", api.ApiSessionRequiredDisableWhenBusy(searchPosts)).Methods("POST") api.BaseRoutes.Post.Handle("", api.ApiSessionRequired(updatePost)).Methods("PUT") api.BaseRoutes.Post.Handle("/patch", api.ApiSessionRequired(patchPost)).Methods("PUT") api.BaseRoutes.PostForUser.Handle("/set_unread", api.ApiSessionRequired(setPostUnread)).Methods("POST") diff --git a/api4/system.go b/api4/system.go index 7b9462c111..c985efd96c 100644 --- a/api4/system.go +++ b/api4/system.go @@ -8,6 +8,7 @@ import ( "fmt" "net/http" "runtime" + "strconv" "time" "github.com/mattermost/mattermost-server/mlog" @@ -16,7 +17,11 @@ import ( "github.com/mattermost/mattermost-server/utils" ) -const REDIRECT_LOCATION_CACHE_SIZE = 10000 +const ( + REDIRECT_LOCATION_CACHE_SIZE = 10000 + DEFAULT_SERVER_BUSY_SECONDS = 3600 + MAX_SERVER_BUSY_SECONDS = 86400 +) var redirectLocationDataCache = utils.NewLru(REDIRECT_LOCATION_CACHE_SIZE) @@ -40,6 +45,10 @@ func (api *API) InitSystem() { api.BaseRoutes.ApiRoot.Handle("/redirect_location", api.ApiSessionRequiredTrustRequester(getRedirectLocation)).Methods("GET") api.BaseRoutes.ApiRoot.Handle("/notifications/ack", api.ApiSessionRequired(pushNotificationAck)).Methods("POST") + + api.BaseRoutes.ApiRoot.Handle("/server_busy", api.ApiSessionRequired(setServerBusy)).Methods("POST") + api.BaseRoutes.ApiRoot.Handle("/server_busy", api.ApiSessionRequired(getServerBusyExpires)).Methods("GET") + api.BaseRoutes.ApiRoot.Handle("/server_busy/clear", api.ApiSessionRequired(clearServerBusy)).Methods("POST") } func getSystemPing(c *Context, w http.ResponseWriter, r *http.Request) { @@ -444,3 +453,51 @@ func pushNotificationAck(c *Context, w http.ResponseWriter, r *http.Request) { ReturnStatusOK(w) } + +func setServerBusy(c *Context, w http.ResponseWriter, r *http.Request) { + if !c.App.SessionHasPermissionTo(c.App.Session, model.PERMISSION_MANAGE_SYSTEM) { + c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM) + return + } + + // number of seconds to keep server marked busy + secs := r.URL.Query().Get("seconds") + if secs == "" { + secs = strconv.FormatInt(DEFAULT_SERVER_BUSY_SECONDS, 10) + } + + i, err := strconv.ParseInt(secs, 10, 64) + if err != nil || i <= 0 || i > MAX_SERVER_BUSY_SECONDS { + c.SetInvalidUrlParam(fmt.Sprintf("seconds must be 1 - %d", MAX_SERVER_BUSY_SECONDS)) + return + } + + c.App.Srv.Busy.Set(time.Second * time.Duration(i)) + mlog.Warn("server busy state activated - non-critical services disabled", mlog.Int64("seconds", i)) + ReturnStatusOK(w) +} + +func clearServerBusy(c *Context, w http.ResponseWriter, r *http.Request) { + if !c.App.SessionHasPermissionTo(c.App.Session, model.PERMISSION_MANAGE_SYSTEM) { + c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM) + return + } + c.App.Srv.Busy.Clear() + mlog.Info("server busy state cleared - non-critical services enabled") + ReturnStatusOK(w) +} + +func getServerBusyExpires(c *Context, w http.ResponseWriter, r *http.Request) { + if !c.App.SessionHasPermissionTo(c.App.Session, model.PERMISSION_MANAGE_SYSTEM) { + c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM) + return + } + + busy := c.App.Srv.Busy + sbs := &model.ServerBusyState{ + Busy: busy.IsBusy(), + Expires: busy.Expires().Unix(), + Expires_ts: busy.Expires().UTC().Format("Mon Jan 2 15:04:05 -0700 MST 2006"), + } + w.Write([]byte(sbs.ToJson())) +} diff --git a/api4/system_test.go b/api4/system_test.go index b3e893f34f..05da0a4266 100644 --- a/api4/system_test.go +++ b/api4/system_test.go @@ -7,6 +7,7 @@ import ( "os" "strings" "testing" + "time" "github.com/mattermost/mattermost-server/mlog" "github.com/mattermost/mattermost-server/model" @@ -507,3 +508,117 @@ func TestRedirectLocation(t *testing.T) { _, resp = Client.GetRedirectLocation("", "") CheckUnauthorizedStatus(t, resp) } + +func TestSetServerBusy(t *testing.T) { + th := Setup().InitBasic() + defer th.TearDown() + + const secs = 30 + + t.Run("as system user", func(t *testing.T) { + ok, resp := th.Client.SetServerBusy(secs) + CheckForbiddenStatus(t, resp) + require.False(t, ok, "should not set server busy due to no permission") + require.False(t, th.App.Srv.Busy.IsBusy(), "server should not be marked busy") + }) + + t.Run("as system admin", func(t *testing.T) { + ok, resp := th.SystemAdminClient.SetServerBusy(secs) + CheckNoError(t, resp) + require.True(t, ok, "should set server busy successfully") + require.True(t, th.App.Srv.Busy.IsBusy(), "server should be marked busy") + }) +} + +func TestSetServerBusyInvalidParam(t *testing.T) { + th := Setup().InitBasic() + defer th.TearDown() + + t.Run("as system admin, invalid param", func(t *testing.T) { + params := []int{-1, 0, MAX_SERVER_BUSY_SECONDS + 1} + for _, p := range params { + ok, resp := th.SystemAdminClient.SetServerBusy(p) + CheckBadRequestStatus(t, resp) + require.False(t, ok, "should not set server busy due to invalid param ", p) + require.False(t, th.App.Srv.Busy.IsBusy(), "server should not be marked busy due to invalid param ", p) + } + }) +} + +func TestClearServerBusy(t *testing.T) { + th := Setup().InitBasic() + defer th.TearDown() + + th.App.Srv.Busy.Set(time.Second * 30) + t.Run("as system user", func(t *testing.T) { + ok, resp := th.Client.ClearServerBusy() + CheckForbiddenStatus(t, resp) + require.False(t, ok, "should not clear server busy flag due to no permission.") + require.True(t, th.App.Srv.Busy.IsBusy(), "server should be marked busy") + }) + + th.App.Srv.Busy.Set(time.Second * 30) + t.Run("as system admin", func(t *testing.T) { + ok, resp := th.SystemAdminClient.ClearServerBusy() + CheckNoError(t, resp) + require.True(t, ok, "should clear server busy flag successfully") + require.False(t, th.App.Srv.Busy.IsBusy(), "server should not be marked busy") + }) +} + +func TestGetServerBusyExpires(t *testing.T) { + th := Setup().InitBasic() + defer th.TearDown() + + th.App.Srv.Busy.Set(time.Second * 30) + + t.Run("as system user", func(t *testing.T) { + _, resp := th.Client.GetServerBusyExpires() + CheckForbiddenStatus(t, resp) + }) + + t.Run("as system admin", func(t *testing.T) { + expires, resp := th.SystemAdminClient.GetServerBusyExpires() + CheckNoError(t, resp) + require.Greater(t, expires.Unix(), time.Now().Unix()) + }) +} + +func TestServerBusy503(t *testing.T) { + th := Setup().InitBasic() + defer th.TearDown() + + th.App.Srv.Busy.Set(time.Second * 30) + + t.Run("search users while busy", func(t *testing.T) { + us := &model.UserSearch{Term: "test"} + _, resp := th.SystemAdminClient.SearchUsers(us) + CheckServiceUnavailableStatus(t, resp) + }) + + t.Run("search teams while busy", func(t *testing.T) { + ts := &model.TeamSearch{} + _, resp := th.SystemAdminClient.SearchTeams(ts) + CheckServiceUnavailableStatus(t, resp) + }) + + t.Run("search channels while busy", func(t *testing.T) { + cs := &model.ChannelSearch{} + _, resp := th.SystemAdminClient.SearchChannels("foo", cs) + CheckServiceUnavailableStatus(t, resp) + }) + + t.Run("search archived channels while busy", func(t *testing.T) { + cs := &model.ChannelSearch{} + _, resp := th.SystemAdminClient.SearchArchivedChannels("foo", cs) + CheckServiceUnavailableStatus(t, resp) + }) + + th.App.Srv.Busy.Clear() + + t.Run("search users while not busy", func(t *testing.T) { + us := &model.UserSearch{Term: "test"} + _, resp := th.SystemAdminClient.SearchUsers(us) + CheckNoError(t, resp) + }) +} diff --git a/api4/team.go b/api4/team.go index 391376e332..aa2daaa98e 100644 --- a/api4/team.go +++ b/api4/team.go @@ -34,7 +34,7 @@ func (api *API) InitTeam() { api.BaseRoutes.Teams.Handle("", api.ApiSessionRequired(createTeam)).Methods("POST") api.BaseRoutes.Teams.Handle("", api.ApiSessionRequired(getAllTeams)).Methods("GET") api.BaseRoutes.Teams.Handle("/{team_id:[A-Za-z0-9]+}/scheme", api.ApiSessionRequired(updateTeamScheme)).Methods("PUT") - api.BaseRoutes.Teams.Handle("/search", api.ApiSessionRequired(searchTeams)).Methods("POST") + api.BaseRoutes.Teams.Handle("/search", api.ApiSessionRequiredDisableWhenBusy(searchTeams)).Methods("POST") api.BaseRoutes.TeamsForUser.Handle("", api.ApiSessionRequired(getTeamsForUser)).Methods("GET") api.BaseRoutes.TeamsForUser.Handle("/unread", api.ApiSessionRequired(getTeamsUnreadForUser)).Methods("GET") diff --git a/api4/user.go b/api4/user.go index d6a8d67b33..036c7a0811 100644 --- a/api4/user.go +++ b/api4/user.go @@ -24,7 +24,7 @@ func (api *API) InitUser() { api.BaseRoutes.Users.Handle("", api.ApiSessionRequired(getUsers)).Methods("GET") api.BaseRoutes.Users.Handle("/ids", api.ApiSessionRequired(getUsersByIds)).Methods("POST") api.BaseRoutes.Users.Handle("/usernames", api.ApiSessionRequired(getUsersByNames)).Methods("POST") - api.BaseRoutes.Users.Handle("/search", api.ApiSessionRequired(searchUsers)).Methods("POST") + api.BaseRoutes.Users.Handle("/search", api.ApiSessionRequiredDisableWhenBusy(searchUsers)).Methods("POST") api.BaseRoutes.Users.Handle("/autocomplete", api.ApiSessionRequired(autocompleteUsers)).Methods("GET") api.BaseRoutes.Users.Handle("/stats", api.ApiSessionRequired(getTotalUsersStats)).Methods("GET") api.BaseRoutes.Users.Handle("/group_channels", api.ApiSessionRequired(getUsersByGroupChannelIds)).Methods("POST") diff --git a/app/busy.go b/app/busy.go new file mode 100644 index 0000000000..a52d973beb --- /dev/null +++ b/app/busy.go @@ -0,0 +1,64 @@ +// Copyright (c) 2019-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "sync" + "sync/atomic" + "time" +) + +type Busy struct { + busy int32 // protected via atomic for fast IsBusy calls + + mux sync.RWMutex + timer *time.Timer + expires time.Time +} + +// IsBusy returns true if the server has been marked as busy. +func (b *Busy) IsBusy() bool { + if b == nil { + return false + } + return atomic.LoadInt32(&b.busy) != 0 +} + +// Set marks the server as busy for dur duration. +func (b *Busy) Set(dur time.Duration) { + b.mux.Lock() + defer b.mux.Unlock() + + b.clear() + atomic.StoreInt32(&b.busy, 1) + + b.timer = time.AfterFunc(dur, b.Clear) + b.expires = time.Now().Add(dur) +} + +// ClearBusy marks the server as not busy. +func (b *Busy) Clear() { + b.mux.Lock() + defer b.mux.Unlock() + b.clear() +} + +// must hold mutex +func (b *Busy) clear() { + if b.timer != nil { + b.timer.Stop() // don't drain timer.C channel for AfterFunc timers. + } + b.timer = nil + b.expires = time.Time{} + atomic.StoreInt32(&b.busy, 0) +} + +// Expires returns the expected time that the server +// will be marked not busy. This expiry can be extended +// via additional calls to SetBusy. +func (b *Busy) Expires() time.Time { + b.mux.RLock() + defer b.mux.RUnlock() + return b.expires +} diff --git a/app/busy_test.go b/app/busy_test.go new file mode 100644 index 0000000000..087abbf0e5 --- /dev/null +++ b/app/busy_test.go @@ -0,0 +1,72 @@ +// Copyright (c) 2019-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestBusySet(t *testing.T) { + busy := &Busy{} + + isNotBusy := func() bool { + return !busy.IsBusy() + } + + require.False(t, busy.IsBusy()) + + busy.Set(time.Millisecond * 100) + require.True(t, busy.IsBusy()) + // should automatically expire after 100ms + require.Eventually(t, isNotBusy, time.Second*5, time.Millisecond*20) + + // test set after auto expiry + busy.Set(time.Second * 30) + require.True(t, busy.IsBusy()) + expire := busy.Expires() + require.Greater(t, expire.Unix(), time.Now().Add(time.Second*10).Unix()) + + // test extending existing expiry + busy.Set(time.Minute * 5) + require.True(t, busy.IsBusy()) + expire = busy.Expires() + require.Greater(t, expire.Unix(), time.Now().Add(time.Minute*2).Unix()) + + busy.Clear() + require.False(t, busy.IsBusy()) +} + +func TestBusyExpires(t *testing.T) { + busy := &Busy{} + + isNotBusy := func() bool { + return !busy.IsBusy() + } + + // get expiry before it is set + expire := busy.Expires() + // should be time.Time zero value + require.Equal(t, time.Time{}.Unix(), expire.Unix()) + + // get expiry after it is set + busy.Set(time.Minute * 5) + expire = busy.Expires() + require.Greater(t, expire.Unix(), time.Now().Add(time.Minute*2).Unix()) + + // get expiry after clear + busy.Clear() + expire = busy.Expires() + // should be time.Time zero value + require.Equal(t, time.Time{}.Unix(), expire.Unix()) + + // get expiry after auto-expire + busy.Set(time.Millisecond * 100) + require.Eventually(t, isNotBusy, time.Second*5, time.Millisecond*20) + expire = busy.Expires() + // should be time.Time zero value + require.Equal(t, time.Time{}.Unix(), expire.Unix()) +} diff --git a/app/server.go b/app/server.go index 2b0fcdef69..1184a576ed 100644 --- a/app/server.go +++ b/app/server.go @@ -53,6 +53,7 @@ type Server struct { Server *http.Server ListenAddr *net.TCPAddr RateLimiter *RateLimiter + Busy *Busy didFinishListen chan struct{} @@ -473,6 +474,7 @@ func (s *Server) Start() error { s.RateLimiter = rateLimiter handler = rateLimiter.RateLimitHandler(handler) } + s.Busy = &Busy{} // Creating a logger for logging errors from http.Server at error level errStdLog, err := s.Log.StdLogAt(mlog.LevelError, mlog.String("source", "httpserver")) diff --git a/app/status.go b/app/status.go index 170d112b12..77df7ea690 100644 --- a/app/status.go +++ b/app/status.go @@ -228,6 +228,10 @@ func (a *App) SetStatusOnline(userId string, manual bool) { } func (a *App) BroadcastStatus(status *model.Status) { + if a.Srv.Busy.IsBusy() { + // this is considered a non-critical service and will be disabled when server busy. + return + } event := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_STATUS_CHANGE, "", "", status.UserId, nil) event.Add("status", status.Status) event.Add("user_id", status.UserId) diff --git a/i18n/en.json b/i18n/en.json index a978bbd6d9..b3c1580c0d 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -1090,6 +1090,10 @@ "id": "api.context.permissions.app_error", "translation": "You do not have the appropriate permissions" }, + { + "id": "api.context.server_busy.app_error", + "translation": "Server is busy, non-critical services are temporarily unavailable" + }, { "id": "api.context.session_expired.app_error", "translation": "Invalid or expired session, please login again." @@ -2806,6 +2810,10 @@ "id": "api.websocket_handler.invalid_param.app_error", "translation": "Invalid {{.Name}} parameter" }, + { + "id": "api.websocket_handler.server_busy.app_error", + "translation": "Server is busy, non-critical services are temporarily unavailable" + }, { "id": "app.admin.test_email.failure", "translation": "Connection unsuccessful: {{.Error}}" diff --git a/model/client4.go b/model/client4.go index b09584cde4..817621e787 100644 --- a/model/client4.go +++ b/model/client4.go @@ -420,6 +420,10 @@ func (c *Client4) GetRedirectLocationRoute() string { return fmt.Sprintf("/redirect_location") } +func (c *Client4) GetServerBusyRoute() string { + return "/server_busy" +} + func (c *Client4) GetUserTermsOfServiceRoute(userId string) string { return c.GetUserRoute(userId) + "/terms_of_service" } @@ -4622,6 +4626,41 @@ func (c *Client4) GetRedirectLocation(urlParam, etag string) (string, *Response) return MapFromJson(r.Body)["location"], BuildResponse(r) } +// SetServerBusy will mark the server as busy, which disables non-critical services for `secs` seconds. +func (c *Client4) SetServerBusy(secs int) (bool, *Response) { + url := fmt.Sprintf("%s?seconds=%d", c.GetServerBusyRoute(), secs) + r, err := c.DoApiPost(url, "") + if err != nil { + return false, BuildErrorResponse(r, err) + } + defer closeBody(r) + return CheckStatusOK(r), BuildResponse(r) +} + +// ClearServerBusy will mark the server as not busy. +func (c *Client4) ClearServerBusy() (bool, *Response) { + r, err := c.DoApiPost(c.GetServerBusyRoute()+"/clear", "") + if err != nil { + return false, BuildErrorResponse(r, err) + } + defer closeBody(r) + return CheckStatusOK(r), BuildResponse(r) +} + +// GetServerBusyExpires returns the time when a server marked busy +// will automatically have the flag cleared. +func (c *Client4) GetServerBusyExpires() (*time.Time, *Response) { + r, err := c.DoApiGet(c.GetServerBusyRoute(), "") + if err != nil { + return nil, BuildErrorResponse(r, err) + } + defer closeBody(r) + + sbs := ServerBusyStateFromJson(r.Body) + expires := time.Unix(sbs.Expires, 0) + return &expires, BuildResponse(r) +} + // RegisterTermsOfServiceAction saves action performed by a user against a specific terms of service. func (c *Client4) RegisterTermsOfServiceAction(userId, termsOfServiceId string, accepted bool) (*bool, *Response) { url := c.GetUserTermsOfServiceRoute(userId) diff --git a/model/system.go b/model/system.go index 334fc43302..3ac37adffe 100644 --- a/model/system.go +++ b/model/system.go @@ -50,3 +50,20 @@ type SystemECDSAKey struct { Y *big.Int `json:"y"` D *big.Int `json:"d,omitempty"` } + +type ServerBusyState struct { + Busy bool `json:"busy"` + Expires int64 `json:"expires"` + Expires_ts string `json:"expires_ts,omitempty"` +} + +func (sbs *ServerBusyState) ToJson() string { + b, _ := json.Marshal(sbs) + return string(b) +} + +func ServerBusyStateFromJson(r io.Reader) *ServerBusyState { + var sbs *ServerBusyState + json.NewDecoder(r).Decode(&sbs) + return sbs +} diff --git a/model/system_test.go b/model/system_test.go index 161c2280f6..29da1bfa3c 100644 --- a/model/system_test.go +++ b/model/system_test.go @@ -6,6 +6,7 @@ package model import ( "strings" "testing" + "time" "github.com/stretchr/testify/require" ) @@ -17,3 +18,13 @@ func TestSystemJson(t *testing.T) { require.Equal(t, "test", result.Name, "ids do not match") } + +func TestServerBusyJson(t *testing.T) { + now := time.Now() + sbs := ServerBusyState{Busy: true, Expires: now.Unix()} + json := sbs.ToJson() + result := ServerBusyStateFromJson(strings.NewReader(json)) + + require.Equal(t, sbs.Busy, result.Busy, "busy state does not match") + require.Equal(t, sbs.Expires, result.Expires, "expiry does not match") +} diff --git a/model/utils.go b/model/utils.go index ea7c7f0781..a883c8cfb3 100644 --- a/model/utils.go +++ b/model/utils.go @@ -207,7 +207,7 @@ func MapToJson(objmap map[string]string) string { return string(b) } -// MapToJson converts a map to a json string +// MapBoolToJson converts a map to a json string func MapBoolToJson(objmap map[string]bool) string { b, _ := json.Marshal(objmap) return string(b) diff --git a/web/context.go b/web/context.go index 9201026108..cb633177f0 100644 --- a/web/context.go +++ b/web/context.go @@ -165,6 +165,10 @@ func (c *Context) SetInvalidUrlParam(parameter string) { c.Err = NewInvalidUrlParamError(parameter) } +func (c *Context) SetServerBusyError() { + c.Err = NewServerBusyError() +} + func (c *Context) HandleEtag(etag string, routeName string, w http.ResponseWriter, r *http.Request) bool { metrics := c.App.Metrics if et := r.Header.Get(model.HEADER_ETAG_CLIENT); len(etag) > 0 { @@ -193,6 +197,10 @@ func NewInvalidUrlParamError(parameter string) *model.AppError { err := model.NewAppError("Context", "api.context.invalid_url_param.app_error", map[string]interface{}{"Name": parameter}, "", http.StatusBadRequest) return err } +func NewServerBusyError() *model.AppError { + err := model.NewAppError("Context", "api.context.server_busy.app_error", nil, "", http.StatusServiceUnavailable) + return err +} func (c *Context) SetPermissionError(permission *model.Permission) { c.Err = c.App.MakePermissionError(permission) diff --git a/web/handlers.go b/web/handlers.go index 3bdaff286e..32d3e14398 100644 --- a/web/handlers.go +++ b/web/handlers.go @@ -66,6 +66,7 @@ type Handler struct { TrustRequester bool RequireMfa bool IsStatic bool + DisableWhenBusy bool cspShaDirective string } @@ -159,6 +160,10 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { c.MfaRequired() } + if c.Err == nil && h.DisableWhenBusy && c.App.Srv.Busy.IsBusy() { + c.SetServerBusyError() + } + if c.Err == nil { h.HandleFunc(c, w, r) } diff --git a/wsapi/user.go b/wsapi/user.go index 8c64cdb480..1db14179df 100644 --- a/wsapi/user.go +++ b/wsapi/user.go @@ -13,6 +13,11 @@ func (api *API) InitUser() { } func (api *API) userTyping(req *model.WebSocketRequest) (map[string]interface{}, *model.AppError) { + if api.App.Srv.Busy.IsBusy() { + // this is considered a non-critical service and will be disabled when server busy. + return nil, NewServerBusyWebSocketError(req.Action) + } + var ok bool var channelId string if channelId, ok = req.Data["channel_id"].(string); !ok || len(channelId) != 26 { diff --git a/wsapi/websocket_handler.go b/wsapi/websocket_handler.go index dc6bd06d9b..b692d58e00 100644 --- a/wsapi/websocket_handler.go +++ b/wsapi/websocket_handler.go @@ -59,3 +59,7 @@ func (wh webSocketHandler) ServeWebSocket(conn *app.WebConn, r *model.WebSocketR func NewInvalidWebSocketParamError(action string, name string) *model.AppError { return model.NewAppError("websocket: "+action, "api.websocket_handler.invalid_param.app_error", map[string]interface{}{"Name": name}, "", http.StatusBadRequest) } + +func NewServerBusyWebSocketError(action string) *model.AppError { + return model.NewAppError("websocket: "+action, "api.websocket_handler.server_busy.app_error", nil, "", http.StatusServiceUnavailable) +}