mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

MM-8607 Add ability to turn off non-critical services when under load (#13212)

Просмотр исходного кода 
* MM-8607: add ability to turn off non-critical services under load

* server busy invalid param unit tests

* MM-8607: rename server busy endpoints

* MM-8607: handle case where App not initialized

* MM-8607: additional unit test cases per feedback.

* MM-8607: use decorator to check isbusy when adding endpoint route

* MM-8607: rename endpoints, use struct for json

* Update api4/system.go

Fix misspelled log output

Co-Authored-By: Saturnino Abril <saturnino.abril@gmail.com>

* MM-8607: fix i18n order; max seconds for server busy expiry
Этот коммит содержится в:
Doug Lauder
2019-11-27 20:41:09 -05:00
коммит произвёл Joram Wilander
родитель 3cb3d874b8
Коммит 5abbe50258
21 изменённых файлов: 445 добавлений и 9 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

5
api4/apitestlib.go
Просмотреть файл

@@ -636,6 +636,11 @@ func CheckInternalErrorStatus(t *testing.T, resp *model.Response) {
checkHTTPStatus(t, resp, http.StatusInternalServerError, true)
}
func CheckServiceUnavailableStatus(t *testing.T, resp *model.Response) {
t.Helper()
checkHTTPStatus(t, resp, http.StatusServiceUnavailable, true)
}
func CheckErrorMessage(t *testing.T, resp *model.Response, errorId string) {
t.Helper()

8
api4/channel.go
Просмотреть файл

@@ -17,8 +17,8 @@ func (api *API) InitChannel() {
api.BaseRoutes.Channels.Handle("", api.ApiSessionRequired(getAllChannels)).Methods("GET")
api.BaseRoutes.Channels.Handle("", api.ApiSessionRequired(createChannel)).Methods("POST")
api.BaseRoutes.Channels.Handle("/direct", api.ApiSessionRequired(createDirectChannel)).Methods("POST")
api.BaseRoutes.Channels.Handle("/search", api.ApiSessionRequired(searchAllChannels)).Methods("POST")
api.BaseRoutes.Channels.Handle("/group/search", api.ApiSessionRequired(searchGroupChannels)).Methods("POST")
api.BaseRoutes.Channels.Handle("/search", api.ApiSessionRequiredDisableWhenBusy(searchAllChannels)).Methods("POST")
api.BaseRoutes.Channels.Handle("/group/search", api.ApiSessionRequiredDisableWhenBusy(searchGroupChannels)).Methods("POST")
api.BaseRoutes.Channels.Handle("/group", api.ApiSessionRequired(createGroupChannel)).Methods("POST")
api.BaseRoutes.Channels.Handle("/members/{user_id:[A-Za-z0-9]+}/view", api.ApiSessionRequired(viewChannel)).Methods("POST")
api.BaseRoutes.Channels.Handle("/{channel_id:[A-Za-z0-9]+}/scheme", api.ApiSessionRequired(updateChannelScheme)).Methods("PUT")
@@ -26,8 +26,8 @@ func (api *API) InitChannel() {
api.BaseRoutes.ChannelsForTeam.Handle("", api.ApiSessionRequired(getPublicChannelsForTeam)).Methods("GET")
api.BaseRoutes.ChannelsForTeam.Handle("/deleted", api.ApiSessionRequired(getDeletedChannelsForTeam)).Methods("GET")
api.BaseRoutes.ChannelsForTeam.Handle("/ids", api.ApiSessionRequired(getPublicChannelsByIdsForTeam)).Methods("POST")
api.BaseRoutes.ChannelsForTeam.Handle("/search", api.ApiSessionRequired(searchChannelsForTeam)).Methods("POST")
api.BaseRoutes.ChannelsForTeam.Handle("/search_archived", api.ApiSessionRequired(searchArchivedChannelsForTeam)).Methods("POST")
api.BaseRoutes.ChannelsForTeam.Handle("/search", api.ApiSessionRequiredDisableWhenBusy(searchChannelsForTeam)).Methods("POST")
api.BaseRoutes.ChannelsForTeam.Handle("/search_archived", api.ApiSessionRequiredDisableWhenBusy(searchArchivedChannelsForTeam)).Methods("POST")
api.BaseRoutes.ChannelsForTeam.Handle("/autocomplete", api.ApiSessionRequired(autocompleteChannelsForTeam)).Methods("GET")
api.BaseRoutes.ChannelsForTeam.Handle("/search_autocomplete", api.ApiSessionRequired(autocompleteChannelsForTeamForSearch)).Methods("GET")
api.BaseRoutes.User.Handle("/teams/{team_id:[A-Za-z0-9]+}/channels", api.ApiSessionRequired(getChannelsForTeamForUser)).Methods("GET")

20
api4/handlers.go
Просмотреть файл

@@ -107,3 +107,23 @@ func (api *API) ApiSessionRequiredTrustRequester(h func(*Context, http.ResponseW
return handler
}
// DisableWhenBusy provides a handler for API endpoints which should be disabled when the server is under load,
// responding with HTTP 503 (Service Unavailable).
func (api *API) ApiSessionRequiredDisableWhenBusy(h func(*Context, http.ResponseWriter, *http.Request)) http.Handler {
handler := &web.Handler{
GetGlobalAppOptions: api.GetGlobalAppOptions,
HandleFunc: h,
HandlerName: web.GetHandlerName(h),
RequireSession: true,
TrustRequester: false,
RequireMfa: false,
IsStatic: false,
DisableWhenBusy: true,
}
if *api.ConfigService.Config().ServiceSettings.WebserverMode == "gzip" {
return gziphandler.GzipHandler(handler)
}
return handler
}

2
api4/post.go
Просмотреть файл

@@ -25,7 +25,7 @@ func (api *API) InitPost() {
api.BaseRoutes.ChannelForUser.Handle("/posts/unread", api.ApiSessionRequired(getPostsForChannelAroundLastUnread)).Methods("GET")
api.BaseRoutes.Team.Handle("/posts/search", api.ApiSessionRequired(searchPosts)).Methods("POST")
api.BaseRoutes.Team.Handle("/posts/search", api.ApiSessionRequiredDisableWhenBusy(searchPosts)).Methods("POST")
api.BaseRoutes.Post.Handle("", api.ApiSessionRequired(updatePost)).Methods("PUT")
api.BaseRoutes.Post.Handle("/patch", api.ApiSessionRequired(patchPost)).Methods("PUT")
api.BaseRoutes.PostForUser.Handle("/set_unread", api.ApiSessionRequired(setPostUnread)).Methods("POST")

59
api4/system.go
Просмотреть файл

@@ -8,6 +8,7 @@ import (
"fmt"
"net/http"
"runtime"
"strconv"
"time"
"github.com/mattermost/mattermost-server/mlog"
@@ -16,7 +17,11 @@ import (
"github.com/mattermost/mattermost-server/utils"
)
const REDIRECT_LOCATION_CACHE_SIZE = 10000
const (
REDIRECT_LOCATION_CACHE_SIZE = 10000
DEFAULT_SERVER_BUSY_SECONDS = 3600
MAX_SERVER_BUSY_SECONDS = 86400
)
var redirectLocationDataCache = utils.NewLru(REDIRECT_LOCATION_CACHE_SIZE)
@@ -40,6 +45,10 @@ func (api *API) InitSystem() {
api.BaseRoutes.ApiRoot.Handle("/redirect_location", api.ApiSessionRequiredTrustRequester(getRedirectLocation)).Methods("GET")
api.BaseRoutes.ApiRoot.Handle("/notifications/ack", api.ApiSessionRequired(pushNotificationAck)).Methods("POST")
api.BaseRoutes.ApiRoot.Handle("/server_busy", api.ApiSessionRequired(setServerBusy)).Methods("POST")
api.BaseRoutes.ApiRoot.Handle("/server_busy", api.ApiSessionRequired(getServerBusyExpires)).Methods("GET")
api.BaseRoutes.ApiRoot.Handle("/server_busy/clear", api.ApiSessionRequired(clearServerBusy)).Methods("POST")
}
func getSystemPing(c *Context, w http.ResponseWriter, r *http.Request) {
@@ -444,3 +453,51 @@ func pushNotificationAck(c *Context, w http.ResponseWriter, r *http.Request) {
ReturnStatusOK(w)
}
func setServerBusy(c *Context, w http.ResponseWriter, r *http.Request) {
if !c.App.SessionHasPermissionTo(c.App.Session, model.PERMISSION_MANAGE_SYSTEM) {
c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM)
return
}
// number of seconds to keep server marked busy
secs := r.URL.Query().Get("seconds")
if secs == "" {
secs = strconv.FormatInt(DEFAULT_SERVER_BUSY_SECONDS, 10)
}
i, err := strconv.ParseInt(secs, 10, 64)
if err != nil || i <= 0 || i > MAX_SERVER_BUSY_SECONDS {
c.SetInvalidUrlParam(fmt.Sprintf("seconds must be 1 - %d", MAX_SERVER_BUSY_SECONDS))
return
}
c.App.Srv.Busy.Set(time.Second * time.Duration(i))
mlog.Warn("server busy state activated - non-critical services disabled", mlog.Int64("seconds", i))
ReturnStatusOK(w)
}
func clearServerBusy(c *Context, w http.ResponseWriter, r *http.Request) {
if !c.App.SessionHasPermissionTo(c.App.Session, model.PERMISSION_MANAGE_SYSTEM) {
c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM)
return
}
c.App.Srv.Busy.Clear()
mlog.Info("server busy state cleared - non-critical services enabled")
ReturnStatusOK(w)
}
func getServerBusyExpires(c *Context, w http.ResponseWriter, r *http.Request) {
if !c.App.SessionHasPermissionTo(c.App.Session, model.PERMISSION_MANAGE_SYSTEM) {
c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM)
return
}
busy := c.App.Srv.Busy
sbs := &model.ServerBusyState{
Busy: busy.IsBusy(),
Expires: busy.Expires().Unix(),
Expires_ts: busy.Expires().UTC().Format("Mon Jan 2 15:04:05 -0700 MST 2006"),
}
w.Write([]byte(sbs.ToJson()))
}

115
api4/system_test.go
Просмотреть файл

@@ -7,6 +7,7 @@ import (
"os"
"strings"
"testing"
"time"
"github.com/mattermost/mattermost-server/mlog"
"github.com/mattermost/mattermost-server/model"
@@ -507,3 +508,117 @@ func TestRedirectLocation(t *testing.T) {
_, resp = Client.GetRedirectLocation("", "")
CheckUnauthorizedStatus(t, resp)
}
func TestSetServerBusy(t *testing.T) {
th := Setup().InitBasic()
defer th.TearDown()
const secs = 30
t.Run("as system user", func(t *testing.T) {
ok, resp := th.Client.SetServerBusy(secs)
CheckForbiddenStatus(t, resp)
require.False(t, ok, "should not set server busy due to no permission")
require.False(t, th.App.Srv.Busy.IsBusy(), "server should not be marked busy")
})
t.Run("as system admin", func(t *testing.T) {
ok, resp := th.SystemAdminClient.SetServerBusy(secs)
CheckNoError(t, resp)
require.True(t, ok, "should set server busy successfully")
require.True(t, th.App.Srv.Busy.IsBusy(), "server should be marked busy")
})
}
func TestSetServerBusyInvalidParam(t *testing.T) {
th := Setup().InitBasic()
defer th.TearDown()
t.Run("as system admin, invalid param", func(t *testing.T) {
params := []int{-1, 0, MAX_SERVER_BUSY_SECONDS + 1}
for _, p := range params {
ok, resp := th.SystemAdminClient.SetServerBusy(p)
CheckBadRequestStatus(t, resp)
require.False(t, ok, "should not set server busy due to invalid param ", p)
require.False(t, th.App.Srv.Busy.IsBusy(), "server should not be marked busy due to invalid param ", p)
}
})
}
func TestClearServerBusy(t *testing.T) {
th := Setup().InitBasic()
defer th.TearDown()
th.App.Srv.Busy.Set(time.Second * 30)
t.Run("as system user", func(t *testing.T) {
ok, resp := th.Client.ClearServerBusy()
CheckForbiddenStatus(t, resp)
require.False(t, ok, "should not clear server busy flag due to no permission.")
require.True(t, th.App.Srv.Busy.IsBusy(), "server should be marked busy")
})
th.App.Srv.Busy.Set(time.Second * 30)
t.Run("as system admin", func(t *testing.T) {
ok, resp := th.SystemAdminClient.ClearServerBusy()
CheckNoError(t, resp)
require.True(t, ok, "should clear server busy flag successfully")
require.False(t, th.App.Srv.Busy.IsBusy(), "server should not be marked busy")
})
}
func TestGetServerBusyExpires(t *testing.T) {
th := Setup().InitBasic()
defer th.TearDown()
th.App.Srv.Busy.Set(time.Second * 30)
t.Run("as system user", func(t *testing.T) {
_, resp := th.Client.GetServerBusyExpires()
CheckForbiddenStatus(t, resp)
})
t.Run("as system admin", func(t *testing.T) {
expires, resp := th.SystemAdminClient.GetServerBusyExpires()
CheckNoError(t, resp)
require.Greater(t, expires.Unix(), time.Now().Unix())
})
}
func TestServerBusy503(t *testing.T) {
th := Setup().InitBasic()
defer th.TearDown()
th.App.Srv.Busy.Set(time.Second * 30)
t.Run("search users while busy", func(t *testing.T) {
us := &model.UserSearch{Term: "test"}
_, resp := th.SystemAdminClient.SearchUsers(us)
CheckServiceUnavailableStatus(t, resp)
})
t.Run("search teams while busy", func(t *testing.T) {
ts := &model.TeamSearch{}
_, resp := th.SystemAdminClient.SearchTeams(ts)
CheckServiceUnavailableStatus(t, resp)
})
t.Run("search channels while busy", func(t *testing.T) {
cs := &model.ChannelSearch{}
_, resp := th.SystemAdminClient.SearchChannels("foo", cs)
CheckServiceUnavailableStatus(t, resp)
})
t.Run("search archived channels while busy", func(t *testing.T) {
cs := &model.ChannelSearch{}
_, resp := th.SystemAdminClient.SearchArchivedChannels("foo", cs)
CheckServiceUnavailableStatus(t, resp)
})
th.App.Srv.Busy.Clear()
t.Run("search users while not busy", func(t *testing.T) {
us := &model.UserSearch{Term: "test"}
_, resp := th.SystemAdminClient.SearchUsers(us)
CheckNoError(t, resp)
})
}

2
api4/team.go
Просмотреть файл

@@ -34,7 +34,7 @@ func (api *API) InitTeam() {
api.BaseRoutes.Teams.Handle("", api.ApiSessionRequired(createTeam)).Methods("POST")
api.BaseRoutes.Teams.Handle("", api.ApiSessionRequired(getAllTeams)).Methods("GET")
api.BaseRoutes.Teams.Handle("/{team_id:[A-Za-z0-9]+}/scheme", api.ApiSessionRequired(updateTeamScheme)).Methods("PUT")
api.BaseRoutes.Teams.Handle("/search", api.ApiSessionRequired(searchTeams)).Methods("POST")
api.BaseRoutes.Teams.Handle("/search", api.ApiSessionRequiredDisableWhenBusy(searchTeams)).Methods("POST")
api.BaseRoutes.TeamsForUser.Handle("", api.ApiSessionRequired(getTeamsForUser)).Methods("GET")
api.BaseRoutes.TeamsForUser.Handle("/unread", api.ApiSessionRequired(getTeamsUnreadForUser)).Methods("GET")

2
api4/user.go
Просмотреть файл

@@ -24,7 +24,7 @@ func (api *API) InitUser() {
api.BaseRoutes.Users.Handle("", api.ApiSessionRequired(getUsers)).Methods("GET")
api.BaseRoutes.Users.Handle("/ids", api.ApiSessionRequired(getUsersByIds)).Methods("POST")
api.BaseRoutes.Users.Handle("/usernames", api.ApiSessionRequired(getUsersByNames)).Methods("POST")
api.BaseRoutes.Users.Handle("/search", api.ApiSessionRequired(searchUsers)).Methods("POST")
api.BaseRoutes.Users.Handle("/search", api.ApiSessionRequiredDisableWhenBusy(searchUsers)).Methods("POST")
api.BaseRoutes.Users.Handle("/autocomplete", api.ApiSessionRequired(autocompleteUsers)).Methods("GET")
api.BaseRoutes.Users.Handle("/stats", api.ApiSessionRequired(getTotalUsersStats)).Methods("GET")
api.BaseRoutes.Users.Handle("/group_channels", api.ApiSessionRequired(getUsersByGroupChannelIds)).Methods("POST")