[MM-31132] app/server: add pprof endpoint (#17001)
* app/server: add pprof endpoint * reflect review comments * make metrics link conditional * app/server: add metrics server setup to licence listeners * refactor a bit * trigger CI Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
Этот коммит содержится в:
Ibrahim Serdar Acikgoz
коммит произвёл
GitHub
GitHub
родитель
5cb5f0a60a
Коммит
f31a9ed1a8
@@ -411,12 +411,13 @@ func (s *Server) SaveConfig(newCfg *model.Config, sendConfigChangeClusterMessage
|
|||||||
return model.NewAppError("saveConfig", "app.save_config.app_error", nil, err.Error(), http.StatusInternalServerError)
|
return model.NewAppError("saveConfig", "app.save_config.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||||
}
|
}
|
||||||
|
|
||||||
if s.Metrics != nil {
|
if s.startMetrics && *s.Config().MetricsSettings.Enable {
|
||||||
if *s.Config().MetricsSettings.Enable {
|
if s.Metrics != nil {
|
||||||
s.Metrics.StartServer()
|
s.Metrics.Register()
|
||||||
} else {
|
|
||||||
s.Metrics.StopServer()
|
|
||||||
}
|
}
|
||||||
|
s.SetupMetricsServer()
|
||||||
|
} else {
|
||||||
|
s.StopMetricsServer()
|
||||||
}
|
}
|
||||||
|
|
||||||
if s.Cluster != nil {
|
if s.Cluster != nil {
|
||||||
|
|||||||
137
app/server.go
137
app/server.go
@@ -9,9 +9,11 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"hash/maphash"
|
"hash/maphash"
|
||||||
|
"html/template"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/http/pprof"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
@@ -29,6 +31,7 @@ import (
|
|||||||
|
|
||||||
"github.com/getsentry/sentry-go"
|
"github.com/getsentry/sentry-go"
|
||||||
sentryhttp "github.com/getsentry/sentry-go/http"
|
sentryhttp "github.com/getsentry/sentry-go/http"
|
||||||
|
"github.com/gorilla/handlers"
|
||||||
"github.com/gorilla/mux"
|
"github.com/gorilla/mux"
|
||||||
"github.com/mailru/easygo/netpoll"
|
"github.com/mailru/easygo/netpoll"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
@@ -93,6 +96,10 @@ type Server struct {
|
|||||||
|
|
||||||
localModeServer *http.Server
|
localModeServer *http.Server
|
||||||
|
|
||||||
|
metricsServer *http.Server
|
||||||
|
metricsRouter *mux.Router
|
||||||
|
metricsLock sync.Mutex
|
||||||
|
|
||||||
didFinishListen chan struct{}
|
didFinishListen chan struct{}
|
||||||
|
|
||||||
goroutineCount int32
|
goroutineCount int32
|
||||||
@@ -592,10 +599,22 @@ func NewServer(options ...Option) (*Server, error) {
|
|||||||
mlog.Error("Error to reset the server status.", mlog.Err(err))
|
mlog.Error("Error to reset the server status.", mlog.Err(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if s.startMetrics && s.Metrics != nil {
|
if s.startMetrics {
|
||||||
s.Metrics.StartServer()
|
s.SetupMetricsServer()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
s.AddLicenseListener(func(oldLicense, newLicense *model.License) {
|
||||||
|
if (oldLicense == nil && newLicense == nil) || !s.startMetrics {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if oldLicense != nil && newLicense != nil && *oldLicense.Features.Metrics == *newLicense.Features.Metrics {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.SetupMetricsServer()
|
||||||
|
})
|
||||||
|
|
||||||
s.SearchEngine.UpdateConfig(s.Config())
|
s.SearchEngine.UpdateConfig(s.Config())
|
||||||
searchConfigListenerId, searchLicenseListenerId := s.StartSearchEngine()
|
searchConfigListenerId, searchLicenseListenerId := s.StartSearchEngine()
|
||||||
s.searchConfigListenerId = searchConfigListenerId
|
s.searchConfigListenerId = searchConfigListenerId
|
||||||
@@ -613,6 +632,24 @@ func NewServer(options ...Option) (*Server, error) {
|
|||||||
return s, nil
|
return s, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Server) SetupMetricsServer() {
|
||||||
|
if !*s.Config().MetricsSettings.Enable {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.StopMetricsServer()
|
||||||
|
|
||||||
|
if err := s.InitMetricsRouter(); err != nil {
|
||||||
|
mlog.Error("Error initiating metrics router.", mlog.Err(err))
|
||||||
|
}
|
||||||
|
|
||||||
|
if s.Metrics != nil {
|
||||||
|
s.Metrics.Register()
|
||||||
|
}
|
||||||
|
|
||||||
|
s.startMetricsServer()
|
||||||
|
}
|
||||||
|
|
||||||
func maxInt(a, b int) int {
|
func maxInt(a, b int) int {
|
||||||
if a > b {
|
if a > b {
|
||||||
return a
|
return a
|
||||||
@@ -854,9 +891,7 @@ func (s *Server) Shutdown() {
|
|||||||
s.Cluster.StopInterNodeCommunication()
|
s.Cluster.StopInterNodeCommunication()
|
||||||
}
|
}
|
||||||
|
|
||||||
if s.Metrics != nil {
|
s.StopMetricsServer()
|
||||||
s.Metrics.StopServer()
|
|
||||||
}
|
|
||||||
|
|
||||||
// This must be done after the cluster is stopped.
|
// This must be done after the cluster is stopped.
|
||||||
if s.Jobs != nil && s.runjobs {
|
if s.Jobs != nil && s.runjobs {
|
||||||
@@ -1467,6 +1502,98 @@ func doCheckWarnMetricStatus(a *App) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Server) StopMetricsServer() {
|
||||||
|
s.metricsLock.Lock()
|
||||||
|
defer s.metricsLock.Unlock()
|
||||||
|
|
||||||
|
if s.metricsServer != nil {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), TimeToWaitForConnectionsToCloseOnServerShutdown)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
s.metricsServer.Shutdown(ctx)
|
||||||
|
s.Log.Info("Metrics and profiling server is stopping", mlog.String("address", *s.Config().MetricsSettings.ListenAddress))
|
||||||
|
s.metricsServer = nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) HandleMetrics(route string, h http.Handler) {
|
||||||
|
if s.metricsRouter != nil {
|
||||||
|
s.metricsRouter.Handle(route, h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) InitMetricsRouter() error {
|
||||||
|
s.metricsRouter = mux.NewRouter()
|
||||||
|
runtime.SetBlockProfileRate(*s.Config().MetricsSettings.BlockProfileRate)
|
||||||
|
|
||||||
|
metricsPage := `
|
||||||
|
<html>
|
||||||
|
<body>{{if .}}
|
||||||
|
<div><a href="/metrics">Metrics</a></div>{{end}}
|
||||||
|
<div><a href="/debug/pprof/">Profiling Root</a></div>
|
||||||
|
<div><a href="/debug/pprof/cmdline">Profiling Command Line</a></div>
|
||||||
|
<div><a href="/debug/pprof/symbol">Profiling Symbols</a></div>
|
||||||
|
<div><a href="/debug/pprof/goroutine">Profiling Goroutines</a></div>
|
||||||
|
<div><a href="/debug/pprof/heap">Profiling Heap</a></div>
|
||||||
|
<div><a href="/debug/pprof/threadcreate">Profiling Threads</a></div>
|
||||||
|
<div><a href="/debug/pprof/block">Profiling Blocking</a></div>
|
||||||
|
<div><a href="/debug/pprof/trace">Profiling Execution Trace</a></div>
|
||||||
|
<div><a href="/debug/pprof/profile">Profiling CPU</a></div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
`
|
||||||
|
metricsPageTmpl, err := template.New("page").Parse(metricsPage)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrap(err, "failed to create template")
|
||||||
|
}
|
||||||
|
|
||||||
|
rootHandler := func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
metricsPageTmpl.Execute(w, s.Metrics != nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
s.metricsRouter.HandleFunc("/", rootHandler)
|
||||||
|
s.metricsRouter.StrictSlash(true)
|
||||||
|
|
||||||
|
s.metricsRouter.Handle("/debug", http.RedirectHandler("/", http.StatusMovedPermanently))
|
||||||
|
s.metricsRouter.HandleFunc("/debug/pprof/", pprof.Index)
|
||||||
|
s.metricsRouter.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
|
||||||
|
s.metricsRouter.HandleFunc("/debug/pprof/profile", pprof.Profile)
|
||||||
|
s.metricsRouter.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
|
||||||
|
s.metricsRouter.HandleFunc("/debug/pprof/trace", pprof.Trace)
|
||||||
|
|
||||||
|
// Manually add support for paths linked to by index page at /debug/pprof/
|
||||||
|
s.metricsRouter.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine"))
|
||||||
|
s.metricsRouter.Handle("/debug/pprof/heap", pprof.Handler("heap"))
|
||||||
|
s.metricsRouter.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
|
||||||
|
s.metricsRouter.Handle("/debug/pprof/block", pprof.Handler("block"))
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) startMetricsServer() {
|
||||||
|
s.metricsLock.Lock()
|
||||||
|
defer s.metricsLock.Unlock()
|
||||||
|
|
||||||
|
if s.metricsServer != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.metricsServer = &http.Server{
|
||||||
|
Addr: *s.Config().MetricsSettings.ListenAddress,
|
||||||
|
Handler: handlers.RecoveryHandler(handlers.PrintRecoveryStack(true))(s.metricsRouter),
|
||||||
|
ReadTimeout: time.Duration(*s.Config().ServiceSettings.ReadTimeout) * time.Second,
|
||||||
|
WriteTimeout: time.Duration(*s.Config().ServiceSettings.WriteTimeout) * time.Second,
|
||||||
|
}
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
if err := s.metricsServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||||
|
mlog.Critical(err.Error())
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
s.Log.Info("Metrics and profiling server is started", mlog.String("address", *s.Config().MetricsSettings.ListenAddress))
|
||||||
|
}
|
||||||
|
|
||||||
func doLicenseExpirationCheck(a *App) {
|
func doLicenseExpirationCheck(a *App) {
|
||||||
a.Srv().LoadLicense()
|
a.Srv().LoadLicense()
|
||||||
license := a.Srv().License()
|
license := a.Srv().License()
|
||||||
|
|||||||
@@ -8,8 +8,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type MetricsInterface interface {
|
type MetricsInterface interface {
|
||||||
StartServer()
|
Register()
|
||||||
StopServer()
|
|
||||||
|
|
||||||
IncrementPostCreate()
|
IncrementPostCreate()
|
||||||
IncrementWebhookPost()
|
IncrementWebhookPost()
|
||||||
|
|||||||
@@ -260,12 +260,7 @@ func (_m *MetricsInterface) ObserveStoreMethodDuration(method string, success st
|
|||||||
_m.Called(method, success, elapsed)
|
_m.Called(method, success, elapsed)
|
||||||
}
|
}
|
||||||
|
|
||||||
// StartServer provides a mock function with given fields:
|
// Register provides a mock function with given fields:
|
||||||
func (_m *MetricsInterface) StartServer() {
|
func (_m *MetricsInterface) Register() {
|
||||||
_m.Called()
|
|
||||||
}
|
|
||||||
|
|
||||||
// StopServer provides a mock function with given fields:
|
|
||||||
func (_m *MetricsInterface) StopServer() {
|
|
||||||
_m.Called()
|
_m.Called()
|
||||||
}
|
}
|
||||||
|
|||||||
Ссылка в новой задаче
Block a user