[MM-31132] app/server: add pprof endpoint (#17001)
* app/server: add pprof endpoint * reflect review comments * make metrics link conditional * app/server: add metrics server setup to licence listeners * refactor a bit * trigger CI Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
Этот коммит содержится в:
Ibrahim Serdar Acikgoz
коммит произвёл
GitHub
GitHub
родитель
5cb5f0a60a
Коммит
f31a9ed1a8
@@ -411,12 +411,13 @@ func (s *Server) SaveConfig(newCfg *model.Config, sendConfigChangeClusterMessage
|
||||
return model.NewAppError("saveConfig", "app.save_config.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
if s.Metrics != nil {
|
||||
if *s.Config().MetricsSettings.Enable {
|
||||
s.Metrics.StartServer()
|
||||
} else {
|
||||
s.Metrics.StopServer()
|
||||
if s.startMetrics && *s.Config().MetricsSettings.Enable {
|
||||
if s.Metrics != nil {
|
||||
s.Metrics.Register()
|
||||
}
|
||||
s.SetupMetricsServer()
|
||||
} else {
|
||||
s.StopMetricsServer()
|
||||
}
|
||||
|
||||
if s.Cluster != nil {
|
||||
|
||||
137
app/server.go
137
app/server.go
@@ -9,9 +9,11 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"hash/maphash"
|
||||
"html/template"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/pprof"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -29,6 +31,7 @@ import (
|
||||
|
||||
"github.com/getsentry/sentry-go"
|
||||
sentryhttp "github.com/getsentry/sentry-go/http"
|
||||
"github.com/gorilla/handlers"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/mailru/easygo/netpoll"
|
||||
"github.com/pkg/errors"
|
||||
@@ -93,6 +96,10 @@ type Server struct {
|
||||
|
||||
localModeServer *http.Server
|
||||
|
||||
metricsServer *http.Server
|
||||
metricsRouter *mux.Router
|
||||
metricsLock sync.Mutex
|
||||
|
||||
didFinishListen chan struct{}
|
||||
|
||||
goroutineCount int32
|
||||
@@ -592,10 +599,22 @@ func NewServer(options ...Option) (*Server, error) {
|
||||
mlog.Error("Error to reset the server status.", mlog.Err(err))
|
||||
}
|
||||
|
||||
if s.startMetrics && s.Metrics != nil {
|
||||
s.Metrics.StartServer()
|
||||
if s.startMetrics {
|
||||
s.SetupMetricsServer()
|
||||
}
|
||||
|
||||
s.AddLicenseListener(func(oldLicense, newLicense *model.License) {
|
||||
if (oldLicense == nil && newLicense == nil) || !s.startMetrics {
|
||||
return
|
||||
}
|
||||
|
||||
if oldLicense != nil && newLicense != nil && *oldLicense.Features.Metrics == *newLicense.Features.Metrics {
|
||||
return
|
||||
}
|
||||
|
||||
s.SetupMetricsServer()
|
||||
})
|
||||
|
||||
s.SearchEngine.UpdateConfig(s.Config())
|
||||
searchConfigListenerId, searchLicenseListenerId := s.StartSearchEngine()
|
||||
s.searchConfigListenerId = searchConfigListenerId
|
||||
@@ -613,6 +632,24 @@ func NewServer(options ...Option) (*Server, error) {
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *Server) SetupMetricsServer() {
|
||||
if !*s.Config().MetricsSettings.Enable {
|
||||
return
|
||||
}
|
||||
|
||||
s.StopMetricsServer()
|
||||
|
||||
if err := s.InitMetricsRouter(); err != nil {
|
||||
mlog.Error("Error initiating metrics router.", mlog.Err(err))
|
||||
}
|
||||
|
||||
if s.Metrics != nil {
|
||||
s.Metrics.Register()
|
||||
}
|
||||
|
||||
s.startMetricsServer()
|
||||
}
|
||||
|
||||
func maxInt(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
@@ -854,9 +891,7 @@ func (s *Server) Shutdown() {
|
||||
s.Cluster.StopInterNodeCommunication()
|
||||
}
|
||||
|
||||
if s.Metrics != nil {
|
||||
s.Metrics.StopServer()
|
||||
}
|
||||
s.StopMetricsServer()
|
||||
|
||||
// This must be done after the cluster is stopped.
|
||||
if s.Jobs != nil && s.runjobs {
|
||||
@@ -1467,6 +1502,98 @@ func doCheckWarnMetricStatus(a *App) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) StopMetricsServer() {
|
||||
s.metricsLock.Lock()
|
||||
defer s.metricsLock.Unlock()
|
||||
|
||||
if s.metricsServer != nil {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), TimeToWaitForConnectionsToCloseOnServerShutdown)
|
||||
defer cancel()
|
||||
|
||||
s.metricsServer.Shutdown(ctx)
|
||||
s.Log.Info("Metrics and profiling server is stopping", mlog.String("address", *s.Config().MetricsSettings.ListenAddress))
|
||||
s.metricsServer = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) HandleMetrics(route string, h http.Handler) {
|
||||
if s.metricsRouter != nil {
|
||||
s.metricsRouter.Handle(route, h)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) InitMetricsRouter() error {
|
||||
s.metricsRouter = mux.NewRouter()
|
||||
runtime.SetBlockProfileRate(*s.Config().MetricsSettings.BlockProfileRate)
|
||||
|
||||
metricsPage := `
|
||||
<html>
|
||||
<body>{{if .}}
|
||||
<div><a href="/metrics">Metrics</a></div>{{end}}
|
||||
<div><a href="/debug/pprof/">Profiling Root</a></div>
|
||||
<div><a href="/debug/pprof/cmdline">Profiling Command Line</a></div>
|
||||
<div><a href="/debug/pprof/symbol">Profiling Symbols</a></div>
|
||||
<div><a href="/debug/pprof/goroutine">Profiling Goroutines</a></div>
|
||||
<div><a href="/debug/pprof/heap">Profiling Heap</a></div>
|
||||
<div><a href="/debug/pprof/threadcreate">Profiling Threads</a></div>
|
||||
<div><a href="/debug/pprof/block">Profiling Blocking</a></div>
|
||||
<div><a href="/debug/pprof/trace">Profiling Execution Trace</a></div>
|
||||
<div><a href="/debug/pprof/profile">Profiling CPU</a></div>
|
||||
</body>
|
||||
</html>
|
||||
`
|
||||
metricsPageTmpl, err := template.New("page").Parse(metricsPage)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create template")
|
||||
}
|
||||
|
||||
rootHandler := func(w http.ResponseWriter, r *http.Request) {
|
||||
metricsPageTmpl.Execute(w, s.Metrics != nil)
|
||||
}
|
||||
|
||||
s.metricsRouter.HandleFunc("/", rootHandler)
|
||||
s.metricsRouter.StrictSlash(true)
|
||||
|
||||
s.metricsRouter.Handle("/debug", http.RedirectHandler("/", http.StatusMovedPermanently))
|
||||
s.metricsRouter.HandleFunc("/debug/pprof/", pprof.Index)
|
||||
s.metricsRouter.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
|
||||
s.metricsRouter.HandleFunc("/debug/pprof/profile", pprof.Profile)
|
||||
s.metricsRouter.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
|
||||
s.metricsRouter.HandleFunc("/debug/pprof/trace", pprof.Trace)
|
||||
|
||||
// Manually add support for paths linked to by index page at /debug/pprof/
|
||||
s.metricsRouter.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine"))
|
||||
s.metricsRouter.Handle("/debug/pprof/heap", pprof.Handler("heap"))
|
||||
s.metricsRouter.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
|
||||
s.metricsRouter.Handle("/debug/pprof/block", pprof.Handler("block"))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) startMetricsServer() {
|
||||
s.metricsLock.Lock()
|
||||
defer s.metricsLock.Unlock()
|
||||
|
||||
if s.metricsServer != nil {
|
||||
return
|
||||
}
|
||||
|
||||
s.metricsServer = &http.Server{
|
||||
Addr: *s.Config().MetricsSettings.ListenAddress,
|
||||
Handler: handlers.RecoveryHandler(handlers.PrintRecoveryStack(true))(s.metricsRouter),
|
||||
ReadTimeout: time.Duration(*s.Config().ServiceSettings.ReadTimeout) * time.Second,
|
||||
WriteTimeout: time.Duration(*s.Config().ServiceSettings.WriteTimeout) * time.Second,
|
||||
}
|
||||
|
||||
go func() {
|
||||
if err := s.metricsServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
mlog.Critical(err.Error())
|
||||
}
|
||||
}()
|
||||
|
||||
s.Log.Info("Metrics and profiling server is started", mlog.String("address", *s.Config().MetricsSettings.ListenAddress))
|
||||
}
|
||||
|
||||
func doLicenseExpirationCheck(a *App) {
|
||||
a.Srv().LoadLicense()
|
||||
license := a.Srv().License()
|
||||
|
||||
@@ -8,8 +8,7 @@ import (
|
||||
)
|
||||
|
||||
type MetricsInterface interface {
|
||||
StartServer()
|
||||
StopServer()
|
||||
Register()
|
||||
|
||||
IncrementPostCreate()
|
||||
IncrementWebhookPost()
|
||||
|
||||
@@ -260,12 +260,7 @@ func (_m *MetricsInterface) ObserveStoreMethodDuration(method string, success st
|
||||
_m.Called(method, success, elapsed)
|
||||
}
|
||||
|
||||
// StartServer provides a mock function with given fields:
|
||||
func (_m *MetricsInterface) StartServer() {
|
||||
_m.Called()
|
||||
}
|
||||
|
||||
// StopServer provides a mock function with given fields:
|
||||
func (_m *MetricsInterface) StopServer() {
|
||||
// Register provides a mock function with given fields:
|
||||
func (_m *MetricsInterface) Register() {
|
||||
_m.Called()
|
||||
}
|
||||
|
||||
Ссылка в новой задаче
Block a user