diff --git a/app/config.go b/app/config.go
index 4ba2cbd0fa..4d3faec5ed 100644
--- a/app/config.go
+++ b/app/config.go
@@ -411,12 +411,13 @@ func (s *Server) SaveConfig(newCfg *model.Config, sendConfigChangeClusterMessage
return model.NewAppError("saveConfig", "app.save_config.app_error", nil, err.Error(), http.StatusInternalServerError)
}
- if s.Metrics != nil {
- if *s.Config().MetricsSettings.Enable {
- s.Metrics.StartServer()
- } else {
- s.Metrics.StopServer()
+ if s.startMetrics && *s.Config().MetricsSettings.Enable {
+ if s.Metrics != nil {
+ s.Metrics.Register()
}
+ s.SetupMetricsServer()
+ } else {
+ s.StopMetricsServer()
}
if s.Cluster != nil {
diff --git a/app/server.go b/app/server.go
index 0000945ef9..f2520de07b 100644
--- a/app/server.go
+++ b/app/server.go
@@ -9,9 +9,11 @@ import (
"encoding/json"
"fmt"
"hash/maphash"
+ "html/template"
"io/ioutil"
"net"
"net/http"
+ "net/http/pprof"
"net/url"
"os"
"os/exec"
@@ -29,6 +31,7 @@ import (
"github.com/getsentry/sentry-go"
sentryhttp "github.com/getsentry/sentry-go/http"
+ "github.com/gorilla/handlers"
"github.com/gorilla/mux"
"github.com/mailru/easygo/netpoll"
"github.com/pkg/errors"
@@ -93,6 +96,10 @@ type Server struct {
localModeServer *http.Server
+ metricsServer *http.Server
+ metricsRouter *mux.Router
+ metricsLock sync.Mutex
+
didFinishListen chan struct{}
goroutineCount int32
@@ -592,10 +599,22 @@ func NewServer(options ...Option) (*Server, error) {
mlog.Error("Error to reset the server status.", mlog.Err(err))
}
- if s.startMetrics && s.Metrics != nil {
- s.Metrics.StartServer()
+ if s.startMetrics {
+ s.SetupMetricsServer()
}
+ s.AddLicenseListener(func(oldLicense, newLicense *model.License) {
+ if (oldLicense == nil && newLicense == nil) || !s.startMetrics {
+ return
+ }
+
+ if oldLicense != nil && newLicense != nil && *oldLicense.Features.Metrics == *newLicense.Features.Metrics {
+ return
+ }
+
+ s.SetupMetricsServer()
+ })
+
s.SearchEngine.UpdateConfig(s.Config())
searchConfigListenerId, searchLicenseListenerId := s.StartSearchEngine()
s.searchConfigListenerId = searchConfigListenerId
@@ -613,6 +632,24 @@ func NewServer(options ...Option) (*Server, error) {
return s, nil
}
+func (s *Server) SetupMetricsServer() {
+ if !*s.Config().MetricsSettings.Enable {
+ return
+ }
+
+ s.StopMetricsServer()
+
+ if err := s.InitMetricsRouter(); err != nil {
+ mlog.Error("Error initiating metrics router.", mlog.Err(err))
+ }
+
+ if s.Metrics != nil {
+ s.Metrics.Register()
+ }
+
+ s.startMetricsServer()
+}
+
func maxInt(a, b int) int {
if a > b {
return a
@@ -854,9 +891,7 @@ func (s *Server) Shutdown() {
s.Cluster.StopInterNodeCommunication()
}
- if s.Metrics != nil {
- s.Metrics.StopServer()
- }
+ s.StopMetricsServer()
// This must be done after the cluster is stopped.
if s.Jobs != nil && s.runjobs {
@@ -1467,6 +1502,98 @@ func doCheckWarnMetricStatus(a *App) {
}
}
+func (s *Server) StopMetricsServer() {
+ s.metricsLock.Lock()
+ defer s.metricsLock.Unlock()
+
+ if s.metricsServer != nil {
+ ctx, cancel := context.WithTimeout(context.Background(), TimeToWaitForConnectionsToCloseOnServerShutdown)
+ defer cancel()
+
+ s.metricsServer.Shutdown(ctx)
+ s.Log.Info("Metrics and profiling server is stopping", mlog.String("address", *s.Config().MetricsSettings.ListenAddress))
+ s.metricsServer = nil
+ }
+}
+
+func (s *Server) HandleMetrics(route string, h http.Handler) {
+ if s.metricsRouter != nil {
+ s.metricsRouter.Handle(route, h)
+ }
+}
+
+func (s *Server) InitMetricsRouter() error {
+ s.metricsRouter = mux.NewRouter()
+ runtime.SetBlockProfileRate(*s.Config().MetricsSettings.BlockProfileRate)
+
+ metricsPage := `
+
+ {{if .}}
+ {{end}}
+
+
+
+
+
+
+
+
+
+
+
+ `
+ metricsPageTmpl, err := template.New("page").Parse(metricsPage)
+ if err != nil {
+ return errors.Wrap(err, "failed to create template")
+ }
+
+ rootHandler := func(w http.ResponseWriter, r *http.Request) {
+ metricsPageTmpl.Execute(w, s.Metrics != nil)
+ }
+
+ s.metricsRouter.HandleFunc("/", rootHandler)
+ s.metricsRouter.StrictSlash(true)
+
+ s.metricsRouter.Handle("/debug", http.RedirectHandler("/", http.StatusMovedPermanently))
+ s.metricsRouter.HandleFunc("/debug/pprof/", pprof.Index)
+ s.metricsRouter.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+ s.metricsRouter.HandleFunc("/debug/pprof/profile", pprof.Profile)
+ s.metricsRouter.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+ s.metricsRouter.HandleFunc("/debug/pprof/trace", pprof.Trace)
+
+ // Manually add support for paths linked to by index page at /debug/pprof/
+ s.metricsRouter.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine"))
+ s.metricsRouter.Handle("/debug/pprof/heap", pprof.Handler("heap"))
+ s.metricsRouter.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
+ s.metricsRouter.Handle("/debug/pprof/block", pprof.Handler("block"))
+
+ return nil
+}
+
+func (s *Server) startMetricsServer() {
+ s.metricsLock.Lock()
+ defer s.metricsLock.Unlock()
+
+ if s.metricsServer != nil {
+ return
+ }
+
+ s.metricsServer = &http.Server{
+ Addr: *s.Config().MetricsSettings.ListenAddress,
+ Handler: handlers.RecoveryHandler(handlers.PrintRecoveryStack(true))(s.metricsRouter),
+ ReadTimeout: time.Duration(*s.Config().ServiceSettings.ReadTimeout) * time.Second,
+ WriteTimeout: time.Duration(*s.Config().ServiceSettings.WriteTimeout) * time.Second,
+ }
+
+ go func() {
+ if err := s.metricsServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+ mlog.Critical(err.Error())
+ }
+ }()
+
+ s.Log.Info("Metrics and profiling server is started", mlog.String("address", *s.Config().MetricsSettings.ListenAddress))
+}
+
func doLicenseExpirationCheck(a *App) {
a.Srv().LoadLicense()
license := a.Srv().License()
diff --git a/einterfaces/metrics.go b/einterfaces/metrics.go
index ea4434aca0..565f2374a7 100644
--- a/einterfaces/metrics.go
+++ b/einterfaces/metrics.go
@@ -8,8 +8,7 @@ import (
)
type MetricsInterface interface {
- StartServer()
- StopServer()
+ Register()
IncrementPostCreate()
IncrementWebhookPost()
diff --git a/einterfaces/mocks/MetricsInterface.go b/einterfaces/mocks/MetricsInterface.go
index f4547baf3e..20480850db 100644
--- a/einterfaces/mocks/MetricsInterface.go
+++ b/einterfaces/mocks/MetricsInterface.go
@@ -260,12 +260,7 @@ func (_m *MetricsInterface) ObserveStoreMethodDuration(method string, success st
_m.Called(method, success, elapsed)
}
-// StartServer provides a mock function with given fields:
-func (_m *MetricsInterface) StartServer() {
- _m.Called()
-}
-
-// StopServer provides a mock function with given fields:
-func (_m *MetricsInterface) StopServer() {
+// Register provides a mock function with given fields:
+func (_m *MetricsInterface) Register() {
_m.Called()
}