[MM-13507] Plugin framework: auto-restart plugins that crash (#10781)

* introduce plugin health check

* implement plugin health check job

* add support for checking pid of plugin process and RPC ping, to determine a plugin's health

* implement restart policy with back-offs

* support "EnableHealthCheck" boolean from config file.

* add tests for supervisor.PerformHealthCheck() and shouldDeactivatePlugin()

* improve error handling. clean up if blocks to be more concise

app/server.go

@@ -188,6 +188,18 @@ func NewServer(options ...Option) (*Server, error) {
 		s.InitEmailBatching()
 	})
 
+	// Start plugin health check job
+	pluginsEnvironment := s.PluginsEnvironment
+	if pluginsEnvironment != nil {
+		pluginsEnvironment.InitPluginHealthCheckJob(*s.Config().PluginSettings.EnableHealthCheck)
+	}
+	s.AddConfigListener(func(_, c *model.Config) {
+		pluginsEnvironment := s.PluginsEnvironment
+		if pluginsEnvironment != nil {
+			pluginsEnvironment.InitPluginHealthCheckJob(*c.PluginSettings.EnableHealthCheck)
+		}
+	})
+
 	mlog.Info(fmt.Sprintf("Current version is %v (%v/%v/%v/%v)", model.CurrentVersion, model.BuildNumber, model.BuildDate, model.BuildHash, model.BuildHashEnterprise))
 	mlog.Info(fmt.Sprintf("Enterprise Enabled: %v", model.BuildEnterpriseReady))
 	pwd, _ := os.Getwd()