[MM-13507] Plugin framework: auto-restart plugins that crash (#10781)

* introduce plugin health check

* implement plugin health check job

* add support for checking pid of plugin process and RPC ping, to determine a plugin's health

* implement restart policy with back-offs

* support "EnableHealthCheck" boolean from config file.

* add tests for supervisor.PerformHealthCheck() and shouldDeactivatePlugin()

* improve error handling. clean up if blocks to be more concise

model/config.go

@@ -2065,12 +2065,13 @@ type PluginState struct {
 }
 
 type PluginSettings struct {
-	Enable          *bool
-	EnableUploads   *bool   `restricted:"true"`
-	Directory       *string `restricted:"true"`
-	ClientDirectory *string `restricted:"true"`
-	Plugins         map[string]map[string]interface{}
-	PluginStates    map[string]*PluginState
+	Enable            *bool
+	EnableUploads     *bool   `restricted:"true"`
+	EnableHealthCheck *bool   `restricted:"true"`
+	Directory         *string `restricted:"true"`
+	ClientDirectory   *string `restricted:"true"`
+	Plugins           map[string]map[string]interface{}
+	PluginStates      map[string]*PluginState
 }
 
 func (s *PluginSettings) SetDefaults() {
@@ -2082,6 +2083,10 @@ func (s *PluginSettings) SetDefaults() {
 		s.EnableUploads = NewBool(false)
 	}
 
+	if s.EnableHealthCheck == nil {
+		s.EnableHealthCheck = NewBool(true)
+	}
+
 	if s.Directory == nil {
 		s.Directory = NewString(PLUGIN_SETTINGS_DEFAULT_DIRECTORY)
 	}