[MM-23281] Persist registeredPlugin in plugin.Environment when plugin is deactivated (#14110)

* store failed timestamps on health check job instead of on registeredPlugin

Update test

* change EnsurePlugin calls

* Make env.SetPluginState private

* Write test for plugin deactivate and PluginStateFailedToStayRunning

* Add license comment

* adjust comments, use time.Since

* Additional PR feedback:

time.Since cleanup
test cleanup
remove duplicate .Store() call

* PR Feedback

- Add test case for reactivating the failed plugin
- Change `crashed` to `healthy` and `hasPluginCrashed` to `isPluginHealthy`
- remove stale timestamps from health check job

* Keep registeredPlugins in env when plugin is deactivated, so the crashed state of a plugin can be persisted.

* PR feedback

* PR feedback from Jesse

Co-authored-by: mattermod <mattermod@users.noreply.github.com>

plugin/health_check_test.go

@@ -118,39 +118,36 @@ func testPluginHealthCheckPanic(t *testing.T) {
 }
 
 func TestShouldDeactivatePlugin(t *testing.T) {
-	bundle := &model.BundleInfo{}
-	rp := newRegisteredPlugin(bundle)
-	require.NotNil(t, rp)
-
 	// No failures, don't restart
-	result := shouldDeactivatePlugin(rp)
+	ftime := []time.Time{}
+	result := shouldDeactivatePlugin(ftime)
 	require.Equal(t, false, result)
 
 	now := time.Now()
 
 	// Failures are recent enough to restart
-	rp = newRegisteredPlugin(bundle)
-	rp.failTimeStamps = append(rp.failTimeStamps, now.Add(-HEALTH_CHECK_DISABLE_DURATION/10*2))
-	rp.failTimeStamps = append(rp.failTimeStamps, now.Add(-HEALTH_CHECK_DISABLE_DURATION/10))
-	rp.failTimeStamps = append(rp.failTimeStamps, now)
+	ftime = []time.Time{}
+	ftime = append(ftime, now.Add(-HEALTH_CHECK_DEACTIVATION_WINDOW/10*2))
+	ftime = append(ftime, now.Add(-HEALTH_CHECK_DEACTIVATION_WINDOW/10))
+	ftime = append(ftime, now)
 
-	result = shouldDeactivatePlugin(rp)
+	result = shouldDeactivatePlugin(ftime)
 	require.Equal(t, true, result)
 
 	// Failures are too spaced out to warrant a restart
-	rp = newRegisteredPlugin(bundle)
-	rp.failTimeStamps = append(rp.failTimeStamps, now.Add(-HEALTH_CHECK_DISABLE_DURATION*2))
-	rp.failTimeStamps = append(rp.failTimeStamps, now.Add(-HEALTH_CHECK_DISABLE_DURATION*1))
-	rp.failTimeStamps = append(rp.failTimeStamps, now)
+	ftime = []time.Time{}
+	ftime = append(ftime, now.Add(-HEALTH_CHECK_DEACTIVATION_WINDOW*2))
+	ftime = append(ftime, now.Add(-HEALTH_CHECK_DEACTIVATION_WINDOW*1))
+	ftime = append(ftime, now)
 
-	result = shouldDeactivatePlugin(rp)
+	result = shouldDeactivatePlugin(ftime)
 	require.Equal(t, false, result)
 
 	// Not enough failures are present to warrant a restart
-	rp = newRegisteredPlugin(bundle)
-	rp.failTimeStamps = append(rp.failTimeStamps, now.Add(-HEALTH_CHECK_DISABLE_DURATION/10))
-	rp.failTimeStamps = append(rp.failTimeStamps, now)
+	ftime = []time.Time{}
+	ftime = append(ftime, now.Add(-HEALTH_CHECK_DEACTIVATION_WINDOW/10))
+	ftime = append(ftime, now)
 
-	result = shouldDeactivatePlugin(rp)
+	result = shouldDeactivatePlugin(ftime)
 	require.Equal(t, false, result)
 }