[MM-23281] Persist registeredPlugin in plugin.Environment when plugin is deactivated (#14110)

* store failed timestamps on health check job instead of on registeredPlugin

Update test

* change EnsurePlugin calls

* Make env.SetPluginState private

* Write test for plugin deactivate and PluginStateFailedToStayRunning

* Add license comment

* adjust comments, use time.Since

* Additional PR feedback:

time.Since cleanup
test cleanup
remove duplicate .Store() call

* PR Feedback

- Add test case for reactivating the failed plugin
- Change `crashed` to `healthy` and `hasPluginCrashed` to `isPluginHealthy`
- remove stale timestamps from health check job

* Keep registeredPlugins in env when plugin is deactivated, so the crashed state of a plugin can be persisted.

* PR feedback

* PR feedback from Jesse

Co-authored-by: mattermod <mattermod@users.noreply.github.com>

app/plugin_health_check_test.go

@@ -0,0 +1,85 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package app
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/plugin"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHealthCheckJob(t *testing.T) {
+	th := Setup(t).InitBasic()
+	defer th.TearDown()
+
+	tearDown, _, _ := SetAppEnvironmentWithPlugins(t, []string{
+		`
+			package main
+
+			import (
+				"github.com/mattermost/mattermost-server/v5/model"
+				"github.com/mattermost/mattermost-server/v5/plugin"
+			)
+
+			type MyPlugin struct {
+				plugin.MattermostPlugin
+			}
+
+			func (p *MyPlugin) MessageWillBePosted(c *plugin.Context, post *model.Post) (*model.Post, string) {
+				panic("Uncaught error")
+			}
+
+			func main() {
+				plugin.ClientMain(&MyPlugin{})
+			}
+			`,
+	}, th.App, th.App.NewPluginAPI)
+	defer tearDown()
+
+	env := th.App.GetPluginsEnvironment()
+	job := env.GetPluginHealthCheckJob()
+	require.NotNil(t, job)
+	bundles := env.Active()
+	require.Equal(t, 1, len(bundles))
+
+	id := bundles[0].Manifest.Id
+
+	// First health check
+	hooks, err := env.HooksForPlugin(id)
+	require.Nil(t, err)
+	hooks.MessageWillBePosted(&plugin.Context{}, &model.Post{})
+	job.CheckPlugin(id)
+	bundles = env.Active()
+	require.Equal(t, 1, len(bundles))
+	require.Equal(t, id, bundles[0].Manifest.Id)
+	require.Equal(t, model.PluginStateRunning, env.GetPluginState(id))
+
+	// Second health check
+	hooks, err = env.HooksForPlugin(id)
+	require.Nil(t, err)
+	hooks.MessageWillBePosted(&plugin.Context{}, &model.Post{})
+	job.CheckPlugin(id)
+	bundles = env.Active()
+	require.Equal(t, 1, len(bundles))
+	require.Equal(t, id, bundles[0].Manifest.Id)
+	require.Equal(t, model.PluginStateRunning, env.GetPluginState(id))
+
+	// Third health check, plugin should be deactivated by the job
+	hooks, err = env.HooksForPlugin(id)
+	require.Nil(t, err)
+	hooks.MessageWillBePosted(&plugin.Context{}, &model.Post{})
+	job.CheckPlugin(id)
+	bundles = env.Active()
+	require.Equal(t, 0, len(bundles))
+	require.Equal(t, model.PluginStateFailedToStayRunning, env.GetPluginState(id))
+
+	// Activated manually, plugin should stay active
+	env.Activate(id)
+	job.CheckPlugin(id)
+	bundles = env.Active()
+	require.Equal(t, 1, len(bundles))
+	require.Equal(t, model.PluginStateRunning, env.GetPluginState(id))
+}