MM-8622: Improved plugin error reporting (#8737)

* allow `Wait()`ing on the supervisor

In the event the plugin supervisor shuts down a plugin for crashing too
many times, the new `Wait()` interface allows the `ActivatePlugin` to
accept a callback function to trigger when `supervisor.Wait()` returns.
If the supervisor shuts down normally, this callback is invoked with
a nil error, otherwise any error reported by the supervisor is passed
along.

* improve plugin activation/deactivation logic

Avoid triggering activation of previously failed-to-start plugins just
becase something in the configuration changed. Now, intelligently
compare the global enable bit as well as the each individual plugin's
enabled bit.

* expose store to manipulate PluginStatuses

* expose API to fetch plugin statuses

* keep track of whether or not plugin sandboxing is supported

* transition plugin statuses

* restore error on plugin activation if already active

* don't initialize test plugins until successfully loaded

* emit websocket events when plugin statuses change

* skip pruning if already initialized

* MM-8622: maintain plugin statuses in memory

Switch away from persisting plugin statuses to the database, and
maintain in memory instead. This will be followed by a cluster interface
to query the in-memory status of plugin statuses from all cluster nodes.

At the same time, rename `cluster_discovery_id` on the `PluginStatus`
model object to `cluster_id`.

* MM-8622: aggregate plugin statuses across cluster

* fetch cluster plugin statuses when emitting websocket notification

* address unit test fixes after rebasing

* relax (poor) racey unit test re: supervisor.Wait()

* make store-mocks

app/plugin_test.go

@@ -7,8 +7,8 @@ import (
 	"errors"
 	"net/http"
 	"net/http/httptest"
-	"strings"
 	"testing"
+	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/stretchr/testify/assert"
@@ -158,6 +158,20 @@ func TestPluginCommands(t *testing.T) {
 
 	require.Nil(t, th.App.EnablePlugin("foo"))
 
+	// Ideally, we would wait for the websocket activation event instead of just sleeping.
+	time.Sleep(500 * time.Millisecond)
+
+	pluginStatuses, err := th.App.GetPluginStatuses()
+	require.Nil(t, err)
+	found := false
+	for _, pluginStatus := range pluginStatuses {
+		if pluginStatus.PluginId == "foo" {
+			require.Equal(t, model.PluginStateRunning, pluginStatus.State)
+			found = true
+		}
+	}
+	require.True(t, found, "failed to find plugin foo in plugin statuses")
+
 	resp, err := th.App.ExecuteCommand(&model.CommandArgs{
 		Command:   "/foo2",
 		TeamId:    th.BasicTeam.Id,
@@ -216,7 +230,46 @@ func TestPluginBadActivation(t *testing.T) {
 
 	t.Run("EnablePlugin bad activation", func(t *testing.T) {
 		err := th.App.EnablePlugin("foo")
-		assert.NotNil(t, err)
-		assert.True(t, strings.Contains(err.DetailedError, "won't activate for some reason"))
+		assert.Nil(t, err)
+
+		// Ideally, we would wait for the websocket activation event instead of just
+		// sleeping.
+		time.Sleep(500 * time.Millisecond)
+
+		pluginStatuses, err := th.App.GetPluginStatuses()
+		require.Nil(t, err)
+		found := false
+		for _, pluginStatus := range pluginStatuses {
+			if pluginStatus.PluginId == "foo" {
+				require.Equal(t, model.PluginStateFailedToStart, pluginStatus.State)
+				found = true
+			}
+		}
+		require.True(t, found, "failed to find plugin foo in plugin statuses")
 	})
 }
+
+func TestGetPluginStatusesDisabled(t *testing.T) {
+	th := Setup().InitBasic()
+	defer th.TearDown()
+
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.PluginSettings.Enable = false
+	})
+
+	_, err := th.App.GetPluginStatuses()
+	require.EqualError(t, err, "GetPluginStatuses: Plugins have been disabled. Please check your logs for details., ")
+}
+
+func TestGetPluginStatuses(t *testing.T) {
+	th := Setup().InitBasic()
+	defer th.TearDown()
+
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.PluginSettings.Enable = true
+	})
+
+	pluginStatuses, err := th.App.GetPluginStatuses()
+	require.Nil(t, err)
+	require.NotNil(t, pluginStatuses)
+}