MM-16261: Synchronize plugins in HA (#11657)

* MM-16272 - Synchronize plugins across cluster (#11611)

* MM-16272 - Synchronize plugins across cluster

* Adding a test

* MM-16272 - Fixed tests

* MM-16272 - PR feedback

* MM-16270 - Plugin Sync (#11615)

* Initial implementation for plugin synch with file store. WIP

* Removed ListAll implementation. Used ListDirectory and change localstore to be consistent and return all items (files and folders) from directory

* Refactored plugin filestore operations out of main install/remove plugin

* Fixing error handling details

* Changes to use structured logging

* More logging fixes

* Wording and comments improvements

* Error handling and control flow improvements

* Changed managed flag check to use os.stat

* Added file store plugin dir and filename consts

* Replaced FileRead to use a the FileReader in PluginSync

* Minor styling and PR feedback changes

* Minor error handling improvements

* Added unit test for SyncPlugins. Changed SyncPlugins to use plugins environment to list available plugins

* PR Feedback improvements

* Minor err handling fix

* Removing FileStorePath from PluginEventData (#11644)

* Fix plugin path (#11654)

* tweak path, logging

Fix an issue not finding the plugins folder in S3. Tweak logging messages to add additional clarity.

* Removing FileExists check when Syncing plugins. Updated localstore to not return an error when directory does not exist

* PR Feedback

* Install prepackaged plugins locally only (#11656)

* s/uninstall/remove

* Updated ClusterMessage comment

* Updated PluginSync to test against s3 + local storage

api4/plugin_test.go

@@ -14,8 +14,10 @@ import (
 	"testing"
 
 	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/testlib"
 	"github.com/mattermost/mattermost-server/utils/fileutils"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestPlugin(t *testing.T) {
@@ -234,3 +236,75 @@ func TestPlugin(t *testing.T) {
 	_, resp = th.SystemAdminClient.RemovePlugin("bad.id")
 	CheckBadRequestStatus(t, resp)
 }
+
+func TestNotifyClusterPluginEvent(t *testing.T) {
+	th := Setup().InitBasic()
+	defer th.TearDown()
+
+	testCluster := &testlib.FakeClusterInterface{}
+	th.Server.Cluster = testCluster
+
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.PluginSettings.Enable = true
+		*cfg.PluginSettings.EnableUploads = true
+	})
+
+	path, _ := fileutils.FindDir("tests")
+	tarData, err := ioutil.ReadFile(filepath.Join(path, "testplugin.tar.gz"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Successful upload
+	manifest, resp := th.SystemAdminClient.UploadPlugin(bytes.NewReader(tarData))
+	CheckNoError(t, resp)
+	require.Equal(t, "testplugin", manifest.Id)
+
+	// Stored in File Store: Upload Plugin case
+	expectedPath := filepath.Join("./plugins", manifest.Id) + ".tar.gz"
+	pluginStored, err := th.App.FileExists(expectedPath)
+	require.Nil(t, err)
+	require.True(t, pluginStored)
+
+	expectedPluginData := model.PluginEventData{
+		Id: manifest.Id,
+	}
+	expectedInstallMessage := &model.ClusterMessage{
+		Event:            model.CLUSTER_EVENT_INSTALL_PLUGIN,
+		SendType:         model.CLUSTER_SEND_RELIABLE,
+		WaitForAllToSend: true,
+		Data:             expectedPluginData.ToJson(),
+	}
+	expectedMessages := findClusterMessages(model.CLUSTER_EVENT_INSTALL_PLUGIN, testCluster.GetMessages())
+	require.Equal(t, []*model.ClusterMessage{expectedInstallMessage}, expectedMessages)
+
+	// Successful remove
+	testCluster.ClearMessages()
+
+	ok, resp := th.SystemAdminClient.RemovePlugin(manifest.Id)
+	CheckNoError(t, resp)
+	require.True(t, ok)
+
+	expectedRemoveMessage := &model.ClusterMessage{
+		Event:            model.CLUSTER_EVENT_REMOVE_PLUGIN,
+		SendType:         model.CLUSTER_SEND_RELIABLE,
+		WaitForAllToSend: true,
+		Data:             expectedPluginData.ToJson(),
+	}
+	expectedMessages = findClusterMessages(model.CLUSTER_EVENT_REMOVE_PLUGIN, testCluster.GetMessages())
+	require.Equal(t, []*model.ClusterMessage{expectedRemoveMessage}, expectedMessages)
+
+	pluginStored, err = th.App.FileExists(expectedPath)
+	require.Nil(t, err)
+	require.False(t, pluginStored)
+}
+
+func findClusterMessages(event string, msgs []*model.ClusterMessage) []*model.ClusterMessage {
+	var result []*model.ClusterMessage
+	for _, msg := range msgs {
+		if msg.Event == event {
+			result = append(result, msg)
+		}
+	}
+	return result
+}