MM-16261: Synchronize plugins in HA (#11657)

* MM-16272 - Synchronize plugins across cluster (#11611)

* MM-16272 - Synchronize plugins across cluster

* Adding a test

* MM-16272 - Fixed tests

* MM-16272 - PR feedback

* MM-16270 - Plugin Sync (#11615)

* Initial implementation for plugin synch with file store. WIP

* Removed ListAll implementation. Used ListDirectory and change localstore to be consistent and return all items (files and folders) from directory

* Refactored plugin filestore operations out of main install/remove plugin

* Fixing error handling details

* Changes to use structured logging

* More logging fixes

* Wording and comments improvements

* Error handling and control flow improvements

* Changed managed flag check to use os.stat

* Added file store plugin dir and filename consts

* Replaced FileRead to use a the FileReader in PluginSync

* Minor styling and PR feedback changes

* Minor error handling improvements

* Added unit test for SyncPlugins. Changed SyncPlugins to use plugins environment to list available plugins

* PR Feedback improvements

* Minor err handling fix

* Removing FileStorePath from PluginEventData (#11644)

* Fix plugin path (#11654)

* tweak path, logging

Fix an issue not finding the plugins folder in S3. Tweak logging messages to add additional clarity.

* Removing FileExists check when Syncing plugins. Updated localstore to not return an error when directory does not exist

* PR Feedback

* Install prepackaged plugins locally only (#11656)

* s/uninstall/remove

* Updated ClusterMessage comment

* Updated PluginSync to test against s3 + local storage

app/plugin_test.go

@@ -7,14 +7,18 @@ import (
 	"bytes"
 	"crypto/sha256"
 	"encoding/base64"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/gorilla/mux"
 	"github.com/mattermost/mattermost-server/model"
 	"github.com/mattermost/mattermost-server/plugin"
+	"github.com/mattermost/mattermost-server/utils/fileutils"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -312,3 +316,92 @@ func TestGetPluginStatuses(t *testing.T) {
 	require.Nil(t, err)
 	require.NotNil(t, pluginStatuses)
 }
+
+func TestPluginSync(t *testing.T) {
+	th := Setup(t).InitBasic()
+	defer th.TearDown()
+
+	testCases := []struct {
+		Description string
+		ConfigFunc  func(cfg *model.Config)
+	}{
+		{
+			"local",
+			func(cfg *model.Config) {
+				cfg.FileSettings.DriverName = model.NewString(model.IMAGE_DRIVER_LOCAL)
+			},
+		},
+		{
+			"s3",
+			func(cfg *model.Config) {
+				s3Host := os.Getenv("CI_MINIO_HOST")
+				if s3Host == "" {
+					s3Host = "dockerhost"
+				}
+
+				s3Port := os.Getenv("CI_MINIO_PORT")
+				if s3Port == "" {
+					s3Port = "9001"
+				}
+
+				s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
+				cfg.FileSettings.DriverName = model.NewString(model.IMAGE_DRIVER_S3)
+				cfg.FileSettings.AmazonS3AccessKeyId = model.NewString(model.MINIO_ACCESS_KEY)
+				cfg.FileSettings.AmazonS3SecretAccessKey = model.NewString(model.MINIO_SECRET_KEY)
+				cfg.FileSettings.AmazonS3Bucket = model.NewString(model.MINIO_BUCKET)
+				cfg.FileSettings.AmazonS3Endpoint = model.NewString(s3Endpoint)
+				cfg.FileSettings.AmazonS3Region = model.NewString("")
+				cfg.FileSettings.AmazonS3SSL = model.NewBool(false)
+
+			},
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.Description, func(t *testing.T) {
+			os.MkdirAll("./test-plugins", os.ModePerm)
+			defer os.RemoveAll("./test-plugins")
+
+			th.App.UpdateConfig(func(cfg *model.Config) {
+				*cfg.PluginSettings.Enable = true
+				*cfg.PluginSettings.Directory = "./test-plugins"
+				*cfg.PluginSettings.ClientDirectory = "./test-client-plugins"
+			})
+			th.App.UpdateConfig(testCase.ConfigFunc)
+
+			env, err := plugin.NewEnvironment(th.App.NewPluginAPI, "./test-plugins", "./test-client-plugins", th.App.Log)
+			require.NoError(t, err)
+			th.App.SetPluginsEnvironment(env)
+
+			// New bundle in the file store case
+			path, _ := fileutils.FindDir("tests")
+			fileReader, err := os.Open(filepath.Join(path, "testplugin.tar.gz"))
+			require.NoError(t, err)
+			defer fileReader.Close()
+
+			_, appErr := th.App.WriteFile(fileReader, th.App.getBundleStorePath("testplugin"))
+			checkNoError(t, appErr)
+
+			appErr = th.App.SyncPlugins()
+			checkNoError(t, appErr)
+
+			// Check if installed
+			pluginStatus, err := env.Statuses()
+			require.Nil(t, err)
+			require.True(t, len(pluginStatus) == 1)
+			require.Equal(t, pluginStatus[0].PluginId, "testplugin")
+
+			// Bundle removed from the file store case
+			appErr = th.App.RemoveFile(th.App.getBundleStorePath("testplugin"))
+			checkNoError(t, appErr)
+
+			appErr = th.App.SyncPlugins()
+			checkNoError(t, appErr)
+
+			// Check if removed
+			pluginStatus, err = env.Statuses()
+			require.Nil(t, err)
+			require.True(t, len(pluginStatus) == 0)
+		})
+	}
+}