MM-16261: Synchronize plugins in HA (#11657)

* MM-16272 - Synchronize plugins across cluster (#11611)

* MM-16272 - Synchronize plugins across cluster

* Adding a test

* MM-16272 - Fixed tests

* MM-16272 - PR feedback

* MM-16270 - Plugin Sync (#11615)

* Initial implementation for plugin synch with file store. WIP

* Removed ListAll implementation. Used ListDirectory and change localstore to be consistent and return all items (files and folders) from directory

* Refactored plugin filestore operations out of main install/remove plugin

* Fixing error handling details

* Changes to use structured logging

* More logging fixes

* Wording and comments improvements

* Error handling and control flow improvements

* Changed managed flag check to use os.stat

* Added file store plugin dir and filename consts

* Replaced FileRead to use a the FileReader in PluginSync

* Minor styling and PR feedback changes

* Minor error handling improvements

* Added unit test for SyncPlugins. Changed SyncPlugins to use plugins environment to list available plugins

* PR Feedback improvements

* Minor err handling fix

* Removing FileStorePath from PluginEventData (#11644)

* Fix plugin path (#11654)

* tweak path, logging

Fix an issue not finding the plugins folder in S3. Tweak logging messages to add additional clarity.

* Removing FileExists check when Syncing plugins. Updated localstore to not return an error when directory does not exist

* PR Feedback

* Install prepackaged plugins locally only (#11656)

* s/uninstall/remove

* Updated ClusterMessage comment

* Updated PluginSync to test against s3 + local storage

app/plugin.go

@@ -12,6 +12,7 @@ import (
 	"github.com/mattermost/mattermost-server/mlog"
 	"github.com/mattermost/mattermost-server/model"
 	"github.com/mattermost/mattermost-server/plugin"
+	"github.com/mattermost/mattermost-server/services/filesstore"
 	"github.com/mattermost/mattermost-server/utils/fileutils"
 )
 
@@ -146,6 +147,10 @@ func (a *App) InitPlugins(pluginDir, webappPluginDir string) {
 	}
 	a.SetPluginsEnvironment(env)
 
+	if err := a.SyncPlugins(); err != nil {
+		mlog.Error("Failed to sync plugins from the file store", mlog.Err(err))
+	}
+
 	prepackagedPluginsDir, found := fileutils.FindDir("prepackaged_plugins")
 	if found {
 		if err := filepath.Walk(prepackagedPluginsDir, func(walkPath string, info os.FileInfo, err error) error {
@@ -155,7 +160,7 @@ func (a *App) InitPlugins(pluginDir, webappPluginDir string) {
 
 			if fileReader, err := os.Open(walkPath); err != nil {
 				mlog.Error("Failed to open prepackaged plugin", mlog.Err(err), mlog.String("path", walkPath))
-			} else if _, err := a.InstallPlugin(fileReader, true); err != nil {
+			} else if _, err := a.installPluginLocally(fileReader, true); err != nil {
 				mlog.Error("Failed to unpack prepackaged plugin", mlog.Err(err), mlog.String("path", walkPath))
 			}
 
@@ -182,6 +187,71 @@ func (a *App) InitPlugins(pluginDir, webappPluginDir string) {
 	a.SyncPluginsActiveState()
 }
 
+// SyncPlugins synchronizes the plugins installed locally
+// with the plugin bundles available in the file store.
+func (a *App) SyncPlugins() *model.AppError {
+	mlog.Info("Syncing plugins from the file store")
+
+	pluginsEnvironment := a.GetPluginsEnvironment()
+	if pluginsEnvironment == nil {
+		return model.NewAppError("SyncPlugins", "app.plugin.disabled.app_error", nil, "", http.StatusNotImplemented)
+	}
+
+	availablePlugins, err := pluginsEnvironment.Available()
+	if err != nil {
+		return model.NewAppError("SyncPlugins", "app.plugin.sync.read_local_folder.app_error", nil, err.Error(), http.StatusInternalServerError)
+	}
+
+	for _, plugin := range availablePlugins {
+		pluginId := plugin.Manifest.Id
+
+		// Only handle managed plugins with .filestore flag file.
+		_, err := os.Stat(filepath.Join(*a.Config().PluginSettings.Directory, pluginId, managedPluginFileName))
+		if os.IsNotExist(err) {
+			mlog.Warn("Skipping sync for unmanaged plugin", mlog.String("plugin_id", pluginId))
+		} else if err != nil {
+			mlog.Error("Skipping sync for plugin after failure to check if managed", mlog.String("plugin_id", pluginId), mlog.Err(err))
+		} else {
+			mlog.Debug("Removing local installation of managed plugin before sync", mlog.String("plugin_id", pluginId))
+			if err := a.removePluginLocally(pluginId); err != nil {
+				mlog.Error("Failed to remove local installation of managed plugin before sync", mlog.String("plugin_id", pluginId), mlog.Err(err))
+			}
+		}
+	}
+
+	// Install plugins from the file store.
+	fileStorePaths, appErr := a.ListDirectory(fileStorePluginFolder)
+	if appErr != nil {
+		return model.NewAppError("SyncPlugins", "app.plugin.sync.list_filestore.app_error", nil, appErr.Error(), http.StatusInternalServerError)
+	}
+	if len(fileStorePaths) == 0 {
+		mlog.Info("Found no files in plugins file store")
+		return nil
+	}
+
+	for _, path := range fileStorePaths {
+		if !strings.HasSuffix(path, ".tar.gz") {
+			mlog.Warn("Ignoring non-plugin in file store", mlog.String("bundle", path))
+			continue
+		}
+
+		var reader filesstore.ReadCloseSeeker
+		reader, appErr = a.FileReader(path)
+		if appErr != nil {
+			mlog.Error("Failed to open plugin bundle from file store.", mlog.String("bundle", path), mlog.Err(appErr))
+			continue
+		}
+		defer reader.Close()
+
+		mlog.Info("Syncing plugin from file store", mlog.String("bundle", path))
+		if _, err := a.installPluginLocally(reader, true); err != nil {
+			mlog.Error("Failed to sync plugin from file store", mlog.String("bundle", path), mlog.Err(err))
+		}
+	}
+
+	return nil
+}
+
 func (a *App) ShutDownPlugins() {
 	a.Srv.PluginsLock.Lock()
 	pluginsEnvironment := a.Srv.PluginsEnvironment