MM-16261: Synchronize plugins in HA (#11657)

* MM-16272 - Synchronize plugins across cluster (#11611)

* MM-16272 - Synchronize plugins across cluster

* Adding a test

* MM-16272 - Fixed tests

* MM-16272 - PR feedback

* MM-16270 - Plugin Sync (#11615)

* Initial implementation for plugin synch with file store. WIP

* Removed ListAll implementation. Used ListDirectory and change localstore to be consistent and return all items (files and folders) from directory

* Refactored plugin filestore operations out of main install/remove plugin

* Fixing error handling details

* Changes to use structured logging

* More logging fixes

* Wording and comments improvements

* Error handling and control flow improvements

* Changed managed flag check to use os.stat

* Added file store plugin dir and filename consts

* Replaced FileRead to use a the FileReader in PluginSync

* Minor styling and PR feedback changes

* Minor error handling improvements

* Added unit test for SyncPlugins. Changed SyncPlugins to use plugins environment to list available plugins

* PR Feedback improvements

* Minor err handling fix

* Removing FileStorePath from PluginEventData (#11644)

* Fix plugin path (#11654)

* tweak path, logging

Fix an issue not finding the plugins folder in S3. Tweak logging messages to add additional clarity.

* Removing FileExists check when Syncing plugins. Updated localstore to not return an error when directory does not exist

* PR Feedback

* Install prepackaged plugins locally only (#11656)

* s/uninstall/remove

* Updated ClusterMessage comment

* Updated PluginSync to test against s3 + local storage

services/filesstore/filesstore_test.go

@@ -237,17 +237,29 @@ func (s *FileBackendTestSuite) TestListDirectory() {
 	path1 := "19700101/" + model.NewId()
 	path2 := "19800101/" + model.NewId()
 
+	paths, err := s.backend.ListDirectory("19700101")
+	s.Nil(err)
+	s.Len(*paths, 0)
+
 	written, err := s.backend.WriteFile(bytes.NewReader(b), path1)
 	s.Nil(err)
 	s.EqualValues(len(b), written, "expected given number of bytes to have been written")
-	defer s.backend.RemoveFile(path1)
 
 	written, err = s.backend.WriteFile(bytes.NewReader(b), path2)
 	s.Nil(err)
 	s.EqualValues(len(b), written, "expected given number of bytes to have been written")
-	defer s.backend.RemoveFile(path2)
 
-	paths, err := s.backend.ListDirectory("")
+	paths, err = s.backend.ListDirectory("19700101")
+	s.Nil(err)
+	s.Len(*paths, 1)
+	s.Equal(path1, (*paths)[0])
+
+	paths, err = s.backend.ListDirectory("19700101/")
+	s.Nil(err)
+	s.Len(*paths, 1)
+	s.Equal(path1, (*paths)[0])
+
+	paths, err = s.backend.ListDirectory("")
 	s.Nil(err)
 
 	found1 := false
@@ -261,6 +273,9 @@ func (s *FileBackendTestSuite) TestListDirectory() {
 	}
 	s.True(found1)
 	s.True(found2)
+
+	s.backend.RemoveFile(path1)
+	s.backend.RemoveFile(path2)
 }
 
 func (s *FileBackendTestSuite) TestRemoveDirectory() {

services/filesstore/localstore.go

@@ -114,12 +114,13 @@ func (b *LocalFileBackend) ListDirectory(path string) (*[]string, *model.AppErro
 	var paths []string
 	fileInfos, err := ioutil.ReadDir(filepath.Join(b.directory, path))
 	if err != nil {
+		if os.IsNotExist(err) {
+			return &paths, nil
+		}
 		return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError)
 	}
 	for _, fileInfo := range fileInfos {
-		if fileInfo.IsDir() {
-			paths = append(paths, filepath.Join(path, fileInfo.Name()))
-		}
+		paths = append(paths, filepath.Join(path, fileInfo.Name()))
 	}
 	return &paths, nil
 }

services/filesstore/s3store.go

@@ -238,9 +238,13 @@ func (b *S3FileBackend) ListDirectory(path string) (*[]string, *model.AppError)
 	}
 
 	doneCh := make(chan struct{})
-
 	defer close(doneCh)
 
+	if !strings.HasSuffix(path, "/") && len(path) > 0 {
+		// s3Clnt returns only the path itself when "/" is not present
+		// appending "/" to make it consistent across all filesstores
+		path = path + "/"
+	}
 	for object := range s3Clnt.ListObjects(b.bucket, path, false, doneCh) {
 		if object.Err != nil {
 			return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, object.Err.Error(), http.StatusInternalServerError)