diff --git a/api4/plugin_test.go b/api4/plugin_test.go index 91512a04bc..e4757c2855 100644 --- a/api4/plugin_test.go +++ b/api4/plugin_test.go @@ -25,7 +25,6 @@ import ( "github.com/mattermost/mattermost-server/v5/model" "github.com/mattermost/mattermost-server/v5/plugin" "github.com/mattermost/mattermost-server/v5/testlib" - "github.com/mattermost/mattermost-server/v5/utils" "github.com/mattermost/mattermost-server/v5/utils/fileutils" ) @@ -1552,9 +1551,9 @@ func TestInstallMarketplacePlugin(t *testing.T) { prepackagedPluginsDir, found := fileutils.FindDir(prepackagedPluginsDir) require.True(t, found, "failed to find prepackaged plugins directory") - err = utils.CopyFile(filepath.Join(path, "testplugin.tar.gz"), filepath.Join(prepackagedPluginsDir, "testplugin.tar.gz")) + err = testlib.CopyFile(filepath.Join(path, "testplugin.tar.gz"), filepath.Join(prepackagedPluginsDir, "testplugin.tar.gz")) require.NoError(t, err) - err = utils.CopyFile(filepath.Join(path, "testplugin.tar.gz.asc"), filepath.Join(prepackagedPluginsDir, "testplugin.tar.gz.sig")) + err = testlib.CopyFile(filepath.Join(path, "testplugin.tar.gz.asc"), filepath.Join(prepackagedPluginsDir, "testplugin.tar.gz.sig")) require.NoError(t, err) th2 := SetupConfig(t, func(cfg *model.Config) { @@ -1687,7 +1686,7 @@ func TestInstallMarketplacePlugin(t *testing.T) { prepackagedPluginsDir, found := fileutils.FindDir(prepackagedPluginsDir) require.True(t, found, "failed to find prepackaged plugins directory") - err = utils.CopyFile(filepath.Join(path, "testplugin.tar.gz"), filepath.Join(prepackagedPluginsDir, "testplugin.tar.gz")) + err = testlib.CopyFile(filepath.Join(path, "testplugin.tar.gz"), filepath.Join(prepackagedPluginsDir, "testplugin.tar.gz")) require.NoError(t, err) th := SetupConfig(t, func(cfg *model.Config) { diff --git a/app/file.go b/app/file.go index 442a31e033..6e69c8cbf6 100644 --- a/app/file.go +++ b/app/file.go @@ -80,7 +80,8 @@ func (a *App) FileBackend() (filesstore.FileBackend, *model.AppError) { } func (a *App) CheckMandatoryS3Fields(settings *model.FileSettings) *model.AppError { - err := filesstore.CheckMandatoryS3Fields(settings) + fileBackendSettings := settings.ToFileBackendSettings(false) + err := fileBackendSettings.CheckMandatoryS3Fields() if err != nil { return model.NewAppError("CheckMandatoryS3Fields", "api.admin.test_s3.missing_s3_bucket", nil, err.Error(), http.StatusBadRequest) } @@ -101,7 +102,7 @@ func (a *App) TestFilesStoreConnection() *model.AppError { func (a *App) TestFilesStoreConnectionWithConfig(cfg *model.FileSettings) *model.AppError { license := a.Srv().License() - backend, err := filesstore.NewFileBackend(cfg, license != nil && *license.Features.Compliance) + backend, err := filesstore.NewFileBackend(cfg.ToFileBackendSettings(license != nil && *license.Features.Compliance)) if err != nil { return model.NewAppError("FileBackend", "api.file.no_driver.app_error", nil, err.Error(), http.StatusInternalServerError) } diff --git a/app/plugin_test.go b/app/plugin_test.go index 4f6c4be75f..8a3b0fef23 100644 --- a/app/plugin_test.go +++ b/app/plugin_test.go @@ -24,7 +24,6 @@ import ( "github.com/mattermost/mattermost-server/v5/model" "github.com/mattermost/mattermost-server/v5/plugin" "github.com/mattermost/mattermost-server/v5/testlib" - "github.com/mattermost/mattermost-server/v5/utils" "github.com/mattermost/mattermost-server/v5/utils/fileutils" ) @@ -746,7 +745,7 @@ func TestProcessPrepackagedPlugins(t *testing.T) { require.True(t, found, "failed to find prepackaged plugins directory") testPluginPath := filepath.Join(testsPath, "testplugin.tar.gz") - fileErr = utils.CopyFile(testPluginPath, filepath.Join(prepackagedPluginsDir, "testplugin.tar.gz")) + fileErr = testlib.CopyFile(testPluginPath, filepath.Join(prepackagedPluginsDir, "testplugin.tar.gz")) require.NoError(t, fileErr) t.Run("automatic, enabled plugin, no signature", func(t *testing.T) { @@ -820,16 +819,16 @@ func TestProcessPrepackagedPlugins(t *testing.T) { // Add signature testPluginSignaturePath := filepath.Join(testsPath, "testplugin.tar.gz.sig") - err := utils.CopyFile(testPluginSignaturePath, filepath.Join(prepackagedPluginsDir, "testplugin.tar.gz.sig")) + err := testlib.CopyFile(testPluginSignaturePath, filepath.Join(prepackagedPluginsDir, "testplugin.tar.gz.sig")) require.NoError(t, err) // Add second plugin testPlugin2Path := filepath.Join(testsPath, "testplugin2.tar.gz") - err = utils.CopyFile(testPlugin2Path, filepath.Join(prepackagedPluginsDir, "testplugin2.tar.gz")) + err = testlib.CopyFile(testPlugin2Path, filepath.Join(prepackagedPluginsDir, "testplugin2.tar.gz")) require.NoError(t, err) testPlugin2SignaturePath := filepath.Join(testsPath, "testplugin2.tar.gz.sig") - err = utils.CopyFile(testPlugin2SignaturePath, filepath.Join(prepackagedPluginsDir, "testplugin2.tar.gz.sig")) + err = testlib.CopyFile(testPlugin2SignaturePath, filepath.Join(prepackagedPluginsDir, "testplugin2.tar.gz.sig")) require.NoError(t, err) plugins := th.App.processPrepackagedPlugins(prepackagedPluginsDir) @@ -855,7 +854,7 @@ func TestProcessPrepackagedPlugins(t *testing.T) { // Add signature testPluginSignaturePath := filepath.Join(testsPath, "testplugin.tar.gz.sig") - err := utils.CopyFile(testPluginSignaturePath, filepath.Join(prepackagedPluginsDir, "testplugin.tar.gz.sig")) + err := testlib.CopyFile(testPluginSignaturePath, filepath.Join(prepackagedPluginsDir, "testplugin.tar.gz.sig")) require.NoError(t, err) // Install first plugin and enable @@ -874,11 +873,11 @@ func TestProcessPrepackagedPlugins(t *testing.T) { // Add second plugin testPlugin2Path := filepath.Join(testsPath, "testplugin2.tar.gz") - err = utils.CopyFile(testPlugin2Path, filepath.Join(prepackagedPluginsDir, "testplugin2.tar.gz")) + err = testlib.CopyFile(testPlugin2Path, filepath.Join(prepackagedPluginsDir, "testplugin2.tar.gz")) require.NoError(t, err) testPlugin2SignaturePath := filepath.Join(testsPath, "testplugin2.tar.gz.sig") - err = utils.CopyFile(testPlugin2SignaturePath, filepath.Join(prepackagedPluginsDir, "testplugin2.tar.gz.sig")) + err = testlib.CopyFile(testPlugin2SignaturePath, filepath.Join(prepackagedPluginsDir, "testplugin2.tar.gz.sig")) require.NoError(t, err) plugins := th.App.processPrepackagedPlugins(prepackagedPluginsDir) @@ -911,11 +910,11 @@ func TestProcessPrepackagedPlugins(t *testing.T) { env := th.App.GetPluginsEnvironment() testPlugin2Path := filepath.Join(testsPath, "testplugin2.tar.gz") - err := utils.CopyFile(testPlugin2Path, filepath.Join(prepackagedPluginsDir, "testplugin2.tar.gz")) + err := testlib.CopyFile(testPlugin2Path, filepath.Join(prepackagedPluginsDir, "testplugin2.tar.gz")) require.NoError(t, err) testPlugin2SignaturePath := filepath.Join(testsPath, "testplugin2.tar.gz.sig") - err = utils.CopyFile(testPlugin2SignaturePath, filepath.Join(prepackagedPluginsDir, "testplugin2.tar.gz.sig")) + err = testlib.CopyFile(testPlugin2SignaturePath, filepath.Join(prepackagedPluginsDir, "testplugin2.tar.gz.sig")) require.NoError(t, err) plugins := th.App.processPrepackagedPlugins(prepackagedPluginsDir) diff --git a/app/server.go b/app/server.go index 6e25b06332..37408eee6d 100644 --- a/app/server.go +++ b/app/server.go @@ -1568,7 +1568,7 @@ func (s *Server) stopSearchEngine() { func (s *Server) FileBackend() (filesstore.FileBackend, *model.AppError) { license := s.License() - backend, err := filesstore.NewFileBackend(&s.Config().FileSettings, license != nil && *license.Features.Compliance) + backend, err := filesstore.NewFileBackend(s.Config().FileSettings.ToFileBackendSettings(license != nil && *license.Features.Compliance)) if err != nil { return nil, model.NewAppError("FileBackend", "api.file.no_driver.app_error", nil, err.Error(), http.StatusInternalServerError) } diff --git a/model/config.go b/model/config.go index 535fba9c53..0039e2144d 100644 --- a/model/config.go +++ b/model/config.go @@ -21,6 +21,7 @@ import ( "github.com/mattermost/ldap" "github.com/mattermost/mattermost-server/v5/mlog" + "github.com/mattermost/mattermost-server/v5/services/filesstore" ) const ( @@ -1452,6 +1453,28 @@ func (s *FileSettings) SetDefaults(isUpdate bool) { } } +func (s *FileSettings) ToFileBackendSettings(enableComplianceFeature bool) filesstore.FileBackendSettings { + if *s.DriverName == IMAGE_DRIVER_LOCAL { + return filesstore.FileBackendSettings{ + DriverName: *s.DriverName, + Directory: *s.Directory, + } + } + return filesstore.FileBackendSettings{ + DriverName: *s.DriverName, + AmazonS3AccessKeyId: *s.AmazonS3AccessKeyId, + AmazonS3SecretAccessKey: *s.AmazonS3SecretAccessKey, + AmazonS3Bucket: *s.AmazonS3Bucket, + AmazonS3PathPrefix: *s.AmazonS3PathPrefix, + AmazonS3Region: *s.AmazonS3Region, + AmazonS3Endpoint: *s.AmazonS3Endpoint, + AmazonS3SSL: s.AmazonS3SSL == nil || *s.AmazonS3SSL, + AmazonS3SignV2: s.AmazonS3SignV2 != nil && *s.AmazonS3SignV2, + AmazonS3SSE: s.AmazonS3SSE != nil && *s.AmazonS3SSE && enableComplianceFeature, + AmazonS3Trace: s.AmazonS3Trace != nil && *s.AmazonS3Trace, + } +} + type EmailSettings struct { EnableSignUpWithEmail *bool `access:"authentication"` EnableSignInWithEmail *bool `access:"authentication"` diff --git a/model/file.go b/model/file.go index 9f76bac174..d2cb8f34d3 100644 --- a/model/file.go +++ b/model/file.go @@ -12,11 +12,6 @@ const ( MaxImageSize = int64(6048 * 4032) // 24 megapixels, roughly 36MB as a raw image ) -var ( - IMAGE_EXTENSIONS = [7]string{".jpg", ".jpeg", ".gif", ".bmp", ".png", ".tiff", "tif"} - IMAGE_MIME_TYPES = map[string]string{".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".gif": "image/gif", ".bmp": "image/bmp", ".png": "image/png", ".tiff": "image/tiff", ".tif": "image/tif"} -) - type FileUploadResponse struct { FileInfos []*FileInfo `json:"file_infos"` ClientIds []string `json:"client_ids"` diff --git a/model/utils.go b/model/utils.go index 80a98fa0f9..3509bec561 100644 --- a/model/utils.go +++ b/model/utils.go @@ -484,24 +484,6 @@ func ParseHashtags(text string) (string, string) { return strings.TrimSpace(hashtagString), strings.TrimSpace(plainString) } -func IsFileExtImage(ext string) bool { - ext = strings.ToLower(ext) - for _, imgExt := range IMAGE_EXTENSIONS { - if ext == imgExt { - return true - } - } - return false -} - -func GetImageMimeType(ext string) string { - ext = strings.ToLower(ext) - if IMAGE_MIME_TYPES[ext] == "" { - return "image" - } - return IMAGE_MIME_TYPES[ext] -} - func ClearMentionTags(post string) string { post = strings.Replace(post, "", "", -1) post = strings.Replace(post, "", "", -1) diff --git a/services/filesstore/filesstore.go b/services/filesstore/filesstore.go index 251b36957f..02541b05e6 100644 --- a/services/filesstore/filesstore.go +++ b/services/filesstore/filesstore.go @@ -8,8 +8,11 @@ import ( "time" "github.com/pkg/errors" +) - "github.com/mattermost/mattermost-server/v5/model" +const ( + driverS3 = "amazons3" + driverLocal = "local" ) type ReadCloseSeeker interface { @@ -35,17 +38,45 @@ type FileBackend interface { RemoveDirectory(path string) error } -func NewFileBackend(settings *model.FileSettings, enableComplianceFeatures bool) (FileBackend, error) { - switch *settings.DriverName { - case model.IMAGE_DRIVER_S3: - backend, err := NewS3FileBackend(settings, enableComplianceFeatures) +type FileBackendSettings struct { + DriverName string + Directory string + AmazonS3AccessKeyId string + AmazonS3SecretAccessKey string + AmazonS3Bucket string + AmazonS3PathPrefix string + AmazonS3Region string + AmazonS3Endpoint string + AmazonS3SSL bool + AmazonS3SignV2 bool + AmazonS3SSE bool + AmazonS3Trace bool +} + +func (settings *FileBackendSettings) CheckMandatoryS3Fields() error { + if settings.AmazonS3Bucket == "" { + return errors.New("missing s3 bucket settings") + } + + // if S3 endpoint is not set call the set defaults to set that + if settings.AmazonS3Endpoint == "" { + settings.AmazonS3Endpoint = "s3.amazonaws.com" + } + + return nil +} + +func NewFileBackend(settings FileBackendSettings) (FileBackend, error) { + switch settings.DriverName { + case driverS3: + backend, err := NewS3FileBackend(settings) if err != nil { return nil, errors.Wrap(err, "unable to connect to the s3 backend") } return backend, nil - case model.IMAGE_DRIVER_LOCAL: + case driverLocal: return &LocalFileBackend{ - directory: *settings.Directory, + directory: settings.Directory, }, nil } return nil, errors.New("no valid filestorage driver found") diff --git a/services/filesstore/filesstore_test.go b/services/filesstore/filesstore_test.go index 92193f7ee9..2e49d1c5c4 100644 --- a/services/filesstore/filesstore_test.go +++ b/services/filesstore/filesstore_test.go @@ -14,16 +14,19 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/xtgo/uuid" "github.com/mattermost/mattermost-server/v5/mlog" - "github.com/mattermost/mattermost-server/v5/model" - "github.com/mattermost/mattermost-server/v5/utils" ) +func randomString() string { + return uuid.NewRandom().String() +} + type FileBackendTestSuite struct { suite.Suite - settings model.FileSettings + settings FileBackendSettings backend FileBackend } @@ -42,9 +45,9 @@ func TestLocalFileBackendTestSuite(t *testing.T) { defer os.RemoveAll(dir) suite.Run(t, &FileBackendTestSuite{ - settings: model.FileSettings{ - DriverName: model.NewString(model.IMAGE_DRIVER_LOCAL), - Directory: &dir, + settings: FileBackendSettings{ + DriverName: driverLocal, + Directory: dir, }, }) } @@ -71,24 +74,22 @@ func runBackendTest(t *testing.T, encrypt bool) { s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port) suite.Run(t, &FileBackendTestSuite{ - settings: model.FileSettings{ - DriverName: model.NewString(model.IMAGE_DRIVER_S3), - AmazonS3AccessKeyId: model.NewString(model.MINIO_ACCESS_KEY), - AmazonS3SecretAccessKey: model.NewString(model.MINIO_SECRET_KEY), - AmazonS3Bucket: model.NewString(model.MINIO_BUCKET), - AmazonS3Region: model.NewString(""), - AmazonS3Endpoint: model.NewString(s3Endpoint), - AmazonS3PathPrefix: model.NewString(""), - AmazonS3SSL: model.NewBool(false), - AmazonS3SSE: model.NewBool(encrypt), + settings: FileBackendSettings{ + DriverName: driverS3, + AmazonS3AccessKeyId: "minioaccesskey", + AmazonS3SecretAccessKey: "miniosecretkey", + AmazonS3Bucket: "mattermost-test", + AmazonS3Region: "", + AmazonS3Endpoint: s3Endpoint, + AmazonS3PathPrefix: "", + AmazonS3SSL: false, + AmazonS3SSE: encrypt, }, }) } func (s *FileBackendTestSuite) SetupTest() { - utils.TranslationsPreInit() - - backend, err := NewFileBackend(&s.settings, true) + backend, err := NewFileBackend(s.settings) require.NoError(s.T(), err) s.backend = backend @@ -102,7 +103,7 @@ func (s *FileBackendTestSuite) TestConnection() { func (s *FileBackendTestSuite) TestReadWriteFile() { b := []byte("test") - path := "tests/" + model.NewId() + path := "tests/" + randomString() written, err := s.backend.WriteFile(bytes.NewReader(b), path) s.Nil(err) @@ -118,7 +119,7 @@ func (s *FileBackendTestSuite) TestReadWriteFile() { func (s *FileBackendTestSuite) TestReadWriteFileImage() { b := []byte("testimage") - path := "tests/" + model.NewId() + ".png" + path := "tests/" + randomString() + ".png" written, err := s.backend.WriteFile(bytes.NewReader(b), path) s.Nil(err) @@ -134,7 +135,7 @@ func (s *FileBackendTestSuite) TestReadWriteFileImage() { func (s *FileBackendTestSuite) TestFileExists() { b := []byte("testimage") - path := "tests/" + model.NewId() + ".png" + path := "tests/" + randomString() + ".png" _, err := s.backend.WriteFile(bytes.NewReader(b), path) s.Nil(err) @@ -151,8 +152,8 @@ func (s *FileBackendTestSuite) TestFileExists() { func (s *FileBackendTestSuite) TestCopyFile() { b := []byte("test") - path1 := "tests/" + model.NewId() - path2 := "tests/" + model.NewId() + path1 := "tests/" + randomString() + path2 := "tests/" + randomString() written, err := s.backend.WriteFile(bytes.NewReader(b), path1) s.Nil(err) @@ -175,8 +176,8 @@ func (s *FileBackendTestSuite) TestCopyFile() { func (s *FileBackendTestSuite) TestCopyFileToDirectoryThatDoesntExist() { b := []byte("test") - path1 := "tests/" + model.NewId() - path2 := "tests/newdirectory/" + model.NewId() + path1 := "tests/" + randomString() + path2 := "tests/newdirectory/" + randomString() written, err := s.backend.WriteFile(bytes.NewReader(b), path1) s.Nil(err) @@ -196,8 +197,8 @@ func (s *FileBackendTestSuite) TestCopyFileToDirectoryThatDoesntExist() { func (s *FileBackendTestSuite) TestMoveFile() { b := []byte("test") - path1 := "tests/" + model.NewId() - path2 := "tests/" + model.NewId() + path1 := "tests/" + randomString() + path2 := "tests/" + randomString() written, err := s.backend.WriteFile(bytes.NewReader(b), path1) s.Nil(err) @@ -218,7 +219,7 @@ func (s *FileBackendTestSuite) TestMoveFile() { func (s *FileBackendTestSuite) TestRemoveFile() { b := []byte("test") - path := "tests/" + model.NewId() + path := "tests/" + randomString() written, err := s.backend.WriteFile(bytes.NewReader(b), path) s.Nil(err) @@ -245,8 +246,8 @@ func (s *FileBackendTestSuite) TestRemoveFile() { func (s *FileBackendTestSuite) TestListDirectory() { b := []byte("test") - path1 := "19700101/" + model.NewId() - path2 := "19800101/" + model.NewId() + path1 := "19700101/" + randomString() + path2 := "19800101/" + randomString() paths, err := s.backend.ListDirectory("19700101") s.Nil(err) @@ -316,7 +317,7 @@ func (s *FileBackendTestSuite) TestRemoveDirectory() { func (s *FileBackendTestSuite) TestAppendFile() { s.Run("should fail if target file is missing", func() { - path := "tests/" + model.NewId() + path := "tests/" + randomString() b := make([]byte, 1024) written, err := s.backend.AppendFile(bytes.NewReader(b), path) s.Error(err) @@ -330,7 +331,7 @@ func (s *FileBackendTestSuite) TestAppendFile() { for i := range b { b[i] = 'A' } - path := "tests/" + model.NewId() + path := "tests/" + randomString() written, err := s.backend.WriteFile(bytes.NewReader(b), path) s.Nil(err) @@ -376,7 +377,7 @@ func (s *FileBackendTestSuite) TestFileSize() { s.Run("valid file", func() { data := make([]byte, rand.Intn(1024*1024)+1) - path := "tests/" + model.NewId() + path := "tests/" + randomString() written, err := s.backend.WriteFile(bytes.NewReader(data), path) s.Nil(err) @@ -397,7 +398,7 @@ func (s *FileBackendTestSuite) TestFileModTime() { }) s.Run("valid file", func() { - path := "tests/" + model.NewId() + path := "tests/" + randomString() data := []byte("some data") written, err := s.backend.WriteFile(bytes.NewReader(data), path) @@ -412,7 +413,7 @@ func (s *FileBackendTestSuite) TestFileModTime() { // We wait 1 second so that the times will differ enough to be testable. time.Sleep(1 * time.Second) - path2 := "tests/" + model.NewId() + path2 := "tests/" + randomString() written, err = s.backend.WriteFile(bytes.NewReader(data), path2) s.Nil(err) s.EqualValues(len(data), written) @@ -426,27 +427,25 @@ func (s *FileBackendTestSuite) TestFileModTime() { } func BenchmarkS3WriteFile(b *testing.B) { - utils.TranslationsPreInit() - - settings := &model.FileSettings{ - DriverName: model.NewString(model.IMAGE_DRIVER_S3), - AmazonS3AccessKeyId: model.NewString(model.MINIO_ACCESS_KEY), - AmazonS3SecretAccessKey: model.NewString(model.MINIO_SECRET_KEY), - AmazonS3Bucket: model.NewString(model.MINIO_BUCKET), - AmazonS3Region: model.NewString(""), - AmazonS3Endpoint: model.NewString("localhost:9000"), - AmazonS3PathPrefix: model.NewString(""), - AmazonS3SSL: model.NewBool(false), - AmazonS3SSE: model.NewBool(false), + settings := FileBackendSettings{ + DriverName: driverS3, + AmazonS3AccessKeyId: "minioaccesskey", + AmazonS3SecretAccessKey: "miniosecretkey", + AmazonS3Bucket: "mattermost-test", + AmazonS3Region: "", + AmazonS3Endpoint: "localhost:9000", + AmazonS3PathPrefix: "", + AmazonS3SSL: false, + AmazonS3SSE: false, } - backend, err := NewFileBackend(settings, true) + backend, err := NewFileBackend(settings) require.NoError(b, err) // This is needed to create the bucket if it doesn't exist. require.NoError(b, backend.TestConnection()) - path := "tests/" + model.NewId() + path := "tests/" + randomString() size := 1 * 1024 * 1024 data := make([]byte, size) diff --git a/services/filesstore/localstore.go b/services/filesstore/localstore.go index 76c68d8ad0..9c37e097dd 100644 --- a/services/filesstore/localstore.go +++ b/services/filesstore/localstore.go @@ -14,7 +14,6 @@ import ( "github.com/pkg/errors" "github.com/mattermost/mattermost-server/v5/mlog" - "github.com/mattermost/mattermost-server/v5/utils" ) const ( @@ -25,6 +24,51 @@ type LocalFileBackend struct { directory string } +// copyFile will copy a file from src path to dst path. +// Overwrites any existing files at dst. +// Permissions are copied from file at src to the new file at dst. +func copyFile(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + + if err = os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil { + return + } + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + if e := out.Close(); e != nil { + err = e + } + }() + + _, err = io.Copy(out, in) + if err != nil { + return + } + + err = out.Sync() + if err != nil { + return + } + + stat, err := os.Stat(src) + if err != nil { + return + } + err = os.Chmod(dst, stat.Mode()) + if err != nil { + return + } + + return +} + func (b *LocalFileBackend) TestConnection() error { f := bytes.NewReader([]byte("testingwrite")) if _, err := writeFileLocally(f, filepath.Join(b.directory, TestFilePath)); err != nil { @@ -81,7 +125,7 @@ func (b *LocalFileBackend) FileModTime(path string) (time.Time, error) { } func (b *LocalFileBackend) CopyFile(oldPath, newPath string) error { - if err := utils.CopyFile(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil { + if err := copyFile(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil { return errors.Wrapf(err, "unable to copy file from %s to %s", oldPath, newPath) } return nil diff --git a/services/filesstore/s3store.go b/services/filesstore/s3store.go index 3b96d1daa1..eefc3e0cca 100644 --- a/services/filesstore/s3store.go +++ b/services/filesstore/s3store.go @@ -18,7 +18,6 @@ import ( "github.com/pkg/errors" "github.com/mattermost/mattermost-server/v5/mlog" - "github.com/mattermost/mattermost-server/v5/model" ) // S3FileBackend contains all necessary information to communicate with @@ -42,19 +41,37 @@ const ( bucketNotFound = "NoSuchBucket" ) +var ( + imageExtensions = map[string]bool{".jpg": true, ".jpeg": true, ".gif": true, ".bmp": true, ".png": true, ".tiff": true, "tif": true} + imageMimeTypes = map[string]string{".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".gif": "image/gif", ".bmp": "image/bmp", ".png": "image/png", ".tiff": "image/tiff", ".tif": "image/tif"} +) + +func isFileExtImage(ext string) bool { + ext = strings.ToLower(ext) + return imageExtensions[ext] +} + +func getImageMimeType(ext string) string { + ext = strings.ToLower(ext) + if imageMimeTypes[ext] == "" { + return "image" + } + return imageMimeTypes[ext] +} + // NewS3FileBackend returns an instance of an S3FileBackend. -func NewS3FileBackend(settings *model.FileSettings, enableComplianceFeatures bool) (*S3FileBackend, error) { +func NewS3FileBackend(settings FileBackendSettings) (*S3FileBackend, error) { backend := &S3FileBackend{ - endpoint: *settings.AmazonS3Endpoint, - accessKey: *settings.AmazonS3AccessKeyId, - secretKey: *settings.AmazonS3SecretAccessKey, - secure: settings.AmazonS3SSL == nil || *settings.AmazonS3SSL, - signV2: settings.AmazonS3SignV2 != nil && *settings.AmazonS3SignV2, - region: *settings.AmazonS3Region, - bucket: *settings.AmazonS3Bucket, - pathPrefix: *settings.AmazonS3PathPrefix, - encrypt: settings.AmazonS3SSE != nil && *settings.AmazonS3SSE && enableComplianceFeatures, - trace: settings.AmazonS3Trace != nil && *settings.AmazonS3Trace, + endpoint: settings.AmazonS3Endpoint, + accessKey: settings.AmazonS3AccessKeyId, + secretKey: settings.AmazonS3SecretAccessKey, + secure: settings.AmazonS3SSL, + signV2: settings.AmazonS3SignV2, + region: settings.AmazonS3Region, + bucket: settings.AmazonS3Bucket, + pathPrefix: settings.AmazonS3PathPrefix, + encrypt: settings.AmazonS3SSE, + trace: settings.AmazonS3Trace, } cli, err := backend.s3New() if err != nil { @@ -264,8 +281,8 @@ func (b *S3FileBackend) MoveFile(oldPath, newPath string) error { func (b *S3FileBackend) WriteFile(fr io.Reader, path string) (int64, error) { var contentType string path = filepath.Join(b.pathPrefix, path) - if ext := filepath.Ext(path); model.IsFileExtImage(ext) { - contentType = model.GetImageMimeType(ext) + if ext := filepath.Ext(path); isFileExtImage(ext) { + contentType = getImageMimeType(ext) } else { contentType = "binary/octet-stream" } @@ -286,8 +303,8 @@ func (b *S3FileBackend) AppendFile(fr io.Reader, path string) (int64, error) { } var contentType string - if ext := filepath.Ext(fp); model.IsFileExtImage(ext) { - contentType = model.GetImageMimeType(ext) + if ext := filepath.Ext(fp); isFileExtImage(ext) { + contentType = getImageMimeType(ext) } else { contentType = "binary/octet-stream" } @@ -406,16 +423,3 @@ func s3PutOptions(encrypted bool, contentType string) s3.PutObjectOptions { return options } - -func CheckMandatoryS3Fields(settings *model.FileSettings) error { - if settings.AmazonS3Bucket == nil || *settings.AmazonS3Bucket == "" { - return errors.New("missing s3 bucket settings") - } - - // if S3 endpoint is not set call the set defaults to set that - if settings.AmazonS3Endpoint == nil || *settings.AmazonS3Endpoint == "" { - settings.SetDefaults(true) - } - - return nil -} diff --git a/services/filesstore/s3store_test.go b/services/filesstore/s3store_test.go index 51e37b3ce8..5bcb2a3db7 100644 --- a/services/filesstore/s3store_test.go +++ b/services/filesstore/s3store_test.go @@ -7,24 +7,22 @@ import ( "testing" "github.com/stretchr/testify/require" - - "github.com/mattermost/mattermost-server/v5/model" ) func TestCheckMandatoryS3Fields(t *testing.T) { - cfg := model.FileSettings{} + cfg := FileBackendSettings{} - err := CheckMandatoryS3Fields(&cfg) + err := cfg.CheckMandatoryS3Fields() require.Error(t, err) require.Equal(t, err.Error(), "missing s3 bucket settings", "should've failed with missing s3 bucket") - cfg.AmazonS3Bucket = model.NewString("test-mm") - err = CheckMandatoryS3Fields(&cfg) + cfg.AmazonS3Bucket = "test-mm" + err = cfg.CheckMandatoryS3Fields() require.NoError(t, err) - cfg.AmazonS3Endpoint = model.NewString("") - err = CheckMandatoryS3Fields(&cfg) - + cfg.AmazonS3Endpoint = "" + err = cfg.CheckMandatoryS3Fields() require.NoError(t, err) - require.Equal(t, *cfg.AmazonS3Endpoint, "s3.amazonaws.com", "should've set the endpoint to the default") + + require.Equal(t, "s3.amazonaws.com", cfg.AmazonS3Endpoint, "should've set the endpoint to the default") } diff --git a/services/mailservice/mail.go b/services/mailservice/mail.go index 29943be434..4de64c9d42 100644 --- a/services/mailservice/mail.go +++ b/services/mailservice/mail.go @@ -295,7 +295,7 @@ func sendMailUsingConfigAdvanced(mail mailData, config *model.Config, enableComp defer c.Quit() defer c.Close() - fileBackend, nErr := filesstore.NewFileBackend(&config.FileSettings, enableComplianceFeatures) + fileBackend, nErr := filesstore.NewFileBackend(config.FileSettings.ToFileBackendSettings(enableComplianceFeatures)) if nErr != nil { return errors.Wrap(nErr, "unable to initialize file backend") } diff --git a/services/mailservice/mail_test.go b/services/mailservice/mail_test.go index 98d4715f7a..82a2232ef9 100644 --- a/services/mailservice/mail_test.go +++ b/services/mailservice/mail_test.go @@ -225,7 +225,7 @@ func TestSendMailUsingConfigAdvanced(t *testing.T) { //Delete all the messages before check the sample email DeleteMailBox("test2@example.com") - fileBackend, err := filesstore.NewFileBackend(&cfg.FileSettings, true) + fileBackend, err := filesstore.NewFileBackend(cfg.FileSettings.ToFileBackendSettings(true)) assert.NoError(t, err) // create two files with the same name that will both be attached to the email @@ -398,7 +398,8 @@ func TestSendMail(t *testing.T) { DriverName: model.NewString(model.IMAGE_DRIVER_LOCAL), Directory: &dir, } - mockBackend, err := filesstore.NewFileBackend(&settings, true) + settings.SetDefaults(true) + mockBackend, err := filesstore.NewFileBackend(settings.ToFileBackendSettings(true)) require.NoError(t, err) mocm := &mockMailer{} diff --git a/testlib/resources.go b/testlib/resources.go index e3db1ad977..e51539f281 100644 --- a/testlib/resources.go +++ b/testlib/resources.go @@ -13,6 +13,7 @@ import ( "github.com/pkg/errors" "github.com/mattermost/mattermost-server/v5/model" + "github.com/mattermost/mattermost-server/v5/services/filesstore" "github.com/mattermost/mattermost-server/v5/utils" "github.com/mattermost/mattermost-server/v5/utils/fileutils" ) @@ -117,6 +118,17 @@ func getTestResourcesToSetup() []testResourceDetails { return testResourcesToSetup } +func CopyFile(src, dst string) error { + fileBackend, err := filesstore.NewFileBackend(filesstore.FileBackendSettings{DriverName: "local", Directory: ""}) + if err != nil { + return errors.Wrapf(err, "failed to copy file %s to %s", src, dst) + } + if err = fileBackend.CopyFile(src, dst); err != nil { + return errors.Wrapf(err, "failed to copy file %s to %s", src, dst) + } + return nil +} + func SetupTestResources() (string, error) { testResourcesToSetup := getTestResourcesToSetup() @@ -151,9 +163,8 @@ func SetupTestResources() (string, error) { if testResource.action == actionCopy { if testResource.resType == resourceTypeFile { - err = utils.CopyFile(testResource.src, resourceDestInTemp) - if err != nil { - return "", errors.Wrapf(err, "failed to copy file %s to %s", testResource.src, resourceDestInTemp) + if err = CopyFile(testResource.src, resourceDestInTemp); err != nil { + return "", err } } else if testResource.resType == resourceTypeFolder { err = utils.CopyDir(testResource.src, resourceDestInTemp)