diff --git a/api4/system_test.go b/api4/system_test.go index 0fbf5b540b..4da7f05bdd 100644 --- a/api4/system_test.go +++ b/api4/system_test.go @@ -404,6 +404,7 @@ func TestS3TestConnection(t *testing.T) { AmazonS3Bucket: model.NewString(""), AmazonS3Endpoint: model.NewString(s3Endpoint), AmazonS3Region: model.NewString(""), + AmazonS3PathPrefix: model.NewString(""), AmazonS3SSL: model.NewBool(false), }, } @@ -420,6 +421,7 @@ func TestS3TestConnection(t *testing.T) { // If this fails, check the test configuration to ensure minio is setup with the // `mattermost-test` bucket defined by model.MINIO_BUCKET. *config.FileSettings.AmazonS3Bucket = model.MINIO_BUCKET + config.FileSettings.AmazonS3PathPrefix = model.NewString("") *config.FileSettings.AmazonS3Region = "us-east-1" _, resp = th.SystemAdminClient.TestS3Connection(&config) CheckOKStatus(t, resp) diff --git a/app/plugin_test.go b/app/plugin_test.go index e4215f1852..6c81b2e59d 100644 --- a/app/plugin_test.go +++ b/app/plugin_test.go @@ -498,6 +498,7 @@ func TestPluginSync(t *testing.T) { cfg.FileSettings.AmazonS3AccessKeyId = model.NewString(model.MINIO_ACCESS_KEY) cfg.FileSettings.AmazonS3SecretAccessKey = model.NewString(model.MINIO_SECRET_KEY) cfg.FileSettings.AmazonS3Bucket = model.NewString(model.MINIO_BUCKET) + cfg.FileSettings.AmazonS3PathPrefix = model.NewString("") cfg.FileSettings.AmazonS3Endpoint = model.NewString(s3Endpoint) cfg.FileSettings.AmazonS3Region = model.NewString("") cfg.FileSettings.AmazonS3SSL = model.NewBool(false) diff --git a/model/config.go b/model/config.go index 4063ec35ca..ce3f933a01 100644 --- a/model/config.go +++ b/model/config.go @@ -1265,6 +1265,7 @@ type FileSettings struct { AmazonS3AccessKeyId *string `restricted:"true"` AmazonS3SecretAccessKey *string `restricted:"true"` AmazonS3Bucket *string `restricted:"true"` + AmazonS3PathPrefix *string `restricted:"true"` AmazonS3Region *string `restricted:"true"` AmazonS3Endpoint *string `restricted:"true"` AmazonS3SSL *bool `restricted:"true"` @@ -1329,6 +1330,10 @@ func (s *FileSettings) SetDefaults(isUpdate bool) { s.AmazonS3Bucket = NewString("") } + if s.AmazonS3PathPrefix == nil { + s.AmazonS3PathPrefix = NewString("") + } + if s.AmazonS3Region == nil { s.AmazonS3Region = NewString("") } diff --git a/services/filesstore/filesstore.go b/services/filesstore/filesstore.go index cc07969bba..e6222c948f 100644 --- a/services/filesstore/filesstore.go +++ b/services/filesstore/filesstore.go @@ -34,15 +34,16 @@ func NewFileBackend(settings *model.FileSettings, enableComplianceFeatures bool) switch *settings.DriverName { case model.IMAGE_DRIVER_S3: return &S3FileBackend{ - endpoint: *settings.AmazonS3Endpoint, - accessKey: *settings.AmazonS3AccessKeyId, - secretKey: *settings.AmazonS3SecretAccessKey, - secure: settings.AmazonS3SSL == nil || *settings.AmazonS3SSL, - signV2: settings.AmazonS3SignV2 != nil && *settings.AmazonS3SignV2, - region: *settings.AmazonS3Region, - bucket: *settings.AmazonS3Bucket, - encrypt: settings.AmazonS3SSE != nil && *settings.AmazonS3SSE && enableComplianceFeatures, - trace: settings.AmazonS3Trace != nil && *settings.AmazonS3Trace, + endpoint: *settings.AmazonS3Endpoint, + accessKey: *settings.AmazonS3AccessKeyId, + secretKey: *settings.AmazonS3SecretAccessKey, + secure: settings.AmazonS3SSL == nil || *settings.AmazonS3SSL, + signV2: settings.AmazonS3SignV2 != nil && *settings.AmazonS3SignV2, + region: *settings.AmazonS3Region, + bucket: *settings.AmazonS3Bucket, + pathPrefix: *settings.AmazonS3PathPrefix, + encrypt: settings.AmazonS3SSE != nil && *settings.AmazonS3SSE && enableComplianceFeatures, + trace: settings.AmazonS3Trace != nil && *settings.AmazonS3Trace, }, nil case model.IMAGE_DRIVER_LOCAL: return &LocalFileBackend{ diff --git a/services/filesstore/filesstore_test.go b/services/filesstore/filesstore_test.go index d25cb5e243..ff7654fa88 100644 --- a/services/filesstore/filesstore_test.go +++ b/services/filesstore/filesstore_test.go @@ -76,6 +76,7 @@ func runBackendTest(t *testing.T, encrypt bool) { AmazonS3Bucket: model.NewString(model.MINIO_BUCKET), AmazonS3Region: model.NewString(""), AmazonS3Endpoint: model.NewString(s3Endpoint), + AmazonS3PathPrefix: model.NewString(""), AmazonS3SSL: model.NewBool(false), AmazonS3SSE: model.NewBool(encrypt), }, diff --git a/services/filesstore/s3store.go b/services/filesstore/s3store.go index 307b8b0700..5e1cd6e8ae 100644 --- a/services/filesstore/s3store.go +++ b/services/filesstore/s3store.go @@ -21,15 +21,16 @@ import ( ) type S3FileBackend struct { - endpoint string - accessKey string - secretKey string - secure bool - signV2 bool - region string - bucket string - encrypt bool - trace bool + endpoint string + accessKey string + secretKey string + secure bool + signV2 bool + region string + bucket string + pathPrefix string + encrypt bool + trace bool } // Similar to s3.New() but allows initialization of signature v2 or signature v4 client. @@ -89,10 +90,13 @@ func (b *S3FileBackend) Reader(path string) (ReadCloseSeeker, *model.AppError) { if err != nil { return nil, model.NewAppError("Reader", "api.file.reader.s3.app_error", nil, err.Error(), http.StatusInternalServerError) } + + path = filepath.Join(b.pathPrefix, path) minioObject, err := s3Clnt.GetObject(b.bucket, path, s3.GetObjectOptions{}) if err != nil { return nil, model.NewAppError("Reader", "api.file.reader.s3.app_error", nil, err.Error(), http.StatusInternalServerError) } + return minioObject, nil } @@ -101,10 +105,13 @@ func (b *S3FileBackend) ReadFile(path string) ([]byte, *model.AppError) { if err != nil { return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) } + + path = filepath.Join(b.pathPrefix, path) minioObject, err := s3Clnt.GetObject(b.bucket, path, s3.GetObjectOptions{}) if err != nil { return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) } + defer minioObject.Close() if f, err := ioutil.ReadAll(minioObject); err != nil { return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) @@ -119,6 +126,8 @@ func (b *S3FileBackend) FileExists(path string) (bool, *model.AppError) { if err != nil { return false, model.NewAppError("FileExists", "api.file.file_exists.s3.app_error", nil, err.Error(), http.StatusInternalServerError) } + + path = filepath.Join(b.pathPrefix, path) _, err = s3Clnt.StatObject(b.bucket, path, s3.StatObjectOptions{}) if err == nil { @@ -138,11 +147,15 @@ func (b *S3FileBackend) CopyFile(oldPath, newPath string) *model.AppError { return model.NewAppError("copyFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) } + oldPath = filepath.Join(b.pathPrefix, oldPath) + newPath = filepath.Join(b.pathPrefix, newPath) + source := s3.NewSourceInfo(b.bucket, oldPath, nil) destination, err := s3.NewDestinationInfo(b.bucket, newPath, encrypt.NewSSE(), nil) if err != nil { return model.NewAppError("copyFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) } + if err = s3Clnt.CopyObject(destination, source); err != nil { return model.NewAppError("copyFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError) } @@ -155,17 +168,23 @@ func (b *S3FileBackend) MoveFile(oldPath, newPath string) *model.AppError { return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) } + oldPath = filepath.Join(b.pathPrefix, oldPath) + newPath = filepath.Join(b.pathPrefix, newPath) + source := s3.NewSourceInfo(b.bucket, oldPath, nil) destination, err := s3.NewDestinationInfo(b.bucket, newPath, encrypt.NewSSE(), nil) if err != nil { return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) } + if err = s3Clnt.CopyObject(destination, source); err != nil { return model.NewAppError("moveFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError) } + if err = s3Clnt.RemoveObject(b.bucket, oldPath); err != nil { return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError) } + return nil } @@ -176,6 +195,7 @@ func (b *S3FileBackend) WriteFile(fr io.Reader, path string) (int64, *model.AppE } var contentType string + path = filepath.Join(b.pathPrefix, path) if ext := filepath.Ext(path); model.IsFileExtImage(ext) { contentType = model.GetImageMimeType(ext) } else { @@ -202,6 +222,7 @@ func (b *S3FileBackend) RemoveFile(path string) *model.AppError { return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) } + path = filepath.Join(b.pathPrefix, path) if err := s3Clnt.RemoveObject(b.bucket, path); err != nil { return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) } @@ -240,11 +261,13 @@ func (b *S3FileBackend) ListDirectory(path string) (*[]string, *model.AppError) doneCh := make(chan struct{}) defer close(doneCh) + path = filepath.Join(b.pathPrefix, path) if !strings.HasSuffix(path, "/") && len(path) > 0 { // s3Clnt returns only the path itself when "/" is not present // appending "/" to make it consistent across all filesstores path = path + "/" } + for object := range s3Clnt.ListObjects(b.bucket, path, false, doneCh) { if object.Err != nil { return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, object.Err.Error(), http.StatusInternalServerError) @@ -263,6 +286,7 @@ func (b *S3FileBackend) RemoveDirectory(path string) *model.AppError { doneCh := make(chan struct{}) + path = filepath.Join(b.pathPrefix, path) for err := range s3Clnt.RemoveObjects(b.bucket, getPathsFromObjectInfos(s3Clnt.ListObjects(b.bucket, path, true, doneCh))) { if err.Err != nil { doneCh <- struct{}{}