From 5e47c97db4d188c43d9cf246f5b590298e717297 Mon Sep 17 00:00:00 2001 From: Nadav Tasher Date: Wed, 13 Nov 2024 08:59:44 +0200 Subject: [PATCH] Added support for S3 storage classes (#28319) * Added support for S3 storage classes * Added missing translations for S3 storage class * Changed default storage class values to preserve original behaviour * Changed storage class description and example * Fix translations ordering * Change configuration defaults to empty strings * Remove redundant empty string check * Validate storage class variable against constants from S3 API docs * Validate export storage class against constants * Use slices for config validation * Applied patch for translation ordering --- api/v4/source/definitions.yaml | 4 +++ .../support/api/on_prem_default_config.json | 4 ++- .../support/server/default_config.ts | 2 ++ server/i18n/en.json | 4 +++ .../platform/shared/filestore/filesstore.go | 3 ++ .../shared/filestore/filesstore_test.go | 3 ++ server/platform/shared/filestore/s3store.go | 9 ++++-- server/public/model/config.go | 32 +++++++++++++++++++ server/tests/test-config.json | 3 +- .../admin_console/admin_definition.tsx | 22 +++++++++++++ webapp/channels/src/i18n/en.json | 3 ++ webapp/platform/types/src/config.ts | 2 ++ 12 files changed, 86 insertions(+), 5 deletions(-) diff --git a/api/v4/source/definitions.yaml b/api/v4/source/definitions.yaml index 2171832e98..a4d3cb9458 100644 --- a/api/v4/source/definitions.yaml +++ b/api/v4/source/definitions.yaml @@ -1429,6 +1429,8 @@ components: type: string AmazonS3SSL: type: boolean + AmazonS3StorageClass: + type: string EmailSettings: type: object properties: @@ -1901,6 +1903,8 @@ components: type: boolean AmazonS3SSL: type: boolean + AmazonS3StorageClass: + type: string EmailSettings: type: object properties: diff --git a/e2e-tests/cypress/tests/support/api/on_prem_default_config.json b/e2e-tests/cypress/tests/support/api/on_prem_default_config.json index 3a34c694a5..ffb5f35d26 100644 --- a/e2e-tests/cypress/tests/support/api/on_prem_default_config.json +++ b/e2e-tests/cypress/tests/support/api/on_prem_default_config.json @@ -229,6 +229,7 @@ "AmazonS3SSE": false, "AmazonS3Trace": false, "AmazonS3RequestTimeoutMilliseconds": 30000, + "AmazonS3StorageClass": "", "DedicatedExportStore": false, "ExportDriverName": "local", "ExportDirectory": "./data/", @@ -243,7 +244,8 @@ "ExportAmazonS3SSE": false, "ExportAmazonS3Trace": false, "ExportAmazonS3RequestTimeoutMilliseconds": 30000, - "ExportAmazonS3PresignExpiresSeconds": 21600 + "ExportAmazonS3PresignExpiresSeconds": 21600, + "ExportAmazonS3StorageClass": "" }, "EmailSettings": { "EnableSignUpWithEmail": true, diff --git a/e2e-tests/playwright/support/server/default_config.ts b/e2e-tests/playwright/support/server/default_config.ts index 4a008a81e2..79b788817b 100644 --- a/e2e-tests/playwright/support/server/default_config.ts +++ b/e2e-tests/playwright/support/server/default_config.ts @@ -319,6 +319,7 @@ const defaultServerConfig: AdminConfig = { AmazonS3Trace: false, AmazonS3RequestTimeoutMilliseconds: 30000, AmazonS3UploadPartSizeBytes: 5242880, + AmazonS3StorageClass: '', DedicatedExportStore: false, ExportDriverName: 'local', ExportDirectory: './data/', @@ -335,6 +336,7 @@ const defaultServerConfig: AdminConfig = { ExportAmazonS3RequestTimeoutMilliseconds: 30000, ExportAmazonS3PresignExpiresSeconds: 21600, ExportAmazonS3UploadPartSizeBytes: 104857600, + ExportAmazonS3StorageClass: '', }, EmailSettings: { EnableSignUpWithEmail: true, diff --git a/server/i18n/en.json b/server/i18n/en.json index 0a7d2d1bff..84d234bf56 100644 --- a/server/i18n/en.json +++ b/server/i18n/en.json @@ -9157,6 +9157,10 @@ "id": "model.config.is_valid.sql_query_timeout.app_error", "translation": "Invalid query timeout for SQL settings. Must be a positive number." }, + { + "id": "model.config.is_valid.storage_class.app_error", + "translation": "Invalid storage class {{.Value}}." + }, { "id": "model.config.is_valid.teammate_name_display.app_error", "translation": "Invalid teammate display. Must be 'full_name', 'nickname_full_name' or 'username'." diff --git a/server/platform/shared/filestore/filesstore.go b/server/platform/shared/filestore/filesstore.go index f30a16f717..b6038fa651 100644 --- a/server/platform/shared/filestore/filesstore.go +++ b/server/platform/shared/filestore/filesstore.go @@ -63,6 +63,7 @@ type FileBackendSettings struct { AmazonS3RequestTimeoutMilliseconds int64 AmazonS3PresignExpiresSeconds int64 AmazonS3UploadPartSizeBytes int64 + AmazonS3StorageClass string } func NewFileBackendSettingsFromConfig(fileSettings *model.FileSettings, enableComplianceFeature bool, skipVerify bool) FileBackendSettings { @@ -87,6 +88,7 @@ func NewFileBackendSettingsFromConfig(fileSettings *model.FileSettings, enableCo AmazonS3RequestTimeoutMilliseconds: *fileSettings.AmazonS3RequestTimeoutMilliseconds, SkipVerify: skipVerify, AmazonS3UploadPartSizeBytes: *fileSettings.AmazonS3UploadPartSizeBytes, + AmazonS3StorageClass: *fileSettings.AmazonS3StorageClass, } } @@ -112,6 +114,7 @@ func NewExportFileBackendSettingsFromConfig(fileSettings *model.FileSettings, en AmazonS3RequestTimeoutMilliseconds: *fileSettings.ExportAmazonS3RequestTimeoutMilliseconds, AmazonS3PresignExpiresSeconds: *fileSettings.ExportAmazonS3PresignExpiresSeconds, AmazonS3UploadPartSizeBytes: *fileSettings.ExportAmazonS3UploadPartSizeBytes, + AmazonS3StorageClass: *fileSettings.ExportAmazonS3StorageClass, SkipVerify: skipVerify, } } diff --git a/server/platform/shared/filestore/filesstore_test.go b/server/platform/shared/filestore/filesstore_test.go index d57fb07abe..ed12b1cc2c 100644 --- a/server/platform/shared/filestore/filesstore_test.go +++ b/server/platform/shared/filestore/filesstore_test.go @@ -758,6 +758,7 @@ func TestNewExportFileBackendSettingsFromConfig(t *testing.T) { ExportAmazonS3RequestTimeoutMilliseconds: model.NewPointer(int64(1000)), ExportAmazonS3PresignExpiresSeconds: model.NewPointer(int64(60000)), ExportAmazonS3UploadPartSizeBytes: model.NewPointer(int64(model.FileSettingsDefaultS3ExportUploadPartSizeBytes)), + ExportAmazonS3StorageClass: model.NewPointer(""), }, enableComplianceFeature, skipVerify) require.Equal(t, expected, actual) @@ -784,6 +785,7 @@ func TestNewExportFileBackendSettingsFromConfig(t *testing.T) { AmazonS3RequestTimeoutMilliseconds: 1000, AmazonS3PresignExpiresSeconds: 60000, AmazonS3UploadPartSizeBytes: model.FileSettingsDefaultS3ExportUploadPartSizeBytes, + AmazonS3StorageClass: "", } actual := NewExportFileBackendSettingsFromConfig(&model.FileSettings{ @@ -801,6 +803,7 @@ func TestNewExportFileBackendSettingsFromConfig(t *testing.T) { ExportAmazonS3RequestTimeoutMilliseconds: model.NewPointer(int64(1000)), ExportAmazonS3PresignExpiresSeconds: model.NewPointer(int64(60000)), ExportAmazonS3UploadPartSizeBytes: model.NewPointer(int64(model.FileSettingsDefaultS3ExportUploadPartSizeBytes)), + ExportAmazonS3StorageClass: model.NewPointer(""), }, enableComplianceFeature, skipVerify) require.Equal(t, expected, actual) diff --git a/server/platform/shared/filestore/s3store.go b/server/platform/shared/filestore/s3store.go index 7a152e62c7..d2f71823c7 100644 --- a/server/platform/shared/filestore/s3store.go +++ b/server/platform/shared/filestore/s3store.go @@ -44,6 +44,7 @@ type S3FileBackend struct { presignExpires time.Duration isCloud bool // field to indicate whether this is running under Mattermost cloud or not. uploadPartSize int64 + storageClass string } type S3FileBackendAuthError struct { @@ -118,6 +119,7 @@ func newS3FileBackend(settings FileBackendSettings, isCloud bool) (*S3FileBacken timeout: timeout, presignExpires: time.Duration(settings.AmazonS3PresignExpiresSeconds) * time.Second, uploadPartSize: settings.AmazonS3UploadPartSizeBytes, + storageClass: settings.AmazonS3StorageClass, } cli, err := backend.s3New(isCloud) if err != nil { @@ -504,7 +506,7 @@ func (b *S3FileBackend) WriteFileContext(ctx context.Context, fr io.Reader, path contentType = "binary/octet-stream" } - options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize) + options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize, b.storageClass) objSize := int64(-1) if b.isCloud { @@ -548,7 +550,7 @@ func (b *S3FileBackend) AppendFile(fr io.Reader, path string) (int64, error) { contentType = "binary/octet-stream" } - options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize) + options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize, b.storageClass) sse := options.ServerSideEncryption partName := fp + ".part" ctx2, cancel2 := context.WithTimeout(context.Background(), b.timeout) @@ -759,13 +761,14 @@ func (b *S3FileBackend) prefixedPath(s string) (string, error) { return filepath.Join(b.pathPrefix, s), nil } -func s3PutOptions(encrypted bool, contentType string, uploadPartSize int64) s3.PutObjectOptions { +func s3PutOptions(encrypted bool, contentType string, uploadPartSize int64, storageClass string) s3.PutObjectOptions { options := s3.PutObjectOptions{} if encrypted { options.ServerSideEncryption = encrypt.NewSSE() } options.ContentType = contentType options.PartSize = uint64(uploadPartSize) + options.StorageClass = storageClass return options } diff --git a/server/public/model/config.go b/server/public/model/config.go index d5a905d6e4..3ccce20062 100644 --- a/server/public/model/config.go +++ b/server/public/model/config.go @@ -15,6 +15,7 @@ import ( "path/filepath" "reflect" "regexp" + "slices" "strconv" "strings" "time" @@ -267,6 +268,19 @@ const ( LocalModeSocketPath = "/var/tmp/mattermost_local.socket" ConnectedWorkspacesSettingsDefaultMaxPostsPerSync = 50 // a bit more than 4 typical screenfulls of posts + + // These storage classes are the valid values for the x-amz-storage-class header. More documentation here https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html#AmazonS3-PutObject-request-header-StorageClass + StorageClassStandard = "STANDARD" + StorageClassReducedRedundancy = "REDUCED_REDUNDANCY" + StorageClassStandardIA = "STANDARD_IA" + StorageClassOnezoneIA = "ONEZONE_IA" + StorageClassIntelligentTiering = "INTELLIGENT_TIERING" + StorageClassGlacier = "GLACIER" + StorageClassDeepArchive = "DEEP_ARCHIVE" + StorageClassOutposts = "OUTPOSTS" + StorageClassGlacierIR = "GLACIER_IR" + StorageClassSnow = "SNOW" + StorageClassExpressOnezone = "EXPRESS_ONEZONE" ) func GetDefaultAppCustomURLSchemes() []string { @@ -1649,6 +1663,7 @@ type FileSettings struct { AmazonS3Trace *bool `access:"environment_file_storage,write_restrictable,cloud_restrictable"` AmazonS3RequestTimeoutMilliseconds *int64 `access:"environment_file_storage,write_restrictable,cloud_restrictable"` // telemetry: none AmazonS3UploadPartSizeBytes *int64 `access:"environment_file_storage,write_restrictable,cloud_restrictable"` // telemetry: none + AmazonS3StorageClass *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"` // telemetry: none // Export store settings DedicatedExportStore *bool `access:"environment_file_storage,write_restrictable"` ExportDriverName *string `access:"environment_file_storage,write_restrictable"` @@ -1666,6 +1681,7 @@ type FileSettings struct { ExportAmazonS3RequestTimeoutMilliseconds *int64 `access:"environment_file_storage,write_restrictable"` // telemetry: none ExportAmazonS3PresignExpiresSeconds *int64 `access:"environment_file_storage,write_restrictable"` // telemetry: none ExportAmazonS3UploadPartSizeBytes *int64 `access:"environment_file_storage,write_restrictable"` // telemetry: none + ExportAmazonS3StorageClass *string `access:"environment_file_storage,write_restrictable"` // telemetry: none } func (s *FileSettings) SetDefaults(isUpdate bool) { @@ -1778,6 +1794,10 @@ func (s *FileSettings) SetDefaults(isUpdate bool) { s.AmazonS3UploadPartSizeBytes = NewPointer(int64(FileSettingsDefaultS3UploadPartSizeBytes)) } + if s.AmazonS3StorageClass == nil { + s.AmazonS3StorageClass = NewPointer("") + } + if s.DedicatedExportStore == nil { s.DedicatedExportStore = NewPointer(false) } @@ -1843,6 +1863,10 @@ func (s *FileSettings) SetDefaults(isUpdate bool) { if s.ExportAmazonS3UploadPartSizeBytes == nil { s.ExportAmazonS3UploadPartSizeBytes = NewPointer(int64(FileSettingsDefaultS3ExportUploadPartSizeBytes)) } + + if s.ExportAmazonS3StorageClass == nil { + s.ExportAmazonS3StorageClass = NewPointer("") + } } type EmailSettings struct { @@ -3958,6 +3982,14 @@ func (s *FileSettings) isValid() *AppError { return NewAppError("Config.IsValid", "model.config.is_valid.amazons3_timeout.app_error", map[string]any{"Value": *s.MaxImageDecoderConcurrency}, "", http.StatusBadRequest) } + if *s.AmazonS3StorageClass != "" && !slices.Contains([]string{StorageClassStandard, StorageClassReducedRedundancy, StorageClassStandardIA, StorageClassOnezoneIA, StorageClassIntelligentTiering, StorageClassGlacier, StorageClassDeepArchive, StorageClassOutposts, StorageClassGlacierIR, StorageClassSnow, StorageClassExpressOnezone}, *s.AmazonS3StorageClass) { + return NewAppError("Config.IsValid", "model.config.is_valid.storage_class.app_error", map[string]any{"Value": *s.AmazonS3StorageClass}, "", http.StatusBadRequest) + } + + if *s.ExportAmazonS3StorageClass != "" && !slices.Contains([]string{StorageClassStandard, StorageClassReducedRedundancy, StorageClassStandardIA, StorageClassOnezoneIA, StorageClassIntelligentTiering, StorageClassGlacier, StorageClassDeepArchive, StorageClassOutposts, StorageClassGlacierIR, StorageClassSnow, StorageClassExpressOnezone}, *s.ExportAmazonS3StorageClass) { + return NewAppError("Config.IsValid", "model.config.is_valid.storage_class.app_error", map[string]any{"Value": *s.ExportAmazonS3StorageClass}, "", http.StatusBadRequest) + } + return nil } diff --git a/server/tests/test-config.json b/server/tests/test-config.json index ca27301807..725ddce0e3 100644 --- a/server/tests/test-config.json +++ b/server/tests/test-config.json @@ -136,7 +136,8 @@ "AmazonS3SSL": true, "AmazonS3SignV2": false, "AmazonS3SSE": false, - "AmazonS3Trace": false + "AmazonS3Trace": false, + "AmazonS3StorageClass": "" }, "EmailSettings": { "EnableSignUpWithEmail": true, diff --git a/webapp/channels/src/components/admin_console/admin_definition.tsx b/webapp/channels/src/components/admin_console/admin_definition.tsx index a9e5b90d1d..f74b8848bf 100644 --- a/webapp/channels/src/components/admin_console/admin_definition.tsx +++ b/webapp/channels/src/components/admin_console/admin_definition.tsx @@ -1097,6 +1097,17 @@ const AdminDefinition: AdminDefinitionType = { it.not(it.stateEquals('FileSettings.DriverName', FILE_STORAGE_DRIVER_S3)), ), }, + { + type: 'text', + key: 'FileSettings.AmazonS3StorageClass', + label: defineMessage({id: 'admin.image.amazonS3StorageClassTitle', defaultMessage: 'Amazon S3 Storage Class:'}), + help_text: defineMessage({id: 'admin.image.amazonS3StorageClassDescription', defaultMessage: 'Storage class for your S3 Compatible Storage provider. Defaults to empty.'}), + placeholder: defineMessage({id: 'admin.image.amazonS3StorageClassExample', defaultMessage: 'E.g.: "STANDARD" or "STANDARD_IA"'}), + isDisabled: it.any( + it.not(it.userHasWritePermissionOnResource(RESOURCE_KEYS.ENVIRONMENT.FILE_STORAGE)), + it.not(it.stateEquals('FileSettings.DriverName', FILE_STORAGE_DRIVER_S3)), + ), + }, { type: 'button', action: testS3Connection, @@ -1281,6 +1292,17 @@ const AdminDefinition: AdminDefinitionType = { it.stateEquals('FileSettings.DedicatedExportStore', false), ), }, + { + type: 'text', + key: 'FileSettings.ExportAmazonS3StorageClass', + label: defineMessage({id: 'admin.image.amazonS3StorageClassTitle', defaultMessage: 'Amazon S3 Storage Class:'}), + help_text: defineMessage({id: 'admin.image.amazonS3StorageClassDescription', defaultMessage: 'Storage class for your S3 Compatible Storage provider. Defaults to empty.'}), + placeholder: defineMessage({id: 'admin.image.amazonS3StorageClassExample', defaultMessage: 'E.g.: "STANDARD" or "STANDARD_IA"'}), + isDisabled: it.any( + it.not(it.userHasWritePermissionOnResource(RESOURCE_KEYS.ENVIRONMENT.FILE_STORAGE)), + it.not(it.stateEquals('FileSettings.DriverName', FILE_STORAGE_DRIVER_S3)), + ), + }, { type: 'button', action: testS3Connection, diff --git a/webapp/channels/src/i18n/en.json b/webapp/channels/src/i18n/en.json index e9435d4798..3053633982 100644 --- a/webapp/channels/src/i18n/en.json +++ b/webapp/channels/src/i18n/en.json @@ -1153,6 +1153,9 @@ "admin.image.amazonS3SSETitle": "Enable Server-Side Encryption for Amazon S3:", "admin.image.amazonS3SSLDescription": "When false, allow insecure connections to Amazon S3. Defaults to secure connections only.", "admin.image.amazonS3SSLTitle": "Enable Secure Amazon S3 Connections:", + "admin.image.amazonS3StorageClassDescription": "Storage class for your S3 Compatible Storage provider. Defaults to empty.", + "admin.image.amazonS3StorageClassExample": "E.g.: \"STANDARD\" or \"STANDARD_IA\"", + "admin.image.amazonS3StorageClassTitle": "Amazon S3 Storage Class:", "admin.image.amazonS3TraceDescription": "(Development Mode) When true, log additional debugging information to the system logs.", "admin.image.amazonS3TraceTitle": "Enable Amazon S3 Debugging:", "admin.image.archiveRecursionDescription": "When enabled, content of documents within ZIP files will be returned in search results. This may have an impact on server performance for large files.", diff --git a/webapp/platform/types/src/config.ts b/webapp/platform/types/src/config.ts index 4364bc285b..8374f7ff3b 100644 --- a/webapp/platform/types/src/config.ts +++ b/webapp/platform/types/src/config.ts @@ -544,6 +544,7 @@ export type FileSettings = { AmazonS3Trace: boolean; AmazonS3RequestTimeoutMilliseconds: number; AmazonS3UploadPartSizeBytes: number; + AmazonS3StorageClass: string; DedicatedExportStore: boolean; ExportDriverName: string; ExportDirectory: string; @@ -560,6 +561,7 @@ export type FileSettings = { ExportAmazonS3RequestTimeoutMilliseconds: number; ExportAmazonS3PresignExpiresSeconds: number; ExportAmazonS3UploadPartSizeBytes: number; + ExportAmazonS3StorageClass: string; }; export type EmailSettings = {