Added support for S3 storage classes (#28319)

* Added support for S3 storage classes * Added missing translations for S3 storage class * Changed default storage class values to preserve original behaviour * Changed storage class description and example * Fix translations ordering * Change configuration defaults to empty strings * Remove redundant empty string check * Validate storage class variable against constants from S3 API docs * Validate export storage class against constants * Use slices for config validation * Applied patch for translation ordering
2024-11-13 08:59:44 +02:00
--- a/api/v4/source/definitions.yaml
+++ b/api/v4/source/definitions.yaml
@@ -1429,6 +1429,8 @@ components:
              type: string
            AmazonS3SSL:
              type: boolean
+            AmazonS3StorageClass:
+              type: string
        EmailSettings:
          type: object
          properties:
@@ -1901,6 +1903,8 @@ components:
              type: boolean
            AmazonS3SSL:
              type: boolean
+            AmazonS3StorageClass:
+              type: string
        EmailSettings:
          type: object
          properties:
--- a/e2e-tests/cypress/tests/support/api/on_prem_default_config.json
+++ b/e2e-tests/cypress/tests/support/api/on_prem_default_config.json
@@ -229,6 +229,7 @@
        "AmazonS3SSE": false,
        "AmazonS3Trace": false,
        "AmazonS3RequestTimeoutMilliseconds": 30000,
+        "AmazonS3StorageClass": "",
        "DedicatedExportStore": false,
        "ExportDriverName": "local",
        "ExportDirectory": "./data/",
@@ -243,7 +244,8 @@
        "ExportAmazonS3SSE": false,
        "ExportAmazonS3Trace": false,
        "ExportAmazonS3RequestTimeoutMilliseconds": 30000,
-        "ExportAmazonS3PresignExpiresSeconds": 21600
+        "ExportAmazonS3PresignExpiresSeconds": 21600,
+        "ExportAmazonS3StorageClass": ""
    },
    "EmailSettings": {
        "EnableSignUpWithEmail": true,
--- a/e2e-tests/playwright/support/server/default_config.ts
+++ b/e2e-tests/playwright/support/server/default_config.ts
@@ -319,6 +319,7 @@ const defaultServerConfig: AdminConfig = {
        AmazonS3Trace: false,
        AmazonS3RequestTimeoutMilliseconds: 30000,
        AmazonS3UploadPartSizeBytes: 5242880,
+        AmazonS3StorageClass: '',
        DedicatedExportStore: false,
        ExportDriverName: 'local',
        ExportDirectory: './data/',
@@ -335,6 +336,7 @@ const defaultServerConfig: AdminConfig = {
        ExportAmazonS3RequestTimeoutMilliseconds: 30000,
        ExportAmazonS3PresignExpiresSeconds: 21600,
        ExportAmazonS3UploadPartSizeBytes: 104857600,
+        ExportAmazonS3StorageClass: '',
    },
    EmailSettings: {
        EnableSignUpWithEmail: true,
--- a/server/i18n/en.json
+++ b/server/i18n/en.json
@@ -9157,6 +9157,10 @@
    "id": "model.config.is_valid.sql_query_timeout.app_error",
    "translation": "Invalid query timeout for SQL settings. Must be a positive number."
  },
+  {
+    "id": "model.config.is_valid.storage_class.app_error",
+    "translation": "Invalid storage class {{.Value}}."
+  },
  {
    "id": "model.config.is_valid.teammate_name_display.app_error",
    "translation": "Invalid teammate display. Must be 'full_name', 'nickname_full_name' or 'username'."
--- a/server/platform/shared/filestore/filesstore.go
+++ b/server/platform/shared/filestore/filesstore.go
@@ -63,6 +63,7 @@ type FileBackendSettings struct {
 	AmazonS3RequestTimeoutMilliseconds int64
 	AmazonS3PresignExpiresSeconds      int64
 	AmazonS3UploadPartSizeBytes        int64
+	AmazonS3StorageClass               string
 }

 func NewFileBackendSettingsFromConfig(fileSettings *model.FileSettings, enableComplianceFeature bool, skipVerify bool) FileBackendSettings {
@@ -87,6 +88,7 @@ func NewFileBackendSettingsFromConfig(fileSettings *model.FileSettings, enableCo
 		AmazonS3RequestTimeoutMilliseconds: *fileSettings.AmazonS3RequestTimeoutMilliseconds,
 		SkipVerify:                         skipVerify,
 		AmazonS3UploadPartSizeBytes:        *fileSettings.AmazonS3UploadPartSizeBytes,
+		AmazonS3StorageClass:               *fileSettings.AmazonS3StorageClass,
 	}
 }

@@ -112,6 +114,7 @@ func NewExportFileBackendSettingsFromConfig(fileSettings *model.FileSettings, en
 		AmazonS3RequestTimeoutMilliseconds: *fileSettings.ExportAmazonS3RequestTimeoutMilliseconds,
 		AmazonS3PresignExpiresSeconds:      *fileSettings.ExportAmazonS3PresignExpiresSeconds,
 		AmazonS3UploadPartSizeBytes:        *fileSettings.ExportAmazonS3UploadPartSizeBytes,
+		AmazonS3StorageClass:               *fileSettings.ExportAmazonS3StorageClass,
 		SkipVerify:                         skipVerify,
 	}
 }
--- a/server/platform/shared/filestore/filesstore_test.go
+++ b/server/platform/shared/filestore/filesstore_test.go
@@ -758,6 +758,7 @@ func TestNewExportFileBackendSettingsFromConfig(t *testing.T) {
 			ExportAmazonS3RequestTimeoutMilliseconds: model.NewPointer(int64(1000)),
 			ExportAmazonS3PresignExpiresSeconds:      model.NewPointer(int64(60000)),
 			ExportAmazonS3UploadPartSizeBytes:        model.NewPointer(int64(model.FileSettingsDefaultS3ExportUploadPartSizeBytes)),
+			ExportAmazonS3StorageClass:               model.NewPointer(""),
 		}, enableComplianceFeature, skipVerify)

 		require.Equal(t, expected, actual)
@@ -784,6 +785,7 @@ func TestNewExportFileBackendSettingsFromConfig(t *testing.T) {
 			AmazonS3RequestTimeoutMilliseconds: 1000,
 			AmazonS3PresignExpiresSeconds:      60000,
 			AmazonS3UploadPartSizeBytes:        model.FileSettingsDefaultS3ExportUploadPartSizeBytes,
+			AmazonS3StorageClass:               "",
 		}

 		actual := NewExportFileBackendSettingsFromConfig(&model.FileSettings{
@@ -801,6 +803,7 @@ func TestNewExportFileBackendSettingsFromConfig(t *testing.T) {
 			ExportAmazonS3RequestTimeoutMilliseconds: model.NewPointer(int64(1000)),
 			ExportAmazonS3PresignExpiresSeconds:      model.NewPointer(int64(60000)),
 			ExportAmazonS3UploadPartSizeBytes:        model.NewPointer(int64(model.FileSettingsDefaultS3ExportUploadPartSizeBytes)),
+			ExportAmazonS3StorageClass:               model.NewPointer(""),
 		}, enableComplianceFeature, skipVerify)

 		require.Equal(t, expected, actual)
--- a/server/platform/shared/filestore/s3store.go
+++ b/server/platform/shared/filestore/s3store.go
@@ -44,6 +44,7 @@ type S3FileBackend struct {
 	presignExpires time.Duration
 	isCloud        bool // field to indicate whether this is running under Mattermost cloud or not.
 	uploadPartSize int64
+	storageClass   string
 }

 type S3FileBackendAuthError struct {
@@ -118,6 +119,7 @@ func newS3FileBackend(settings FileBackendSettings, isCloud bool) (*S3FileBacken
 		timeout:        timeout,
 		presignExpires: time.Duration(settings.AmazonS3PresignExpiresSeconds) * time.Second,
 		uploadPartSize: settings.AmazonS3UploadPartSizeBytes,
+		storageClass:   settings.AmazonS3StorageClass,
 	}
 	cli, err := backend.s3New(isCloud)
 	if err != nil {
@@ -504,7 +506,7 @@ func (b *S3FileBackend) WriteFileContext(ctx context.Context, fr io.Reader, path
 		contentType = "binary/octet-stream"
 	}

-	options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize)
+	options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize, b.storageClass)

 	objSize := int64(-1)
 	if b.isCloud {
@@ -548,7 +550,7 @@ func (b *S3FileBackend) AppendFile(fr io.Reader, path string) (int64, error) {
 		contentType = "binary/octet-stream"
 	}

-	options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize)
+	options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize, b.storageClass)
 	sse := options.ServerSideEncryption
 	partName := fp + ".part"
 	ctx2, cancel2 := context.WithTimeout(context.Background(), b.timeout)
@@ -759,13 +761,14 @@ func (b *S3FileBackend) prefixedPath(s string) (string, error) {
 	return filepath.Join(b.pathPrefix, s), nil
 }

-func s3PutOptions(encrypted bool, contentType string, uploadPartSize int64) s3.PutObjectOptions {
+func s3PutOptions(encrypted bool, contentType string, uploadPartSize int64, storageClass string) s3.PutObjectOptions {
 	options := s3.PutObjectOptions{}
 	if encrypted {
 		options.ServerSideEncryption = encrypt.NewSSE()
 	}
 	options.ContentType = contentType
 	options.PartSize = uint64(uploadPartSize)
+	options.StorageClass = storageClass

 	return options
 }
--- a/server/public/model/config.go
+++ b/server/public/model/config.go
@@ -15,6 +15,7 @@ import (
 	"path/filepath"
 	"reflect"
 	"regexp"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -267,6 +268,19 @@ const (
 	LocalModeSocketPath = "/var/tmp/mattermost_local.socket"

 	ConnectedWorkspacesSettingsDefaultMaxPostsPerSync = 50 // a bit more than 4 typical screenfulls of posts
+
+	// These storage classes are the valid values for the x-amz-storage-class header. More documentation here https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html#AmazonS3-PutObject-request-header-StorageClass
+	StorageClassStandard           = "STANDARD"
+	StorageClassReducedRedundancy  = "REDUCED_REDUNDANCY"
+	StorageClassStandardIA         = "STANDARD_IA"
+	StorageClassOnezoneIA          = "ONEZONE_IA"
+	StorageClassIntelligentTiering = "INTELLIGENT_TIERING"
+	StorageClassGlacier            = "GLACIER"
+	StorageClassDeepArchive        = "DEEP_ARCHIVE"
+	StorageClassOutposts           = "OUTPOSTS"
+	StorageClassGlacierIR          = "GLACIER_IR"
+	StorageClassSnow               = "SNOW"
+	StorageClassExpressOnezone     = "EXPRESS_ONEZONE"
 )

 func GetDefaultAppCustomURLSchemes() []string {
@@ -1649,6 +1663,7 @@ type FileSettings struct {
 	AmazonS3Trace                      *bool   `access:"environment_file_storage,write_restrictable,cloud_restrictable"`
 	AmazonS3RequestTimeoutMilliseconds *int64  `access:"environment_file_storage,write_restrictable,cloud_restrictable"` // telemetry: none
 	AmazonS3UploadPartSizeBytes        *int64  `access:"environment_file_storage,write_restrictable,cloud_restrictable"` // telemetry: none
+	AmazonS3StorageClass               *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"` // telemetry: none
 	// Export store settings
 	DedicatedExportStore                     *bool   `access:"environment_file_storage,write_restrictable"`
 	ExportDriverName                         *string `access:"environment_file_storage,write_restrictable"`
@@ -1666,6 +1681,7 @@ type FileSettings struct {
 	ExportAmazonS3RequestTimeoutMilliseconds *int64  `access:"environment_file_storage,write_restrictable"` // telemetry: none
 	ExportAmazonS3PresignExpiresSeconds      *int64  `access:"environment_file_storage,write_restrictable"` // telemetry: none
 	ExportAmazonS3UploadPartSizeBytes        *int64  `access:"environment_file_storage,write_restrictable"` // telemetry: none
+	ExportAmazonS3StorageClass               *string `access:"environment_file_storage,write_restrictable"` // telemetry: none
 }

 func (s *FileSettings) SetDefaults(isUpdate bool) {
@@ -1778,6 +1794,10 @@ func (s *FileSettings) SetDefaults(isUpdate bool) {
 		s.AmazonS3UploadPartSizeBytes = NewPointer(int64(FileSettingsDefaultS3UploadPartSizeBytes))
 	}

+	if s.AmazonS3StorageClass == nil {
+		s.AmazonS3StorageClass = NewPointer("")
+	}
+
 	if s.DedicatedExportStore == nil {
 		s.DedicatedExportStore = NewPointer(false)
 	}
@@ -1843,6 +1863,10 @@ func (s *FileSettings) SetDefaults(isUpdate bool) {
 	if s.ExportAmazonS3UploadPartSizeBytes == nil {
 		s.ExportAmazonS3UploadPartSizeBytes = NewPointer(int64(FileSettingsDefaultS3ExportUploadPartSizeBytes))
 	}
+
+	if s.ExportAmazonS3StorageClass == nil {
+		s.ExportAmazonS3StorageClass = NewPointer("")
+	}
 }

 type EmailSettings struct {
@@ -3958,6 +3982,14 @@ func (s *FileSettings) isValid() *AppError {
 		return NewAppError("Config.IsValid", "model.config.is_valid.amazons3_timeout.app_error", map[string]any{"Value": *s.MaxImageDecoderConcurrency}, "", http.StatusBadRequest)
 	}

+	if *s.AmazonS3StorageClass != "" && !slices.Contains([]string{StorageClassStandard, StorageClassReducedRedundancy, StorageClassStandardIA, StorageClassOnezoneIA, StorageClassIntelligentTiering, StorageClassGlacier, StorageClassDeepArchive, StorageClassOutposts, StorageClassGlacierIR, StorageClassSnow, StorageClassExpressOnezone}, *s.AmazonS3StorageClass) {
+		return NewAppError("Config.IsValid", "model.config.is_valid.storage_class.app_error", map[string]any{"Value": *s.AmazonS3StorageClass}, "", http.StatusBadRequest)
+	}
+
+	if *s.ExportAmazonS3StorageClass != "" && !slices.Contains([]string{StorageClassStandard, StorageClassReducedRedundancy, StorageClassStandardIA, StorageClassOnezoneIA, StorageClassIntelligentTiering, StorageClassGlacier, StorageClassDeepArchive, StorageClassOutposts, StorageClassGlacierIR, StorageClassSnow, StorageClassExpressOnezone}, *s.ExportAmazonS3StorageClass) {
+		return NewAppError("Config.IsValid", "model.config.is_valid.storage_class.app_error", map[string]any{"Value": *s.ExportAmazonS3StorageClass}, "", http.StatusBadRequest)
+	}
+
 	return nil
 }

--- a/server/tests/test-config.json
+++ b/server/tests/test-config.json
@@ -136,7 +136,8 @@
        "AmazonS3SSL": true,
        "AmazonS3SignV2": false,
        "AmazonS3SSE": false,
-        "AmazonS3Trace": false
+        "AmazonS3Trace": false,
+        "AmazonS3StorageClass": ""
    },
    "EmailSettings": {
        "EnableSignUpWithEmail": true,
--- a/webapp/channels/src/components/admin_console/admin_definition.tsx
+++ b/webapp/channels/src/components/admin_console/admin_definition.tsx
@@ -1097,6 +1097,17 @@ const AdminDefinition: AdminDefinitionType = {
                                it.not(it.stateEquals('FileSettings.DriverName', FILE_STORAGE_DRIVER_S3)),
                            ),
                        },
+                        {
+                            type: 'text',
+                            key: 'FileSettings.AmazonS3StorageClass',
+                            label: defineMessage({id: 'admin.image.amazonS3StorageClassTitle', defaultMessage: 'Amazon S3 Storage Class:'}),
+                            help_text: defineMessage({id: 'admin.image.amazonS3StorageClassDescription', defaultMessage: 'Storage class for your S3 Compatible Storage provider. Defaults to empty.'}),
+                            placeholder: defineMessage({id: 'admin.image.amazonS3StorageClassExample', defaultMessage: 'E.g.: "STANDARD" or "STANDARD_IA"'}),
+                            isDisabled: it.any(
+                                it.not(it.userHasWritePermissionOnResource(RESOURCE_KEYS.ENVIRONMENT.FILE_STORAGE)),
+                                it.not(it.stateEquals('FileSettings.DriverName', FILE_STORAGE_DRIVER_S3)),
+                            ),
+                        },
                        {
                            type: 'button',
                            action: testS3Connection,
@@ -1281,6 +1292,17 @@ const AdminDefinition: AdminDefinitionType = {
                                it.stateEquals('FileSettings.DedicatedExportStore', false),
                            ),
                        },
+                        {
+                            type: 'text',
+                            key: 'FileSettings.ExportAmazonS3StorageClass',
+                            label: defineMessage({id: 'admin.image.amazonS3StorageClassTitle', defaultMessage: 'Amazon S3 Storage Class:'}),
+                            help_text: defineMessage({id: 'admin.image.amazonS3StorageClassDescription', defaultMessage: 'Storage class for your S3 Compatible Storage provider. Defaults to empty.'}),
+                            placeholder: defineMessage({id: 'admin.image.amazonS3StorageClassExample', defaultMessage: 'E.g.: "STANDARD" or "STANDARD_IA"'}),
+                            isDisabled: it.any(
+                                it.not(it.userHasWritePermissionOnResource(RESOURCE_KEYS.ENVIRONMENT.FILE_STORAGE)),
+                                it.not(it.stateEquals('FileSettings.DriverName', FILE_STORAGE_DRIVER_S3)),
+                            ),
+                        },
                        {
                            type: 'button',
                            action: testS3Connection,
--- a/webapp/channels/src/i18n/en.json
+++ b/webapp/channels/src/i18n/en.json
@@ -1153,6 +1153,9 @@
  "admin.image.amazonS3SSETitle": "Enable Server-Side Encryption for Amazon S3:",
  "admin.image.amazonS3SSLDescription": "When false, allow insecure connections to Amazon S3. Defaults to secure connections only.",
  "admin.image.amazonS3SSLTitle": "Enable Secure Amazon S3 Connections:",
+  "admin.image.amazonS3StorageClassDescription": "Storage class for your S3 Compatible Storage provider. Defaults to empty.",
+  "admin.image.amazonS3StorageClassExample": "E.g.: \"STANDARD\" or \"STANDARD_IA\"",
+  "admin.image.amazonS3StorageClassTitle": "Amazon S3 Storage Class:",
  "admin.image.amazonS3TraceDescription": "(Development Mode) When true, log additional debugging information to the system logs.",
  "admin.image.amazonS3TraceTitle": "Enable Amazon S3 Debugging:",
  "admin.image.archiveRecursionDescription": "When enabled, content of documents within ZIP files will be returned in search results. This may have an impact on server performance for large files.",
--- a/webapp/platform/types/src/config.ts
+++ b/webapp/platform/types/src/config.ts
@@ -544,6 +544,7 @@ export type FileSettings = {
    AmazonS3Trace: boolean;
    AmazonS3RequestTimeoutMilliseconds: number;
    AmazonS3UploadPartSizeBytes: number;
+    AmazonS3StorageClass: string;
    DedicatedExportStore: boolean;
    ExportDriverName: string;
    ExportDirectory: string;
@@ -560,6 +561,7 @@ export type FileSettings = {
    ExportAmazonS3RequestTimeoutMilliseconds: number;
    ExportAmazonS3PresignExpiresSeconds: number;
    ExportAmazonS3UploadPartSizeBytes: number;
+    ExportAmazonS3StorageClass: string;
 };

 export type EmailSettings = {