Added support for S3 storage classes (#28319)

* Added support for S3 storage classes * Added missing translations for S3 storage class * Changed default storage class values to preserve original behaviour * Changed storage class description and example * Fix translations ordering * Change configuration defaults to empty strings * Remove redundant empty string check * Validate storage class variable against constants from S3 API docs * Validate export storage class against constants * Use slices for config validation * Applied patch for translation ordering
2024-11-13 08:59:44 +02:00
--- a/server/i18n/en.json
+++ b/server/i18n/en.json
@@ -9157,6 +9157,10 @@
    "id": "model.config.is_valid.sql_query_timeout.app_error",
    "translation": "Invalid query timeout for SQL settings. Must be a positive number."
  },
+  {
+    "id": "model.config.is_valid.storage_class.app_error",
+    "translation": "Invalid storage class {{.Value}}."
+  },
  {
    "id": "model.config.is_valid.teammate_name_display.app_error",
    "translation": "Invalid teammate display. Must be 'full_name', 'nickname_full_name' or 'username'."
--- a/server/platform/shared/filestore/filesstore.go
+++ b/server/platform/shared/filestore/filesstore.go
@@ -63,6 +63,7 @@ type FileBackendSettings struct {
 	AmazonS3RequestTimeoutMilliseconds int64
 	AmazonS3PresignExpiresSeconds      int64
 	AmazonS3UploadPartSizeBytes        int64
+	AmazonS3StorageClass               string
 }

 func NewFileBackendSettingsFromConfig(fileSettings *model.FileSettings, enableComplianceFeature bool, skipVerify bool) FileBackendSettings {
@@ -87,6 +88,7 @@ func NewFileBackendSettingsFromConfig(fileSettings *model.FileSettings, enableCo
 		AmazonS3RequestTimeoutMilliseconds: *fileSettings.AmazonS3RequestTimeoutMilliseconds,
 		SkipVerify:                         skipVerify,
 		AmazonS3UploadPartSizeBytes:        *fileSettings.AmazonS3UploadPartSizeBytes,
+		AmazonS3StorageClass:               *fileSettings.AmazonS3StorageClass,
 	}
 }

@@ -112,6 +114,7 @@ func NewExportFileBackendSettingsFromConfig(fileSettings *model.FileSettings, en
 		AmazonS3RequestTimeoutMilliseconds: *fileSettings.ExportAmazonS3RequestTimeoutMilliseconds,
 		AmazonS3PresignExpiresSeconds:      *fileSettings.ExportAmazonS3PresignExpiresSeconds,
 		AmazonS3UploadPartSizeBytes:        *fileSettings.ExportAmazonS3UploadPartSizeBytes,
+		AmazonS3StorageClass:               *fileSettings.ExportAmazonS3StorageClass,
 		SkipVerify:                         skipVerify,
 	}
 }
--- a/server/platform/shared/filestore/filesstore_test.go
+++ b/server/platform/shared/filestore/filesstore_test.go
@@ -758,6 +758,7 @@ func TestNewExportFileBackendSettingsFromConfig(t *testing.T) {
 			ExportAmazonS3RequestTimeoutMilliseconds: model.NewPointer(int64(1000)),
 			ExportAmazonS3PresignExpiresSeconds:      model.NewPointer(int64(60000)),
 			ExportAmazonS3UploadPartSizeBytes:        model.NewPointer(int64(model.FileSettingsDefaultS3ExportUploadPartSizeBytes)),
+			ExportAmazonS3StorageClass:               model.NewPointer(""),
 		}, enableComplianceFeature, skipVerify)

 		require.Equal(t, expected, actual)
@@ -784,6 +785,7 @@ func TestNewExportFileBackendSettingsFromConfig(t *testing.T) {
 			AmazonS3RequestTimeoutMilliseconds: 1000,
 			AmazonS3PresignExpiresSeconds:      60000,
 			AmazonS3UploadPartSizeBytes:        model.FileSettingsDefaultS3ExportUploadPartSizeBytes,
+			AmazonS3StorageClass:               "",
 		}

 		actual := NewExportFileBackendSettingsFromConfig(&model.FileSettings{
@@ -801,6 +803,7 @@ func TestNewExportFileBackendSettingsFromConfig(t *testing.T) {
 			ExportAmazonS3RequestTimeoutMilliseconds: model.NewPointer(int64(1000)),
 			ExportAmazonS3PresignExpiresSeconds:      model.NewPointer(int64(60000)),
 			ExportAmazonS3UploadPartSizeBytes:        model.NewPointer(int64(model.FileSettingsDefaultS3ExportUploadPartSizeBytes)),
+			ExportAmazonS3StorageClass:               model.NewPointer(""),
 		}, enableComplianceFeature, skipVerify)

 		require.Equal(t, expected, actual)
--- a/server/platform/shared/filestore/s3store.go
+++ b/server/platform/shared/filestore/s3store.go
@@ -44,6 +44,7 @@ type S3FileBackend struct {
 	presignExpires time.Duration
 	isCloud        bool // field to indicate whether this is running under Mattermost cloud or not.
 	uploadPartSize int64
+	storageClass   string
 }

 type S3FileBackendAuthError struct {
@@ -118,6 +119,7 @@ func newS3FileBackend(settings FileBackendSettings, isCloud bool) (*S3FileBacken
 		timeout:        timeout,
 		presignExpires: time.Duration(settings.AmazonS3PresignExpiresSeconds) * time.Second,
 		uploadPartSize: settings.AmazonS3UploadPartSizeBytes,
+		storageClass:   settings.AmazonS3StorageClass,
 	}
 	cli, err := backend.s3New(isCloud)
 	if err != nil {
@@ -504,7 +506,7 @@ func (b *S3FileBackend) WriteFileContext(ctx context.Context, fr io.Reader, path
 		contentType = "binary/octet-stream"
 	}

-	options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize)
+	options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize, b.storageClass)

 	objSize := int64(-1)
 	if b.isCloud {
@@ -548,7 +550,7 @@ func (b *S3FileBackend) AppendFile(fr io.Reader, path string) (int64, error) {
 		contentType = "binary/octet-stream"
 	}

-	options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize)
+	options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize, b.storageClass)
 	sse := options.ServerSideEncryption
 	partName := fp + ".part"
 	ctx2, cancel2 := context.WithTimeout(context.Background(), b.timeout)
@@ -759,13 +761,14 @@ func (b *S3FileBackend) prefixedPath(s string) (string, error) {
 	return filepath.Join(b.pathPrefix, s), nil
 }

-func s3PutOptions(encrypted bool, contentType string, uploadPartSize int64) s3.PutObjectOptions {
+func s3PutOptions(encrypted bool, contentType string, uploadPartSize int64, storageClass string) s3.PutObjectOptions {
 	options := s3.PutObjectOptions{}
 	if encrypted {
 		options.ServerSideEncryption = encrypt.NewSSE()
 	}
 	options.ContentType = contentType
 	options.PartSize = uint64(uploadPartSize)
+	options.StorageClass = storageClass

 	return options
 }
--- a/server/public/model/config.go
+++ b/server/public/model/config.go
@@ -15,6 +15,7 @@ import (
 	"path/filepath"
 	"reflect"
 	"regexp"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -267,6 +268,19 @@ const (
 	LocalModeSocketPath = "/var/tmp/mattermost_local.socket"

 	ConnectedWorkspacesSettingsDefaultMaxPostsPerSync = 50 // a bit more than 4 typical screenfulls of posts
+
+	// These storage classes are the valid values for the x-amz-storage-class header. More documentation here https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html#AmazonS3-PutObject-request-header-StorageClass
+	StorageClassStandard           = "STANDARD"
+	StorageClassReducedRedundancy  = "REDUCED_REDUNDANCY"
+	StorageClassStandardIA         = "STANDARD_IA"
+	StorageClassOnezoneIA          = "ONEZONE_IA"
+	StorageClassIntelligentTiering = "INTELLIGENT_TIERING"
+	StorageClassGlacier            = "GLACIER"
+	StorageClassDeepArchive        = "DEEP_ARCHIVE"
+	StorageClassOutposts           = "OUTPOSTS"
+	StorageClassGlacierIR          = "GLACIER_IR"
+	StorageClassSnow               = "SNOW"
+	StorageClassExpressOnezone     = "EXPRESS_ONEZONE"
 )

 func GetDefaultAppCustomURLSchemes() []string {
@@ -1649,6 +1663,7 @@ type FileSettings struct {
 	AmazonS3Trace                      *bool   `access:"environment_file_storage,write_restrictable,cloud_restrictable"`
 	AmazonS3RequestTimeoutMilliseconds *int64  `access:"environment_file_storage,write_restrictable,cloud_restrictable"` // telemetry: none
 	AmazonS3UploadPartSizeBytes        *int64  `access:"environment_file_storage,write_restrictable,cloud_restrictable"` // telemetry: none
+	AmazonS3StorageClass               *string `access:"environment_file_storage,write_restrictable,cloud_restrictable"` // telemetry: none
 	// Export store settings
 	DedicatedExportStore                     *bool   `access:"environment_file_storage,write_restrictable"`
 	ExportDriverName                         *string `access:"environment_file_storage,write_restrictable"`
@@ -1666,6 +1681,7 @@ type FileSettings struct {
 	ExportAmazonS3RequestTimeoutMilliseconds *int64  `access:"environment_file_storage,write_restrictable"` // telemetry: none
 	ExportAmazonS3PresignExpiresSeconds      *int64  `access:"environment_file_storage,write_restrictable"` // telemetry: none
 	ExportAmazonS3UploadPartSizeBytes        *int64  `access:"environment_file_storage,write_restrictable"` // telemetry: none
+	ExportAmazonS3StorageClass               *string `access:"environment_file_storage,write_restrictable"` // telemetry: none
 }

 func (s *FileSettings) SetDefaults(isUpdate bool) {
@@ -1778,6 +1794,10 @@ func (s *FileSettings) SetDefaults(isUpdate bool) {
 		s.AmazonS3UploadPartSizeBytes = NewPointer(int64(FileSettingsDefaultS3UploadPartSizeBytes))
 	}

+	if s.AmazonS3StorageClass == nil {
+		s.AmazonS3StorageClass = NewPointer("")
+	}
+
 	if s.DedicatedExportStore == nil {
 		s.DedicatedExportStore = NewPointer(false)
 	}
@@ -1843,6 +1863,10 @@ func (s *FileSettings) SetDefaults(isUpdate bool) {
 	if s.ExportAmazonS3UploadPartSizeBytes == nil {
 		s.ExportAmazonS3UploadPartSizeBytes = NewPointer(int64(FileSettingsDefaultS3ExportUploadPartSizeBytes))
 	}
+
+	if s.ExportAmazonS3StorageClass == nil {
+		s.ExportAmazonS3StorageClass = NewPointer("")
+	}
 }

 type EmailSettings struct {
@@ -3958,6 +3982,14 @@ func (s *FileSettings) isValid() *AppError {
 		return NewAppError("Config.IsValid", "model.config.is_valid.amazons3_timeout.app_error", map[string]any{"Value": *s.MaxImageDecoderConcurrency}, "", http.StatusBadRequest)
 	}

+	if *s.AmazonS3StorageClass != "" && !slices.Contains([]string{StorageClassStandard, StorageClassReducedRedundancy, StorageClassStandardIA, StorageClassOnezoneIA, StorageClassIntelligentTiering, StorageClassGlacier, StorageClassDeepArchive, StorageClassOutposts, StorageClassGlacierIR, StorageClassSnow, StorageClassExpressOnezone}, *s.AmazonS3StorageClass) {
+		return NewAppError("Config.IsValid", "model.config.is_valid.storage_class.app_error", map[string]any{"Value": *s.AmazonS3StorageClass}, "", http.StatusBadRequest)
+	}
+
+	if *s.ExportAmazonS3StorageClass != "" && !slices.Contains([]string{StorageClassStandard, StorageClassReducedRedundancy, StorageClassStandardIA, StorageClassOnezoneIA, StorageClassIntelligentTiering, StorageClassGlacier, StorageClassDeepArchive, StorageClassOutposts, StorageClassGlacierIR, StorageClassSnow, StorageClassExpressOnezone}, *s.ExportAmazonS3StorageClass) {
+		return NewAppError("Config.IsValid", "model.config.is_valid.storage_class.app_error", map[string]any{"Value": *s.ExportAmazonS3StorageClass}, "", http.StatusBadRequest)
+	}
+
 	return nil
 }

--- a/server/tests/test-config.json
+++ b/server/tests/test-config.json
@@ -136,7 +136,8 @@
        "AmazonS3SSL": true,
        "AmazonS3SignV2": false,
        "AmazonS3SSE": false,
-        "AmazonS3Trace": false
+        "AmazonS3Trace": false,
+        "AmazonS3StorageClass": ""
    },
    "EmailSettings": {
        "EnableSignUpWithEmail": true,