Added support for S3 storage classes (#28319)

* Added support for S3 storage classes

* Added missing translations for S3 storage class

* Changed default storage class values to preserve original behaviour

* Changed storage class description and example

* Fix translations ordering

* Change configuration defaults to empty strings

* Remove redundant empty string check

* Validate storage class variable against constants from S3 API docs

* Validate export storage class against constants

* Use slices for config validation

* Applied patch for translation ordering
Этот коммит содержится в:
Nadav Tasher
2024-11-13 08:59:44 +02:00
коммит произвёл GitHub
родитель 8c3f4a5818
Коммит 5e47c97db4
12 изменённых файлов: 86 добавлений и 5 удалений

Просмотреть файл

@@ -63,6 +63,7 @@ type FileBackendSettings struct {
AmazonS3RequestTimeoutMilliseconds int64
AmazonS3PresignExpiresSeconds int64
AmazonS3UploadPartSizeBytes int64
AmazonS3StorageClass string
}
func NewFileBackendSettingsFromConfig(fileSettings *model.FileSettings, enableComplianceFeature bool, skipVerify bool) FileBackendSettings {
@@ -87,6 +88,7 @@ func NewFileBackendSettingsFromConfig(fileSettings *model.FileSettings, enableCo
AmazonS3RequestTimeoutMilliseconds: *fileSettings.AmazonS3RequestTimeoutMilliseconds,
SkipVerify: skipVerify,
AmazonS3UploadPartSizeBytes: *fileSettings.AmazonS3UploadPartSizeBytes,
AmazonS3StorageClass: *fileSettings.AmazonS3StorageClass,
}
}
@@ -112,6 +114,7 @@ func NewExportFileBackendSettingsFromConfig(fileSettings *model.FileSettings, en
AmazonS3RequestTimeoutMilliseconds: *fileSettings.ExportAmazonS3RequestTimeoutMilliseconds,
AmazonS3PresignExpiresSeconds: *fileSettings.ExportAmazonS3PresignExpiresSeconds,
AmazonS3UploadPartSizeBytes: *fileSettings.ExportAmazonS3UploadPartSizeBytes,
AmazonS3StorageClass: *fileSettings.ExportAmazonS3StorageClass,
SkipVerify: skipVerify,
}
}

Просмотреть файл

@@ -758,6 +758,7 @@ func TestNewExportFileBackendSettingsFromConfig(t *testing.T) {
ExportAmazonS3RequestTimeoutMilliseconds: model.NewPointer(int64(1000)),
ExportAmazonS3PresignExpiresSeconds: model.NewPointer(int64(60000)),
ExportAmazonS3UploadPartSizeBytes: model.NewPointer(int64(model.FileSettingsDefaultS3ExportUploadPartSizeBytes)),
ExportAmazonS3StorageClass: model.NewPointer(""),
}, enableComplianceFeature, skipVerify)
require.Equal(t, expected, actual)
@@ -784,6 +785,7 @@ func TestNewExportFileBackendSettingsFromConfig(t *testing.T) {
AmazonS3RequestTimeoutMilliseconds: 1000,
AmazonS3PresignExpiresSeconds: 60000,
AmazonS3UploadPartSizeBytes: model.FileSettingsDefaultS3ExportUploadPartSizeBytes,
AmazonS3StorageClass: "",
}
actual := NewExportFileBackendSettingsFromConfig(&model.FileSettings{
@@ -801,6 +803,7 @@ func TestNewExportFileBackendSettingsFromConfig(t *testing.T) {
ExportAmazonS3RequestTimeoutMilliseconds: model.NewPointer(int64(1000)),
ExportAmazonS3PresignExpiresSeconds: model.NewPointer(int64(60000)),
ExportAmazonS3UploadPartSizeBytes: model.NewPointer(int64(model.FileSettingsDefaultS3ExportUploadPartSizeBytes)),
ExportAmazonS3StorageClass: model.NewPointer(""),
}, enableComplianceFeature, skipVerify)
require.Equal(t, expected, actual)

Просмотреть файл

@@ -44,6 +44,7 @@ type S3FileBackend struct {
presignExpires time.Duration
isCloud bool // field to indicate whether this is running under Mattermost cloud or not.
uploadPartSize int64
storageClass string
}
type S3FileBackendAuthError struct {
@@ -118,6 +119,7 @@ func newS3FileBackend(settings FileBackendSettings, isCloud bool) (*S3FileBacken
timeout: timeout,
presignExpires: time.Duration(settings.AmazonS3PresignExpiresSeconds) * time.Second,
uploadPartSize: settings.AmazonS3UploadPartSizeBytes,
storageClass: settings.AmazonS3StorageClass,
}
cli, err := backend.s3New(isCloud)
if err != nil {
@@ -504,7 +506,7 @@ func (b *S3FileBackend) WriteFileContext(ctx context.Context, fr io.Reader, path
contentType = "binary/octet-stream"
}
options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize)
options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize, b.storageClass)
objSize := int64(-1)
if b.isCloud {
@@ -548,7 +550,7 @@ func (b *S3FileBackend) AppendFile(fr io.Reader, path string) (int64, error) {
contentType = "binary/octet-stream"
}
options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize)
options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize, b.storageClass)
sse := options.ServerSideEncryption
partName := fp + ".part"
ctx2, cancel2 := context.WithTimeout(context.Background(), b.timeout)
@@ -759,13 +761,14 @@ func (b *S3FileBackend) prefixedPath(s string) (string, error) {
return filepath.Join(b.pathPrefix, s), nil
}
func s3PutOptions(encrypted bool, contentType string, uploadPartSize int64) s3.PutObjectOptions {
func s3PutOptions(encrypted bool, contentType string, uploadPartSize int64, storageClass string) s3.PutObjectOptions {
options := s3.PutObjectOptions{}
if encrypted {
options.ServerSideEncryption = encrypt.NewSSE()
}
options.ContentType = contentType
options.PartSize = uint64(uploadPartSize)
options.StorageClass = storageClass
return options
}