S3store/image type (#30451)
* add video type * add video mime-type * combine MIME type maps and update tests * correct MIME type detection for video files * format code * lint fix * simplified mimeType * change avi mime type * add avi type
Этот коммит содержится в:
коммит произвёл
GitHub
родитель
c0f1cbf727
Коммит
596053b9af
@@ -11,6 +11,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
|
"mime"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
@@ -62,28 +63,18 @@ const (
|
|||||||
invalidBucket = "InvalidBucketName"
|
invalidBucket = "InvalidBucketName"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
|
||||||
imageExtensions = map[string]bool{".jpg": true, ".jpeg": true, ".gif": true, ".bmp": true, ".png": true, ".tiff": true, "tif": true}
|
|
||||||
imageMimeTypes = map[string]string{".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".gif": "image/gif", ".bmp": "image/bmp", ".png": "image/png", ".tiff": "image/tiff", ".tif": "image/tif"}
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
var (
|
||||||
// Ensure that the ReaderAt interface is implemented.
|
// Ensure that the ReaderAt interface is implemented.
|
||||||
_ io.ReaderAt = (*s3WithCancel)(nil)
|
_ io.ReaderAt = (*s3WithCancel)(nil)
|
||||||
_ FileBackendWithLinkGenerator = (*S3FileBackend)(nil)
|
_ FileBackendWithLinkGenerator = (*S3FileBackend)(nil)
|
||||||
)
|
)
|
||||||
|
|
||||||
func isFileExtImage(ext string) bool {
|
func getContentType(ext string) string {
|
||||||
ext = strings.ToLower(ext)
|
mimeType := mime.TypeByExtension(strings.ToLower(ext))
|
||||||
return imageExtensions[ext]
|
if mimeType == "" {
|
||||||
}
|
mimeType = "application/octet-stream"
|
||||||
|
|
||||||
func getImageMimeType(ext string) string {
|
|
||||||
ext = strings.ToLower(ext)
|
|
||||||
if imageMimeTypes[ext] == "" {
|
|
||||||
return "image"
|
|
||||||
}
|
}
|
||||||
return imageMimeTypes[ext]
|
return mimeType
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *S3FileBackendAuthError) Error() string {
|
func (s *S3FileBackendAuthError) Error() string {
|
||||||
@@ -502,11 +493,8 @@ func (b *S3FileBackend) WriteFile(fr io.Reader, path string) (int64, error) {
|
|||||||
func (b *S3FileBackend) WriteFileContext(ctx context.Context, fr io.Reader, path string) (int64, error) {
|
func (b *S3FileBackend) WriteFileContext(ctx context.Context, fr io.Reader, path string) (int64, error) {
|
||||||
var contentType string
|
var contentType string
|
||||||
path = filepath.Join(b.pathPrefix, path)
|
path = filepath.Join(b.pathPrefix, path)
|
||||||
if ext := filepath.Ext(path); isFileExtImage(ext) {
|
ext := filepath.Ext(path)
|
||||||
contentType = getImageMimeType(ext)
|
contentType = getContentType(ext)
|
||||||
} else {
|
|
||||||
contentType = "binary/octet-stream"
|
|
||||||
}
|
|
||||||
|
|
||||||
options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize, b.storageClass)
|
options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize, b.storageClass)
|
||||||
|
|
||||||
@@ -545,12 +533,7 @@ func (b *S3FileBackend) AppendFile(fr io.Reader, path string) (int64, error) {
|
|||||||
return 0, errors.Wrapf(err2, "unable to find the file %s to append the data", path)
|
return 0, errors.Wrapf(err2, "unable to find the file %s to append the data", path)
|
||||||
}
|
}
|
||||||
|
|
||||||
var contentType string
|
contentType := getContentType(filepath.Ext(fp))
|
||||||
if ext := filepath.Ext(fp); isFileExtImage(ext) {
|
|
||||||
contentType = getImageMimeType(ext)
|
|
||||||
} else {
|
|
||||||
contentType = "binary/octet-stream"
|
|
||||||
}
|
|
||||||
|
|
||||||
options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize, b.storageClass)
|
options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize, b.storageClass)
|
||||||
sse := options.ServerSideEncryption
|
sse := options.ServerSideEncryption
|
||||||
|
|||||||
@@ -379,3 +379,78 @@ func TestListDirectory(t *testing.T) {
|
|||||||
err = fileBackend.RemoveDirectory("19700101")
|
err = fileBackend.RemoveDirectory("19700101")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestWriteFileVideoMimeTypes(t *testing.T) {
|
||||||
|
s3Host := os.Getenv("CI_MINIO_HOST")
|
||||||
|
if s3Host == "" {
|
||||||
|
s3Host = "localhost"
|
||||||
|
}
|
||||||
|
|
||||||
|
s3Port := os.Getenv("CI_MINIO_PORT")
|
||||||
|
if s3Port == "" {
|
||||||
|
s3Port = "9000"
|
||||||
|
}
|
||||||
|
|
||||||
|
s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
|
||||||
|
|
||||||
|
// Generate a random bucket name
|
||||||
|
b := make([]byte, 30)
|
||||||
|
rand.Read(b)
|
||||||
|
bucketName := base64.StdEncoding.EncodeToString(b)
|
||||||
|
bucketName = strings.ToLower(bucketName)
|
||||||
|
bucketName = strings.Replace(bucketName, "+", "", -1)
|
||||||
|
bucketName = strings.Replace(bucketName, "/", "", -1)
|
||||||
|
|
||||||
|
cfg := FileBackendSettings{
|
||||||
|
DriverName: model.ImageDriverS3,
|
||||||
|
AmazonS3AccessKeyId: model.MinioAccessKey,
|
||||||
|
AmazonS3SecretAccessKey: model.MinioSecretKey,
|
||||||
|
AmazonS3Bucket: bucketName,
|
||||||
|
AmazonS3Endpoint: s3Endpoint,
|
||||||
|
AmazonS3SSL: false,
|
||||||
|
AmazonS3RequestTimeoutMilliseconds: 5000,
|
||||||
|
}
|
||||||
|
|
||||||
|
fileBackend, err := NewS3FileBackend(cfg)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
err = fileBackend.MakeBucket()
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Test video types
|
||||||
|
// Test video types with multiple valid MIME types
|
||||||
|
testContent := []byte("test-video-content")
|
||||||
|
videoTypes := map[string][]string{
|
||||||
|
".avi": {"video/vnd.avi", "video/x-msvideo"},
|
||||||
|
".mpeg": {"video/mpeg"},
|
||||||
|
".mp4": {"video/mp4"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for ext, validMimeTypes := range videoTypes {
|
||||||
|
t.Run(strings.TrimPrefix(ext, "."), func(t *testing.T) {
|
||||||
|
path := "test" + ext
|
||||||
|
reader := bytes.NewReader(testContent)
|
||||||
|
|
||||||
|
written, err := fileBackend.WriteFile(reader, path)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, int64(len(testContent)), written)
|
||||||
|
|
||||||
|
// Verify the file exists with correct mime type
|
||||||
|
props, err := fileBackend.client.StatObject(
|
||||||
|
context.Background(),
|
||||||
|
bucketName,
|
||||||
|
path,
|
||||||
|
s3.StatObjectOptions{},
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Ensure the MIME type is one of the expected values
|
||||||
|
assert.Contains(t, validMimeTypes, props.ContentType, "Unexpected MIME type: %s", props.ContentType)
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
err = fileBackend.RemoveFile(path)
|
||||||
|
require.NoError(t, err)
|
||||||
|
}()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Ссылка в новой задаче
Block a user