mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

Moving away from goamz to use minio-go instead. (#4193)

Просмотр исходного кода 
minio-go does fully managed way of handling S3 API requests

  - Automatic bucket location management across all s3 regions.
  - Transparently upload large files in multipart if file 64MB
    or larger.
  - Right GetObject() API provides compatibility with
    io.ReadWriteSeeker interface.
  - Various other APIs including bulk deletes, server side object
    copy, bucket policies and bucket notifications.

Fixes #4182
Этот коммит содержится в:
Harshavardhana
2016-10-26 05:21:07 -07:00
коммит произвёл Christopher Speller
родитель b354d25d37
Коммит f02620b291
100 изменённых файлов: 19497 добавлений и 517 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

110
api/file.go
Просмотреть файл

@@ -18,21 +18,21 @@ import (
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
l4g "github.com/alecthomas/log4go"
"github.com/disintegration/imaging"
"github.com/goamz/goamz/aws"
"github.com/goamz/goamz/s3"
"github.com/gorilla/mux"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/utils"
"github.com/rwcarlsen/goexif/exif"
_ "golang.org/x/image/bmp"
s3 "github.com/minio/minio-go"
)
const (
@@ -738,25 +738,22 @@ func getInfoForFilename(post *model.Post, teamId string, filename string) *model
func WriteFile(f []byte, path string) *model.AppError {
if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
var auth aws.Auth
auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey
s := s3.New(auth, awsRegion())
bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket)
endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint
accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId
secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey
secure := *utils.Cfg.FileSettings.AmazonS3SSL
s3Clnt, err := s3.New(endpoint, accessKey, secretKey, secure)
if err != nil {
return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error())
}
bucket := utils.Cfg.FileSettings.AmazonS3Bucket
ext := filepath.Ext(path)
var err error
if model.IsFileExtImage(ext) {
options := s3.Options{}
err = bucket.Put(path, f, model.GetImageMimeType(ext), s3.Private, options)
_, err = s3Clnt.PutObject(bucket, path, bytes.NewReader(f), model.GetImageMimeType(ext))
} else {
options := s3.Options{}
err = bucket.Put(path, f, "binary/octet-stream", s3.Private, options)
_, err = s3Clnt.PutObject(bucket, path, bytes.NewReader(f), "binary/octet-stream")
}
if err != nil {
return model.NewLocAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error())
}
@@ -773,28 +770,24 @@ func WriteFile(f []byte, path string) *model.AppError {
func MoveFile(oldPath, newPath string) *model.AppError {
if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
fileBytes, _ := ReadFile(oldPath)
if fileBytes == nil {
return model.NewLocAppError("moveFile", "api.file.move_file.get_from_s3.app_error", nil, "")
endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint
accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId
secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey
secure := *utils.Cfg.FileSettings.AmazonS3SSL
s3Clnt, err := s3.New(endpoint, accessKey, secretKey, secure)
if err != nil {
return model.NewLocAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error())
}
bucket := utils.Cfg.FileSettings.AmazonS3Bucket
var auth aws.Auth
auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey
s := s3.New(auth, awsRegion())
bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket)
if err := bucket.Del(oldPath); err != nil {
var copyConds = s3.NewCopyConditions()
if err = s3Clnt.CopyObject(bucket, newPath, "/"+path.Join(bucket, oldPath), copyConds); err != nil {
return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error())
}
if err := WriteFile(fileBytes, newPath); err != nil {
return err
if err = s3Clnt.RemoveObject(bucket, oldPath); err != nil {
return model.NewLocAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error())
}
} else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
if err := os.MkdirAll(filepath.Dir(utils.Cfg.FileSettings.Directory+newPath), 0774); err != nil {
return model.NewLocAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error())
}
@@ -824,26 +817,23 @@ func writeFileLocally(f []byte, path string) *model.AppError {
func ReadFile(path string) ([]byte, *model.AppError) {
if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
var auth aws.Auth
auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey
s := s3.New(auth, awsRegion())
bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket)
// try to get the file from S3 with some basic retry logic
tries := 0
for {
tries++
f, err := bucket.Get(path)
if f != nil {
return f, nil
} else if tries >= 3 {
return nil, model.NewLocAppError("ReadFile", "api.file.read_file.get.app_error", nil, "path="+path+", err="+err.Error())
}
time.Sleep(3000 * time.Millisecond)
endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint
accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId
secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey
secure := *utils.Cfg.FileSettings.AmazonS3SSL
s3Clnt, err := s3.New(endpoint, accessKey, secretKey, secure)
if err != nil {
return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error())
}
bucket := utils.Cfg.FileSettings.AmazonS3Bucket
reader, err := s3Clnt.GetObject(bucket, path)
if err != nil {
return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error())
}
if f, err := ioutil.ReadAll(reader); err != nil {
return nil, model.NewLocAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error())
} else {
return f, nil
}
} else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
if f, err := ioutil.ReadFile(utils.Cfg.FileSettings.Directory + path); err != nil {
@@ -878,17 +868,3 @@ func openFileWriteStream(path string) (io.Writer, *model.AppError) {
func closeFileWriteStream(file io.Writer) {
file.(*os.File).Close()
}
func awsRegion() aws.Region {
if region, ok := aws.Regions[utils.Cfg.FileSettings.AmazonS3Region]; ok {
return region
}
return aws.Region{
Name: utils.Cfg.FileSettings.AmazonS3Region,
S3Endpoint: utils.Cfg.FileSettings.AmazonS3Endpoint,
S3BucketEndpoint: utils.Cfg.FileSettings.AmazonS3BucketEndpoint,
S3LocationConstraint: *utils.Cfg.FileSettings.AmazonS3LocationConstraint,
S3LowercaseBucket: *utils.Cfg.FileSettings.AmazonS3LowercaseBucket,
}
}

33
api/file_test.go
Просмотреть файл

@@ -6,11 +6,6 @@ package api
import (
"bytes"
"fmt"
"github.com/goamz/goamz/aws"
"github.com/goamz/goamz/s3"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/store"
"github.com/mattermost/platform/utils"
"io"
"io/ioutil"
"net/http"
@@ -18,6 +13,12 @@ import (
"strings"
"testing"
"time"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/store"
"github.com/mattermost/platform/utils"
s3 "github.com/minio/minio-go"
)
func TestUploadFile(t *testing.T) {
@@ -830,25 +831,27 @@ func readTestFile(name string) ([]byte, error) {
func cleanupTestFile(info *model.FileInfo) error {
if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
var auth aws.Auth
auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.FileSettings.AmazonS3Region])
bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket)
if err := bucket.Del(info.Path); err != nil {
endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint
accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId
secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey
secure := *utils.Cfg.FileSettings.AmazonS3SSL
s3Clnt, err := s3.New(endpoint, accessKey, secretKey, secure)
if err != nil {
return err
}
bucket := utils.Cfg.FileSettings.AmazonS3Bucket
if err := s3Clnt.RemoveObject(bucket, info.Path); err != nil {
return err
}
if info.ThumbnailPath != "" {
if err := bucket.Del(info.ThumbnailPath); err != nil {
if err := s3Clnt.RemoveObject(bucket, info.ThumbnailPath); err != nil {
return err
}
}
if info.PreviewPath != "" {
if err := bucket.Del(info.PreviewPath); err != nil {
if err := s3Clnt.RemoveObject(bucket, info.PreviewPath); err != nil {
return err
}
}

2
api/user.go
Просмотреть файл

@@ -1238,7 +1238,7 @@ func getProfileImage(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
} else {
path := "/users/" + id + "/profile.png"
path := "users/" + id + "/profile.png"
if data, err := ReadFile(path); err != nil {

41
api/user_test.go
Просмотреть файл

@@ -16,12 +16,11 @@ import (
"testing"
"time"
"github.com/goamz/goamz/aws"
"github.com/goamz/goamz/s3"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/store"
"github.com/mattermost/platform/utils"
s3 "github.com/minio/minio-go"
)
func TestCreateUser(t *testing.T) {
@@ -674,14 +673,16 @@ func TestUserCreateImage(t *testing.T) {
Client.DoApiGet("/users/"+user.Id+"/image", "", "")
if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
var auth aws.Auth
auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.FileSettings.AmazonS3Region])
bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket)
if err := bucket.Del("/users/" + user.Id + "/profile.png"); err != nil {
endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint
accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId
secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey
secure := *utils.Cfg.FileSettings.AmazonS3SSL
s3Clnt, err := s3.New(endpoint, accessKey, secretKey, secure)
if err != nil {
t.Fatal(err)
}
bucket := utils.Cfg.FileSettings.AmazonS3Bucket
if err = s3Clnt.RemoveObject(bucket, "/users/"+user.Id+"/profile.png"); err != nil {
t.Fatal(err)
}
} else {
@@ -774,14 +775,16 @@ func TestUserUploadProfileImage(t *testing.T) {
Client.DoApiGet("/users/"+user.Id+"/image", "", "")
if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
var auth aws.Auth
auth.AccessKey = utils.Cfg.FileSettings.AmazonS3AccessKeyId
auth.SecretKey = utils.Cfg.FileSettings.AmazonS3SecretAccessKey
s := s3.New(auth, aws.Regions[utils.Cfg.FileSettings.AmazonS3Region])
bucket := s.Bucket(utils.Cfg.FileSettings.AmazonS3Bucket)
if err := bucket.Del("users/" + user.Id + "/profile.png"); err != nil {
endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint
accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId
secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey
secure := *utils.Cfg.FileSettings.AmazonS3SSL
s3Clnt, err := s3.New(endpoint, accessKey, secretKey, secure)
if err != nil {
t.Fatal(err)
}
bucket := utils.Cfg.FileSettings.AmazonS3Bucket
if err = s3Clnt.RemoveObject(bucket, "/users/"+user.Id+"/profile.png"); err != nil {
t.Fatal(err)
}
} else {