mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

[MM-27328] Upgrade minio-go to v7 (#15177)

Просмотр исходного кода 
* Upgrade minio-go to v7

* Update to v7.0.3

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
Этот коммит содержится в:
Claudio Costa
2020-08-12 19:53:47 +02:00
коммит произвёл GitHub
родитель 5566395032
Коммит c89c56ab2e
166 изменённых файлов: 5005 добавлений и 3002 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

11
services/filesstore/filesstore_test.go
Просмотреть файл

@@ -158,11 +158,14 @@ func (s *FileBackendTestSuite) TestCopyFile() {
s.Nil(err)
defer s.backend.RemoveFile(path2)
_, err = s.backend.ReadFile(path1)
data1, err := s.backend.ReadFile(path1)
s.Nil(err)
_, err = s.backend.ReadFile(path2)
data2, err := s.backend.ReadFile(path2)
s.Nil(err)
s.Equal(b, data1)
s.Equal(b, data2)
}
func (s *FileBackendTestSuite) TestCopyFileToDirectoryThatDoesntExist() {
@@ -202,8 +205,10 @@ func (s *FileBackendTestSuite) TestMoveFile() {
_, err = s.backend.ReadFile(path1)
s.Error(err)
_, err = s.backend.ReadFile(path2)
data, err := s.backend.ReadFile(path2)
s.Nil(err)
s.Equal(b, data)
}
func (s *FileBackendTestSuite) TestRemoveFile() {

95
services/filesstore/s3store.go
Просмотреть файл

@@ -5,6 +5,7 @@ package filesstore
import (
"bytes"
"context"
"io"
"io/ioutil"
"net/http"
@@ -12,9 +13,9 @@ import (
"path/filepath"
"strings"
s3 "github.com/minio/minio-go/v6"
"github.com/minio/minio-go/v6/pkg/credentials"
"github.com/minio/minio-go/v6/pkg/encrypt"
s3 "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/encrypt"
"github.com/mattermost/mattermost-server/v5/mlog"
"github.com/mattermost/mattermost-server/v5/model"
@@ -49,7 +50,12 @@ func (b *S3FileBackend) s3New() (*s3.Client, error) {
creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV4)
}
s3Clnt, err := s3.NewWithCredentials(b.endpoint, creds, b.secure, b.region)
opts := s3.Options{
Creds: creds,
Secure: b.secure,
Region: b.region,
}
s3Clnt, err := s3.New(b.endpoint, &opts)
if err != nil {
return nil, err
}
@@ -67,14 +73,14 @@ func (b *S3FileBackend) TestConnection() *model.AppError {
return model.NewAppError("TestFileConnection", "api.file.test_connection.s3.connection.app_error", nil, err.Error(), http.StatusInternalServerError)
}
exists, err := s3Clnt.BucketExists(b.bucket)
exists, err := s3Clnt.BucketExists(context.Background(), b.bucket)
if err != nil {
return model.NewAppError("TestFileConnection", "api.file.test_connection.s3.bucket_exists.app_error", nil, err.Error(), http.StatusInternalServerError)
}
if !exists {
mlog.Warn("Bucket specified does not exist. Attempting to create...")
err := s3Clnt.MakeBucket(b.bucket, b.region)
err := s3Clnt.MakeBucket(context.Background(), b.bucket, s3.MakeBucketOptions{Region: b.region})
if err != nil {
mlog.Error("Unable to create bucket.")
return model.NewAppError("TestFileConnection", "api.file.test_connection.s3.bucked_create.app_error", nil, err.Error(), http.StatusInternalServerError)
@@ -92,7 +98,7 @@ func (b *S3FileBackend) Reader(path string) (ReadCloseSeeker, *model.AppError) {
}
path = filepath.Join(b.pathPrefix, path)
minioObject, err := s3Clnt.GetObject(b.bucket, path, s3.GetObjectOptions{})
minioObject, err := s3Clnt.GetObject(context.Background(), b.bucket, path, s3.GetObjectOptions{})
if err != nil {
return nil, model.NewAppError("Reader", "api.file.reader.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
@@ -107,7 +113,7 @@ func (b *S3FileBackend) ReadFile(path string) ([]byte, *model.AppError) {
}
path = filepath.Join(b.pathPrefix, path)
minioObject, err := s3Clnt.GetObject(b.bucket, path, s3.GetObjectOptions{})
minioObject, err := s3Clnt.GetObject(context.Background(), b.bucket, path, s3.GetObjectOptions{})
if err != nil {
return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
@@ -128,7 +134,7 @@ func (b *S3FileBackend) FileExists(path string) (bool, *model.AppError) {
}
path = filepath.Join(b.pathPrefix, path)
_, err = s3Clnt.StatObject(b.bucket, path, s3.StatObjectOptions{})
_, err = s3Clnt.StatObject(context.Background(), b.bucket, path, s3.StatObjectOptions{})
if err == nil {
return true, nil
@@ -149,14 +155,17 @@ func (b *S3FileBackend) CopyFile(oldPath, newPath string) *model.AppError {
oldPath = filepath.Join(b.pathPrefix, oldPath)
newPath = filepath.Join(b.pathPrefix, newPath)
source := s3.NewSourceInfo(b.bucket, oldPath, nil)
destination, err := s3.NewDestinationInfo(b.bucket, newPath, encrypt.NewSSE(), nil)
if err != nil {
return model.NewAppError("copyFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
srcOpts := s3.CopySrcOptions{
Bucket: b.bucket,
Object: oldPath,
Encryption: encrypt.NewSSE(),
}
if err = s3Clnt.CopyObject(destination, source); err != nil {
dstOpts := s3.CopyDestOptions{
Bucket: b.bucket,
Object: newPath,
Encryption: encrypt.NewSSE(),
}
if _, err = s3Clnt.CopyObject(context.Background(), dstOpts, srcOpts); err != nil {
return model.NewAppError("copyFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
return nil
@@ -170,18 +179,22 @@ func (b *S3FileBackend) MoveFile(oldPath, newPath string) *model.AppError {
oldPath = filepath.Join(b.pathPrefix, oldPath)
newPath = filepath.Join(b.pathPrefix, newPath)
source := s3.NewSourceInfo(b.bucket, oldPath, nil)
destination, err := s3.NewDestinationInfo(b.bucket, newPath, encrypt.NewSSE(), nil)
if err != nil {
return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
srcOpts := s3.CopySrcOptions{
Bucket: b.bucket,
Object: oldPath,
Encryption: encrypt.NewSSE(),
}
dstOpts := s3.CopyDestOptions{
Bucket: b.bucket,
Object: newPath,
Encryption: encrypt.NewSSE(),
}
if err = s3Clnt.CopyObject(destination, source); err != nil {
if _, err = s3Clnt.CopyObject(context.Background(), dstOpts, srcOpts); err != nil {
return model.NewAppError("moveFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
if err = s3Clnt.RemoveObject(b.bucket, oldPath); err != nil {
if err = s3Clnt.RemoveObject(context.Background(), b.bucket, oldPath, s3.RemoveObjectOptions{}); err != nil {
return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
@@ -208,12 +221,12 @@ func (b *S3FileBackend) WriteFile(fr io.Reader, path string) (int64, *model.AppE
if err != nil {
return 0, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
written, err := s3Clnt.PutObject(b.bucket, path, &buf, int64(buf.Len()), options)
info, err := s3Clnt.PutObject(context.Background(), b.bucket, path, &buf, int64(buf.Len()), options)
if err != nil {
return written, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
return info.Size, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
return written, nil
return info.Size, nil
}
func (b *S3FileBackend) RemoveFile(path string) *model.AppError {
@@ -223,15 +236,15 @@ func (b *S3FileBackend) RemoveFile(path string) *model.AppError {
}
path = filepath.Join(b.pathPrefix, path)
if err := s3Clnt.RemoveObject(b.bucket, path); err != nil {
if err := s3Clnt.RemoveObject(context.Background(), b.bucket, path, s3.RemoveObjectOptions{}); err != nil {
return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
return nil
}
func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan string {
out := make(chan string, 1)
func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan s3.ObjectInfo {
out := make(chan s3.ObjectInfo, 1)
go func() {
defer close(out)
@@ -243,7 +256,7 @@ func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan string {
break
}
out <- info.Key
out <- info
}
}()
@@ -258,9 +271,6 @@ func (b *S3FileBackend) ListDirectory(path string) (*[]string, *model.AppError)
return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
doneCh := make(chan struct{})
defer close(doneCh)
path = filepath.Join(b.pathPrefix, path)
if !strings.HasSuffix(path, "/") && len(path) > 0 {
// s3Clnt returns only the path itself when "/" is not present
@@ -268,7 +278,10 @@ func (b *S3FileBackend) ListDirectory(path string) (*[]string, *model.AppError)
path = path + "/"
}
for object := range s3Clnt.ListObjects(b.bucket, path, false, doneCh) {
opts := s3.ListObjectsOptions{
Prefix: path,
}
for object := range s3Clnt.ListObjects(context.Background(), b.bucket, opts) {
if object.Err != nil {
return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, object.Err.Error(), http.StatusInternalServerError)
}
@@ -283,18 +296,18 @@ func (b *S3FileBackend) RemoveDirectory(path string) *model.AppError {
if err != nil {
return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
doneCh := make(chan struct{})
path = filepath.Join(b.pathPrefix, path)
for err := range s3Clnt.RemoveObjects(b.bucket, getPathsFromObjectInfos(s3Clnt.ListObjects(b.bucket, path, true, doneCh))) {
opts := s3.ListObjectsOptions{
Prefix: filepath.Join(b.pathPrefix, path),
Recursive: true,
}
list := s3Clnt.ListObjects(context.Background(), b.bucket, opts)
objectsCh := s3Clnt.RemoveObjects(context.Background(), b.bucket, getPathsFromObjectInfos(list), s3.RemoveObjectsOptions{})
for err := range objectsCh {
if err.Err != nil {
doneCh <- struct{}{}
return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Err.Error(), http.StatusInternalServerError)
}
}
close(doneCh)
return nil
}