Move file backend to its own service (#9435)
* Move file backend to its own service * Moving utils/inbucket to mailservice package
Этот коммит содержится в:
Jesús Espino
коммит произвёл
Christopher Speller
Christopher Speller
родитель
f1be975d7a
Коммит
a08df883b4
48
services/filesstore/filesstore.go
Обычный файл
48
services/filesstore/filesstore.go
Обычный файл
@@ -0,0 +1,48 @@
|
||||
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
|
||||
// See License.txt for license information.
|
||||
|
||||
package filesstore
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/mattermost/mattermost-server/model"
|
||||
)
|
||||
|
||||
type FileBackend interface {
|
||||
TestConnection() *model.AppError
|
||||
|
||||
Reader(path string) (io.ReadCloser, *model.AppError)
|
||||
ReadFile(path string) ([]byte, *model.AppError)
|
||||
FileExists(path string) (bool, *model.AppError)
|
||||
CopyFile(oldPath, newPath string) *model.AppError
|
||||
MoveFile(oldPath, newPath string) *model.AppError
|
||||
WriteFile(fr io.Reader, path string) (int64, *model.AppError)
|
||||
RemoveFile(path string) *model.AppError
|
||||
|
||||
ListDirectory(path string) (*[]string, *model.AppError)
|
||||
RemoveDirectory(path string) *model.AppError
|
||||
}
|
||||
|
||||
func NewFileBackend(settings *model.FileSettings, enableComplianceFeatures bool) (FileBackend, *model.AppError) {
|
||||
switch *settings.DriverName {
|
||||
case model.IMAGE_DRIVER_S3:
|
||||
return &S3FileBackend{
|
||||
endpoint: settings.AmazonS3Endpoint,
|
||||
accessKey: settings.AmazonS3AccessKeyId,
|
||||
secretKey: settings.AmazonS3SecretAccessKey,
|
||||
secure: settings.AmazonS3SSL == nil || *settings.AmazonS3SSL,
|
||||
signV2: settings.AmazonS3SignV2 != nil && *settings.AmazonS3SignV2,
|
||||
region: settings.AmazonS3Region,
|
||||
bucket: settings.AmazonS3Bucket,
|
||||
encrypt: settings.AmazonS3SSE != nil && *settings.AmazonS3SSE && enableComplianceFeatures,
|
||||
trace: settings.AmazonS3Trace != nil && *settings.AmazonS3Trace,
|
||||
}, nil
|
||||
case model.IMAGE_DRIVER_LOCAL:
|
||||
return &LocalFileBackend{
|
||||
directory: settings.Directory,
|
||||
}, nil
|
||||
}
|
||||
return nil, model.NewAppError("NewFileBackend", "api.file.no_driver.app_error", nil, "", http.StatusInternalServerError)
|
||||
}
|
||||
288
services/filesstore/filesstore_test.go
Обычный файл
288
services/filesstore/filesstore_test.go
Обычный файл
@@ -0,0 +1,288 @@
|
||||
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
|
||||
// See License.txt for license information.
|
||||
|
||||
package filesstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/mattermost/mattermost-server/mlog"
|
||||
"github.com/mattermost/mattermost-server/model"
|
||||
"github.com/mattermost/mattermost-server/utils"
|
||||
)
|
||||
|
||||
type FileBackendTestSuite struct {
|
||||
suite.Suite
|
||||
|
||||
settings model.FileSettings
|
||||
backend FileBackend
|
||||
}
|
||||
|
||||
func TestLocalFileBackendTestSuite(t *testing.T) {
|
||||
// Setup a global logger to catch tests logging outside of app context
|
||||
// The global logger will be stomped by apps initalizing but that's fine for testing. Ideally this won't happen.
|
||||
mlog.InitGlobalLogger(mlog.NewLogger(&mlog.LoggerConfiguration{
|
||||
EnableConsole: true,
|
||||
ConsoleJson: true,
|
||||
ConsoleLevel: "error",
|
||||
EnableFile: false,
|
||||
}))
|
||||
|
||||
dir, err := ioutil.TempDir("", "")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
suite.Run(t, &FileBackendTestSuite{
|
||||
settings: model.FileSettings{
|
||||
DriverName: model.NewString(model.IMAGE_DRIVER_LOCAL),
|
||||
Directory: dir,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func TestS3FileBackendTestSuite(t *testing.T) {
|
||||
runBackendTest(t, false)
|
||||
}
|
||||
|
||||
func TestS3FileBackendTestSuiteWithEncryption(t *testing.T) {
|
||||
runBackendTest(t, true)
|
||||
}
|
||||
|
||||
func runBackendTest(t *testing.T, encrypt bool) {
|
||||
s3Host := os.Getenv("CI_HOST")
|
||||
if s3Host == "" {
|
||||
s3Host = "dockerhost"
|
||||
}
|
||||
|
||||
s3Port := os.Getenv("CI_MINIO_PORT")
|
||||
if s3Port == "" {
|
||||
s3Port = "9001"
|
||||
}
|
||||
|
||||
s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
|
||||
|
||||
suite.Run(t, &FileBackendTestSuite{
|
||||
settings: model.FileSettings{
|
||||
DriverName: model.NewString(model.IMAGE_DRIVER_S3),
|
||||
AmazonS3AccessKeyId: model.MINIO_ACCESS_KEY,
|
||||
AmazonS3SecretAccessKey: model.MINIO_SECRET_KEY,
|
||||
AmazonS3Bucket: model.MINIO_BUCKET,
|
||||
AmazonS3Endpoint: s3Endpoint,
|
||||
AmazonS3SSL: model.NewBool(false),
|
||||
AmazonS3SSE: model.NewBool(encrypt),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) SetupTest() {
|
||||
utils.TranslationsPreInit()
|
||||
|
||||
backend, err := NewFileBackend(&s.settings, true)
|
||||
require.Nil(s.T(), err)
|
||||
s.backend = backend
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestConnection() {
|
||||
s.Nil(s.backend.TestConnection())
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestReadWriteFile() {
|
||||
b := []byte("test")
|
||||
path := "tests/" + model.NewId()
|
||||
|
||||
written, err := s.backend.WriteFile(bytes.NewReader(b), path)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
defer s.backend.RemoveFile(path)
|
||||
|
||||
read, err := s.backend.ReadFile(path)
|
||||
s.Nil(err)
|
||||
|
||||
readString := string(read)
|
||||
s.EqualValues(readString, "test")
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestReadWriteFileImage() {
|
||||
b := []byte("testimage")
|
||||
path := "tests/" + model.NewId() + ".png"
|
||||
|
||||
written, err := s.backend.WriteFile(bytes.NewReader(b), path)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
defer s.backend.RemoveFile(path)
|
||||
|
||||
read, err := s.backend.ReadFile(path)
|
||||
s.Nil(err)
|
||||
|
||||
readString := string(read)
|
||||
s.EqualValues(readString, "testimage")
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestFileExists() {
|
||||
b := []byte("testimage")
|
||||
path := "tests/" + model.NewId() + ".png"
|
||||
|
||||
_, err := s.backend.WriteFile(bytes.NewReader(b), path)
|
||||
s.Nil(err)
|
||||
defer s.backend.RemoveFile(path)
|
||||
|
||||
res, err := s.backend.FileExists(path)
|
||||
s.Nil(err)
|
||||
s.True(res)
|
||||
|
||||
res, err = s.backend.FileExists("tests/idontexist.png")
|
||||
s.Nil(err)
|
||||
s.False(res)
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestCopyFile() {
|
||||
b := []byte("test")
|
||||
path1 := "tests/" + model.NewId()
|
||||
path2 := "tests/" + model.NewId()
|
||||
|
||||
written, err := s.backend.WriteFile(bytes.NewReader(b), path1)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
defer s.backend.RemoveFile(path1)
|
||||
|
||||
err = s.backend.CopyFile(path1, path2)
|
||||
s.Nil(err)
|
||||
defer s.backend.RemoveFile(path2)
|
||||
|
||||
_, err = s.backend.ReadFile(path1)
|
||||
s.Nil(err)
|
||||
|
||||
_, err = s.backend.ReadFile(path2)
|
||||
s.Nil(err)
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestCopyFileToDirectoryThatDoesntExist() {
|
||||
b := []byte("test")
|
||||
path1 := "tests/" + model.NewId()
|
||||
path2 := "tests/newdirectory/" + model.NewId()
|
||||
|
||||
written, err := s.backend.WriteFile(bytes.NewReader(b), path1)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
defer s.backend.RemoveFile(path1)
|
||||
|
||||
err = s.backend.CopyFile(path1, path2)
|
||||
s.Nil(err)
|
||||
defer s.backend.RemoveFile(path2)
|
||||
|
||||
_, err = s.backend.ReadFile(path1)
|
||||
s.Nil(err)
|
||||
|
||||
_, err = s.backend.ReadFile(path2)
|
||||
s.Nil(err)
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestMoveFile() {
|
||||
b := []byte("test")
|
||||
path1 := "tests/" + model.NewId()
|
||||
path2 := "tests/" + model.NewId()
|
||||
|
||||
written, err := s.backend.WriteFile(bytes.NewReader(b), path1)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
defer s.backend.RemoveFile(path1)
|
||||
|
||||
s.Nil(s.backend.MoveFile(path1, path2))
|
||||
defer s.backend.RemoveFile(path2)
|
||||
|
||||
_, err = s.backend.ReadFile(path1)
|
||||
s.Error(err)
|
||||
|
||||
_, err = s.backend.ReadFile(path2)
|
||||
s.Nil(err)
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestRemoveFile() {
|
||||
b := []byte("test")
|
||||
path := "tests/" + model.NewId()
|
||||
|
||||
written, err := s.backend.WriteFile(bytes.NewReader(b), path)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
s.Nil(s.backend.RemoveFile(path))
|
||||
|
||||
_, err = s.backend.ReadFile(path)
|
||||
s.Error(err)
|
||||
|
||||
written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/foo")
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
|
||||
written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/bar")
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
|
||||
written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/asdf")
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
|
||||
s.Nil(s.backend.RemoveDirectory("tests2"))
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestListDirectory() {
|
||||
b := []byte("test")
|
||||
path1 := "19700101/" + model.NewId()
|
||||
path2 := "19800101/" + model.NewId()
|
||||
|
||||
written, err := s.backend.WriteFile(bytes.NewReader(b), path1)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
defer s.backend.RemoveFile(path1)
|
||||
|
||||
written, err = s.backend.WriteFile(bytes.NewReader(b), path2)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
defer s.backend.RemoveFile(path2)
|
||||
|
||||
paths, err := s.backend.ListDirectory("")
|
||||
s.Nil(err)
|
||||
|
||||
found1 := false
|
||||
found2 := false
|
||||
for _, path := range *paths {
|
||||
if path == "19700101" {
|
||||
found1 = true
|
||||
} else if path == "19800101" {
|
||||
found2 = true
|
||||
}
|
||||
}
|
||||
s.True(found1)
|
||||
s.True(found2)
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestRemoveDirectory() {
|
||||
b := []byte("test")
|
||||
|
||||
written, err := s.backend.WriteFile(bytes.NewReader(b), "tests2/foo")
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
|
||||
written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/bar")
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
|
||||
written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/aaa")
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
|
||||
s.Nil(s.backend.RemoveDirectory("tests2"))
|
||||
|
||||
_, err = s.backend.ReadFile("tests2/foo")
|
||||
s.Error(err)
|
||||
_, err = s.backend.ReadFile("tests2/bar")
|
||||
s.Error(err)
|
||||
_, err = s.backend.ReadFile("tests2/asdf")
|
||||
s.Error(err)
|
||||
}
|
||||
131
services/filesstore/localstore.go
Обычный файл
131
services/filesstore/localstore.go
Обычный файл
@@ -0,0 +1,131 @@
|
||||
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
|
||||
// See License.txt for license information.
|
||||
|
||||
package filesstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/mattermost/mattermost-server/mlog"
|
||||
"github.com/mattermost/mattermost-server/model"
|
||||
"github.com/mattermost/mattermost-server/utils"
|
||||
)
|
||||
|
||||
const (
|
||||
TEST_FILE_PATH = "/testfile"
|
||||
)
|
||||
|
||||
type LocalFileBackend struct {
|
||||
directory string
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) TestConnection() *model.AppError {
|
||||
f := bytes.NewReader([]byte("testingwrite"))
|
||||
if _, err := writeFileLocally(f, filepath.Join(b.directory, TEST_FILE_PATH)); err != nil {
|
||||
return model.NewAppError("TestFileConnection", "api.file.test_connection.local.connection.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
os.Remove(filepath.Join(b.directory, TEST_FILE_PATH))
|
||||
mlog.Info("Able to write files to local storage.")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) Reader(path string) (io.ReadCloser, *model.AppError) {
|
||||
if f, err := os.Open(filepath.Join(b.directory, path)); err != nil {
|
||||
return nil, model.NewAppError("Reader", "api.file.reader.reading_local.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
} else {
|
||||
return f, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) ReadFile(path string) ([]byte, *model.AppError) {
|
||||
if f, err := ioutil.ReadFile(filepath.Join(b.directory, path)); err != nil {
|
||||
return nil, model.NewAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
} else {
|
||||
return f, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) FileExists(path string) (bool, *model.AppError) {
|
||||
_, err := os.Stat(filepath.Join(b.directory, path))
|
||||
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
} else if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
return false, model.NewAppError("ReadFile", "api.file.file_exists.exists_local.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) CopyFile(oldPath, newPath string) *model.AppError {
|
||||
if err := utils.CopyFile(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil {
|
||||
return model.NewAppError("copyFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) MoveFile(oldPath, newPath string) *model.AppError {
|
||||
if err := os.MkdirAll(filepath.Dir(filepath.Join(b.directory, newPath)), 0774); err != nil {
|
||||
return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
if err := os.Rename(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil {
|
||||
return model.NewAppError("moveFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) WriteFile(fr io.Reader, path string) (int64, *model.AppError) {
|
||||
return writeFileLocally(fr, filepath.Join(b.directory, path))
|
||||
}
|
||||
|
||||
func writeFileLocally(fr io.Reader, path string) (int64, *model.AppError) {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0774); err != nil {
|
||||
directory, _ := filepath.Abs(filepath.Dir(path))
|
||||
return 0, model.NewAppError("WriteFile", "api.file.write_file_locally.create_dir.app_error", nil, "directory="+directory+", err="+err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
fw, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
return 0, model.NewAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
defer fw.Close()
|
||||
written, err := io.Copy(fw, fr)
|
||||
if err != nil {
|
||||
return written, model.NewAppError("WriteFile", "api.file.write_file_locally.writing.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) RemoveFile(path string) *model.AppError {
|
||||
if err := os.Remove(filepath.Join(b.directory, path)); err != nil {
|
||||
return model.NewAppError("RemoveFile", "utils.file.remove_file.local.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) ListDirectory(path string) (*[]string, *model.AppError) {
|
||||
var paths []string
|
||||
if fileInfos, err := ioutil.ReadDir(filepath.Join(b.directory, path)); err != nil {
|
||||
return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
} else {
|
||||
for _, fileInfo := range fileInfos {
|
||||
if fileInfo.IsDir() {
|
||||
paths = append(paths, filepath.Join(path, fileInfo.Name()))
|
||||
}
|
||||
}
|
||||
}
|
||||
return &paths, nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) RemoveDirectory(path string) *model.AppError {
|
||||
if err := os.RemoveAll(filepath.Join(b.directory, path)); err != nil {
|
||||
return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.local.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
215
services/filesstore/mocks/FileBackend.go
Обычный файл
215
services/filesstore/mocks/FileBackend.go
Обычный файл
@@ -0,0 +1,215 @@
|
||||
// Code generated by mockery v1.0.0. DO NOT EDIT.
|
||||
|
||||
// Regenerate this file using `make files-store-mocks`.
|
||||
|
||||
package mocks
|
||||
|
||||
import io "io"
|
||||
import mock "github.com/stretchr/testify/mock"
|
||||
import model "github.com/mattermost/mattermost-server/model"
|
||||
|
||||
// FileBackend is an autogenerated mock type for the FileBackend type
|
||||
type FileBackend struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
// CopyFile provides a mock function with given fields: oldPath, newPath
|
||||
func (_m *FileBackend) CopyFile(oldPath string, newPath string) *model.AppError {
|
||||
ret := _m.Called(oldPath, newPath)
|
||||
|
||||
var r0 *model.AppError
|
||||
if rf, ok := ret.Get(0).(func(string, string) *model.AppError); ok {
|
||||
r0 = rf(oldPath, newPath)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*model.AppError)
|
||||
}
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// FileExists provides a mock function with given fields: path
|
||||
func (_m *FileBackend) FileExists(path string) (bool, *model.AppError) {
|
||||
ret := _m.Called(path)
|
||||
|
||||
var r0 bool
|
||||
if rf, ok := ret.Get(0).(func(string) bool); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
r0 = ret.Get(0).(bool)
|
||||
}
|
||||
|
||||
var r1 *model.AppError
|
||||
if rf, ok := ret.Get(1).(func(string) *model.AppError); ok {
|
||||
r1 = rf(path)
|
||||
} else {
|
||||
if ret.Get(1) != nil {
|
||||
r1 = ret.Get(1).(*model.AppError)
|
||||
}
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// ListDirectory provides a mock function with given fields: path
|
||||
func (_m *FileBackend) ListDirectory(path string) (*[]string, *model.AppError) {
|
||||
ret := _m.Called(path)
|
||||
|
||||
var r0 *[]string
|
||||
if rf, ok := ret.Get(0).(func(string) *[]string); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*[]string)
|
||||
}
|
||||
}
|
||||
|
||||
var r1 *model.AppError
|
||||
if rf, ok := ret.Get(1).(func(string) *model.AppError); ok {
|
||||
r1 = rf(path)
|
||||
} else {
|
||||
if ret.Get(1) != nil {
|
||||
r1 = ret.Get(1).(*model.AppError)
|
||||
}
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MoveFile provides a mock function with given fields: oldPath, newPath
|
||||
func (_m *FileBackend) MoveFile(oldPath string, newPath string) *model.AppError {
|
||||
ret := _m.Called(oldPath, newPath)
|
||||
|
||||
var r0 *model.AppError
|
||||
if rf, ok := ret.Get(0).(func(string, string) *model.AppError); ok {
|
||||
r0 = rf(oldPath, newPath)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*model.AppError)
|
||||
}
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// ReadFile provides a mock function with given fields: path
|
||||
func (_m *FileBackend) ReadFile(path string) ([]byte, *model.AppError) {
|
||||
ret := _m.Called(path)
|
||||
|
||||
var r0 []byte
|
||||
if rf, ok := ret.Get(0).(func(string) []byte); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).([]byte)
|
||||
}
|
||||
}
|
||||
|
||||
var r1 *model.AppError
|
||||
if rf, ok := ret.Get(1).(func(string) *model.AppError); ok {
|
||||
r1 = rf(path)
|
||||
} else {
|
||||
if ret.Get(1) != nil {
|
||||
r1 = ret.Get(1).(*model.AppError)
|
||||
}
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// Reader provides a mock function with given fields: path
|
||||
func (_m *FileBackend) Reader(path string) (io.ReadCloser, *model.AppError) {
|
||||
ret := _m.Called(path)
|
||||
|
||||
var r0 io.ReadCloser
|
||||
if rf, ok := ret.Get(0).(func(string) io.ReadCloser); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(io.ReadCloser)
|
||||
}
|
||||
}
|
||||
|
||||
var r1 *model.AppError
|
||||
if rf, ok := ret.Get(1).(func(string) *model.AppError); ok {
|
||||
r1 = rf(path)
|
||||
} else {
|
||||
if ret.Get(1) != nil {
|
||||
r1 = ret.Get(1).(*model.AppError)
|
||||
}
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// RemoveDirectory provides a mock function with given fields: path
|
||||
func (_m *FileBackend) RemoveDirectory(path string) *model.AppError {
|
||||
ret := _m.Called(path)
|
||||
|
||||
var r0 *model.AppError
|
||||
if rf, ok := ret.Get(0).(func(string) *model.AppError); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*model.AppError)
|
||||
}
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// RemoveFile provides a mock function with given fields: path
|
||||
func (_m *FileBackend) RemoveFile(path string) *model.AppError {
|
||||
ret := _m.Called(path)
|
||||
|
||||
var r0 *model.AppError
|
||||
if rf, ok := ret.Get(0).(func(string) *model.AppError); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*model.AppError)
|
||||
}
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// TestConnection provides a mock function with given fields:
|
||||
func (_m *FileBackend) TestConnection() *model.AppError {
|
||||
ret := _m.Called()
|
||||
|
||||
var r0 *model.AppError
|
||||
if rf, ok := ret.Get(0).(func() *model.AppError); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*model.AppError)
|
||||
}
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// WriteFile provides a mock function with given fields: fr, path
|
||||
func (_m *FileBackend) WriteFile(fr io.Reader, path string) (int64, *model.AppError) {
|
||||
ret := _m.Called(fr, path)
|
||||
|
||||
var r0 int64
|
||||
if rf, ok := ret.Get(0).(func(io.Reader, string) int64); ok {
|
||||
r0 = rf(fr, path)
|
||||
} else {
|
||||
r0 = ret.Get(0).(int64)
|
||||
}
|
||||
|
||||
var r1 *model.AppError
|
||||
if rf, ok := ret.Get(1).(func(io.Reader, string) *model.AppError); ok {
|
||||
r1 = rf(fr, path)
|
||||
} else {
|
||||
if ret.Get(1) != nil {
|
||||
r1 = ret.Get(1).(*model.AppError)
|
||||
}
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
294
services/filesstore/s3store.go
Обычный файл
294
services/filesstore/s3store.go
Обычный файл
@@ -0,0 +1,294 @@
|
||||
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
|
||||
// See License.txt for license information.
|
||||
|
||||
package filesstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
s3 "github.com/minio/minio-go"
|
||||
"github.com/minio/minio-go/pkg/credentials"
|
||||
"github.com/minio/minio-go/pkg/encrypt"
|
||||
|
||||
"github.com/mattermost/mattermost-server/mlog"
|
||||
"github.com/mattermost/mattermost-server/model"
|
||||
)
|
||||
|
||||
type S3FileBackend struct {
|
||||
endpoint string
|
||||
accessKey string
|
||||
secretKey string
|
||||
secure bool
|
||||
signV2 bool
|
||||
region string
|
||||
bucket string
|
||||
encrypt bool
|
||||
trace bool
|
||||
}
|
||||
|
||||
// Similar to s3.New() but allows initialization of signature v2 or signature v4 client.
|
||||
// If signV2 input is false, function always returns signature v4.
|
||||
//
|
||||
// Additionally this function also takes a user defined region, if set
|
||||
// disables automatic region lookup.
|
||||
func (b *S3FileBackend) s3New() (*s3.Client, error) {
|
||||
var creds *credentials.Credentials
|
||||
|
||||
if b.accessKey == "" && b.secretKey == "" {
|
||||
creds = credentials.NewIAM("")
|
||||
} else if b.signV2 {
|
||||
creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV2)
|
||||
} else {
|
||||
creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV4)
|
||||
}
|
||||
|
||||
s3Clnt, err := s3.NewWithCredentials(b.endpoint, creds, b.secure, b.region)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if b.trace {
|
||||
s3Clnt.TraceOn(os.Stdout)
|
||||
}
|
||||
|
||||
return s3Clnt, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) TestConnection() *model.AppError {
|
||||
s3Clnt, err := b.s3New()
|
||||
if err != nil {
|
||||
return model.NewAppError("TestFileConnection", "api.file.test_connection.s3.connection.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
exists, err := s3Clnt.BucketExists(b.bucket)
|
||||
if err != nil {
|
||||
return model.NewAppError("TestFileConnection", "api.file.test_connection.s3.bucket_exists.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
if !exists {
|
||||
mlog.Warn("Bucket specified does not exist. Attempting to create...")
|
||||
err := s3Clnt.MakeBucket(b.bucket, b.region)
|
||||
if err != nil {
|
||||
mlog.Error("Unable to create bucket.")
|
||||
return model.NewAppError("TestFileConnection", "api.file.test_connection.s3.bucked_create.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
mlog.Info("Connection to S3 or minio is good. Bucket exists.")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Caller must close the first return value
|
||||
func (b *S3FileBackend) Reader(path string) (io.ReadCloser, *model.AppError) {
|
||||
s3Clnt, err := b.s3New()
|
||||
if err != nil {
|
||||
return nil, model.NewAppError("Reader", "api.file.reader.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
minioObject, err := s3Clnt.GetObject(b.bucket, path, s3.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, model.NewAppError("Reader", "api.file.reader.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
return minioObject, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) ReadFile(path string) ([]byte, *model.AppError) {
|
||||
s3Clnt, err := b.s3New()
|
||||
if err != nil {
|
||||
return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
minioObject, err := s3Clnt.GetObject(b.bucket, path, s3.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
defer minioObject.Close()
|
||||
if f, err := ioutil.ReadAll(minioObject); err != nil {
|
||||
return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
} else {
|
||||
return f, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) FileExists(path string) (bool, *model.AppError) {
|
||||
s3Clnt, err := b.s3New()
|
||||
|
||||
if err != nil {
|
||||
return false, model.NewAppError("FileExists", "api.file.file_exists.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
_, err = s3Clnt.StatObject(b.bucket, path, s3.StatObjectOptions{})
|
||||
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
if err.(s3.ErrorResponse).Code == "NoSuchKey" {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return false, model.NewAppError("FileExists", "api.file.file_exists.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) CopyFile(oldPath, newPath string) *model.AppError {
|
||||
s3Clnt, err := b.s3New()
|
||||
if err != nil {
|
||||
return model.NewAppError("copyFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
source := s3.NewSourceInfo(b.bucket, oldPath, nil)
|
||||
destination, err := s3.NewDestinationInfo(b.bucket, newPath, encrypt.NewSSE(), nil)
|
||||
if err != nil {
|
||||
return model.NewAppError("copyFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
if err = s3Clnt.CopyObject(destination, source); err != nil {
|
||||
return model.NewAppError("copyFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) MoveFile(oldPath, newPath string) *model.AppError {
|
||||
s3Clnt, err := b.s3New()
|
||||
if err != nil {
|
||||
return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
source := s3.NewSourceInfo(b.bucket, oldPath, nil)
|
||||
destination, err := s3.NewDestinationInfo(b.bucket, newPath, encrypt.NewSSE(), nil)
|
||||
if err != nil {
|
||||
return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
if err = s3Clnt.CopyObject(destination, source); err != nil {
|
||||
return model.NewAppError("moveFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
if err = s3Clnt.RemoveObject(b.bucket, oldPath); err != nil {
|
||||
return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) WriteFile(fr io.Reader, path string) (int64, *model.AppError) {
|
||||
s3Clnt, err := b.s3New()
|
||||
if err != nil {
|
||||
return 0, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
var contentType string
|
||||
if ext := filepath.Ext(path); model.IsFileExtImage(ext) {
|
||||
contentType = model.GetImageMimeType(ext)
|
||||
} else {
|
||||
contentType = "binary/octet-stream"
|
||||
}
|
||||
|
||||
options := s3PutOptions(b.encrypt, contentType)
|
||||
var buf bytes.Buffer
|
||||
_, err = buf.ReadFrom(fr)
|
||||
if err != nil {
|
||||
return 0, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
written, err := s3Clnt.PutObject(b.bucket, path, &buf, int64(buf.Len()), options)
|
||||
if err != nil {
|
||||
return written, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) RemoveFile(path string) *model.AppError {
|
||||
s3Clnt, err := b.s3New()
|
||||
if err != nil {
|
||||
return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
if err := s3Clnt.RemoveObject(b.bucket, path); err != nil {
|
||||
return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan string {
|
||||
out := make(chan string, 1)
|
||||
|
||||
go func() {
|
||||
defer close(out)
|
||||
|
||||
for {
|
||||
info, done := <-in
|
||||
|
||||
if !done {
|
||||
break
|
||||
}
|
||||
|
||||
out <- info.Key
|
||||
}
|
||||
}()
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) ListDirectory(path string) (*[]string, *model.AppError) {
|
||||
var paths []string
|
||||
|
||||
s3Clnt, err := b.s3New()
|
||||
if err != nil {
|
||||
return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
doneCh := make(chan struct{})
|
||||
|
||||
defer close(doneCh)
|
||||
|
||||
for object := range s3Clnt.ListObjects(b.bucket, path, false, doneCh) {
|
||||
if object.Err != nil {
|
||||
return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, object.Err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
paths = append(paths, strings.Trim(object.Key, "/"))
|
||||
}
|
||||
|
||||
return &paths, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) RemoveDirectory(path string) *model.AppError {
|
||||
s3Clnt, err := b.s3New()
|
||||
if err != nil {
|
||||
return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
doneCh := make(chan struct{})
|
||||
|
||||
for err := range s3Clnt.RemoveObjects(b.bucket, getPathsFromObjectInfos(s3Clnt.ListObjects(b.bucket, path, true, doneCh))) {
|
||||
if err.Err != nil {
|
||||
doneCh <- struct{}{}
|
||||
return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
|
||||
close(doneCh)
|
||||
return nil
|
||||
}
|
||||
|
||||
func s3PutOptions(encrypted bool, contentType string) s3.PutObjectOptions {
|
||||
options := s3.PutObjectOptions{}
|
||||
if encrypted {
|
||||
options.ServerSideEncryption = encrypt.NewSSE()
|
||||
}
|
||||
options.ContentType = contentType
|
||||
|
||||
return options
|
||||
}
|
||||
|
||||
func CheckMandatoryS3Fields(settings *model.FileSettings) *model.AppError {
|
||||
if len(settings.AmazonS3Bucket) == 0 {
|
||||
return model.NewAppError("S3File", "api.admin.test_s3.missing_s3_bucket", nil, "", http.StatusBadRequest)
|
||||
}
|
||||
|
||||
// if S3 endpoint is not set call the set defaults to set that
|
||||
if len(settings.AmazonS3Endpoint) == 0 {
|
||||
settings.SetDefaults()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
32
services/filesstore/s3store_test.go
Обычный файл
32
services/filesstore/s3store_test.go
Обычный файл
@@ -0,0 +1,32 @@
|
||||
// Copyright (c) 2018-present Mattermost, Inc. All Rights Reserved.
|
||||
// See License.txt for license information.
|
||||
|
||||
package filesstore
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/mattermost/mattermost-server/model"
|
||||
)
|
||||
|
||||
func TestCheckMandatoryS3Fields(t *testing.T) {
|
||||
cfg := model.FileSettings{}
|
||||
|
||||
err := CheckMandatoryS3Fields(&cfg)
|
||||
if err == nil || err.Message != "api.admin.test_s3.missing_s3_bucket" {
|
||||
t.Fatal("should've failed with missing s3 bucket")
|
||||
}
|
||||
|
||||
cfg.AmazonS3Bucket = "test-mm"
|
||||
err = CheckMandatoryS3Fields(&cfg)
|
||||
if err != nil {
|
||||
t.Fatal("should've not failed")
|
||||
}
|
||||
|
||||
cfg.AmazonS3Endpoint = ""
|
||||
err = CheckMandatoryS3Fields(&cfg)
|
||||
if err != nil || cfg.AmazonS3Endpoint != "s3.amazonaws.com" {
|
||||
t.Fatal("should've not failed because it should set the endpoint to the default")
|
||||
}
|
||||
|
||||
}
|
||||
Ссылка в новой задаче
Block a user