Moving filesstore services into shared folder (#16940)
* Moving filesstore services into shared folder * Fixing app-layers generation * Renaming from filesstore to filestore
Этот коммит содержится в:
Jesús Espino
коммит произвёл
GitHub
GitHub
родитель
5f9ab3783a
Коммит
78355ae2a7
@@ -1,83 +0,0 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package filesstore
|
||||
|
||||
import (
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
driverS3 = "amazons3"
|
||||
driverLocal = "local"
|
||||
)
|
||||
|
||||
type ReadCloseSeeker interface {
|
||||
io.ReadCloser
|
||||
io.Seeker
|
||||
}
|
||||
|
||||
type FileBackend interface {
|
||||
TestConnection() error
|
||||
|
||||
Reader(path string) (ReadCloseSeeker, error)
|
||||
ReadFile(path string) ([]byte, error)
|
||||
FileExists(path string) (bool, error)
|
||||
FileSize(path string) (int64, error)
|
||||
CopyFile(oldPath, newPath string) error
|
||||
MoveFile(oldPath, newPath string) error
|
||||
WriteFile(fr io.Reader, path string) (int64, error)
|
||||
AppendFile(fr io.Reader, path string) (int64, error)
|
||||
RemoveFile(path string) error
|
||||
FileModTime(path string) (time.Time, error)
|
||||
|
||||
ListDirectory(path string) ([]string, error)
|
||||
RemoveDirectory(path string) error
|
||||
}
|
||||
|
||||
type FileBackendSettings struct {
|
||||
DriverName string
|
||||
Directory string
|
||||
AmazonS3AccessKeyId string
|
||||
AmazonS3SecretAccessKey string
|
||||
AmazonS3Bucket string
|
||||
AmazonS3PathPrefix string
|
||||
AmazonS3Region string
|
||||
AmazonS3Endpoint string
|
||||
AmazonS3SSL bool
|
||||
AmazonS3SignV2 bool
|
||||
AmazonS3SSE bool
|
||||
AmazonS3Trace bool
|
||||
}
|
||||
|
||||
func (settings *FileBackendSettings) CheckMandatoryS3Fields() error {
|
||||
if settings.AmazonS3Bucket == "" {
|
||||
return errors.New("missing s3 bucket settings")
|
||||
}
|
||||
|
||||
// if S3 endpoint is not set call the set defaults to set that
|
||||
if settings.AmazonS3Endpoint == "" {
|
||||
settings.AmazonS3Endpoint = "s3.amazonaws.com"
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewFileBackend(settings FileBackendSettings) (FileBackend, error) {
|
||||
switch settings.DriverName {
|
||||
case driverS3:
|
||||
backend, err := NewS3FileBackend(settings)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "unable to connect to the s3 backend")
|
||||
}
|
||||
return backend, nil
|
||||
case driverLocal:
|
||||
return &LocalFileBackend{
|
||||
directory: settings.Directory,
|
||||
}, nil
|
||||
}
|
||||
return nil, errors.New("no valid filestorage driver found")
|
||||
}
|
||||
@@ -1,462 +0,0 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package filesstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
"github.com/xtgo/uuid"
|
||||
|
||||
"github.com/mattermost/mattermost-server/v5/mlog"
|
||||
)
|
||||
|
||||
func randomString() string {
|
||||
return uuid.NewRandom().String()
|
||||
}
|
||||
|
||||
type FileBackendTestSuite struct {
|
||||
suite.Suite
|
||||
|
||||
settings FileBackendSettings
|
||||
backend FileBackend
|
||||
}
|
||||
|
||||
func TestLocalFileBackendTestSuite(t *testing.T) {
|
||||
// Setup a global logger to catch tests logging outside of app context
|
||||
// The global logger will be stomped by apps initializing but that's fine for testing. Ideally this won't happen.
|
||||
mlog.InitGlobalLogger(mlog.NewLogger(&mlog.LoggerConfiguration{
|
||||
EnableConsole: true,
|
||||
ConsoleJson: true,
|
||||
ConsoleLevel: "error",
|
||||
EnableFile: false,
|
||||
}))
|
||||
|
||||
dir, err := ioutil.TempDir("", "")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
suite.Run(t, &FileBackendTestSuite{
|
||||
settings: FileBackendSettings{
|
||||
DriverName: driverLocal,
|
||||
Directory: dir,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func TestS3FileBackendTestSuite(t *testing.T) {
|
||||
runBackendTest(t, false)
|
||||
}
|
||||
|
||||
func TestS3FileBackendTestSuiteWithEncryption(t *testing.T) {
|
||||
runBackendTest(t, true)
|
||||
}
|
||||
|
||||
func runBackendTest(t *testing.T, encrypt bool) {
|
||||
s3Host := os.Getenv("CI_MINIO_HOST")
|
||||
if s3Host == "" {
|
||||
s3Host = "localhost"
|
||||
}
|
||||
|
||||
s3Port := os.Getenv("CI_MINIO_PORT")
|
||||
if s3Port == "" {
|
||||
s3Port = "9000"
|
||||
}
|
||||
|
||||
s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
|
||||
|
||||
suite.Run(t, &FileBackendTestSuite{
|
||||
settings: FileBackendSettings{
|
||||
DriverName: driverS3,
|
||||
AmazonS3AccessKeyId: "minioaccesskey",
|
||||
AmazonS3SecretAccessKey: "miniosecretkey",
|
||||
AmazonS3Bucket: "mattermost-test",
|
||||
AmazonS3Region: "",
|
||||
AmazonS3Endpoint: s3Endpoint,
|
||||
AmazonS3PathPrefix: "",
|
||||
AmazonS3SSL: false,
|
||||
AmazonS3SSE: encrypt,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) SetupTest() {
|
||||
backend, err := NewFileBackend(s.settings)
|
||||
require.NoError(s.T(), err)
|
||||
s.backend = backend
|
||||
|
||||
// This is needed to create the bucket if it doesn't exist.
|
||||
s.Nil(s.backend.TestConnection())
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestConnection() {
|
||||
s.Nil(s.backend.TestConnection())
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestReadWriteFile() {
|
||||
b := []byte("test")
|
||||
path := "tests/" + randomString()
|
||||
|
||||
written, err := s.backend.WriteFile(bytes.NewReader(b), path)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
defer s.backend.RemoveFile(path)
|
||||
|
||||
read, err := s.backend.ReadFile(path)
|
||||
s.Nil(err)
|
||||
|
||||
readString := string(read)
|
||||
s.EqualValues(readString, "test")
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestReadWriteFileImage() {
|
||||
b := []byte("testimage")
|
||||
path := "tests/" + randomString() + ".png"
|
||||
|
||||
written, err := s.backend.WriteFile(bytes.NewReader(b), path)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
defer s.backend.RemoveFile(path)
|
||||
|
||||
read, err := s.backend.ReadFile(path)
|
||||
s.Nil(err)
|
||||
|
||||
readString := string(read)
|
||||
s.EqualValues(readString, "testimage")
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestFileExists() {
|
||||
b := []byte("testimage")
|
||||
path := "tests/" + randomString() + ".png"
|
||||
|
||||
_, err := s.backend.WriteFile(bytes.NewReader(b), path)
|
||||
s.Nil(err)
|
||||
defer s.backend.RemoveFile(path)
|
||||
|
||||
res, err := s.backend.FileExists(path)
|
||||
s.Nil(err)
|
||||
s.True(res)
|
||||
|
||||
res, err = s.backend.FileExists("tests/idontexist.png")
|
||||
s.Nil(err)
|
||||
s.False(res)
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestCopyFile() {
|
||||
b := []byte("test")
|
||||
path1 := "tests/" + randomString()
|
||||
path2 := "tests/" + randomString()
|
||||
|
||||
written, err := s.backend.WriteFile(bytes.NewReader(b), path1)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
defer s.backend.RemoveFile(path1)
|
||||
|
||||
err = s.backend.CopyFile(path1, path2)
|
||||
s.Nil(err)
|
||||
defer s.backend.RemoveFile(path2)
|
||||
|
||||
data1, err := s.backend.ReadFile(path1)
|
||||
s.Nil(err)
|
||||
|
||||
data2, err := s.backend.ReadFile(path2)
|
||||
s.Nil(err)
|
||||
|
||||
s.Equal(b, data1)
|
||||
s.Equal(b, data2)
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestCopyFileToDirectoryThatDoesntExist() {
|
||||
b := []byte("test")
|
||||
path1 := "tests/" + randomString()
|
||||
path2 := "tests/newdirectory/" + randomString()
|
||||
|
||||
written, err := s.backend.WriteFile(bytes.NewReader(b), path1)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
defer s.backend.RemoveFile(path1)
|
||||
|
||||
err = s.backend.CopyFile(path1, path2)
|
||||
s.Nil(err)
|
||||
defer s.backend.RemoveFile(path2)
|
||||
|
||||
_, err = s.backend.ReadFile(path1)
|
||||
s.Nil(err)
|
||||
|
||||
_, err = s.backend.ReadFile(path2)
|
||||
s.Nil(err)
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestMoveFile() {
|
||||
b := []byte("test")
|
||||
path1 := "tests/" + randomString()
|
||||
path2 := "tests/" + randomString()
|
||||
|
||||
written, err := s.backend.WriteFile(bytes.NewReader(b), path1)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
defer s.backend.RemoveFile(path1)
|
||||
|
||||
s.Nil(s.backend.MoveFile(path1, path2))
|
||||
defer s.backend.RemoveFile(path2)
|
||||
|
||||
_, err = s.backend.ReadFile(path1)
|
||||
s.Error(err)
|
||||
|
||||
data, err := s.backend.ReadFile(path2)
|
||||
s.Nil(err)
|
||||
|
||||
s.Equal(b, data)
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestRemoveFile() {
|
||||
b := []byte("test")
|
||||
path := "tests/" + randomString()
|
||||
|
||||
written, err := s.backend.WriteFile(bytes.NewReader(b), path)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
s.Nil(s.backend.RemoveFile(path))
|
||||
|
||||
_, err = s.backend.ReadFile(path)
|
||||
s.Error(err)
|
||||
|
||||
written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/foo")
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
|
||||
written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/bar")
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
|
||||
written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/asdf")
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
|
||||
s.Nil(s.backend.RemoveDirectory("tests2"))
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestListDirectory() {
|
||||
b := []byte("test")
|
||||
path1 := "19700101/" + randomString()
|
||||
path2 := "19800101/" + randomString()
|
||||
|
||||
paths, err := s.backend.ListDirectory("19700101")
|
||||
s.Nil(err)
|
||||
s.Len(paths, 0)
|
||||
|
||||
written, err := s.backend.WriteFile(bytes.NewReader(b), path1)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
|
||||
written, err = s.backend.WriteFile(bytes.NewReader(b), path2)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
|
||||
paths, err = s.backend.ListDirectory("19700101")
|
||||
s.Nil(err)
|
||||
s.Len(paths, 1)
|
||||
s.Equal(path1, (paths)[0])
|
||||
|
||||
paths, err = s.backend.ListDirectory("19700101/")
|
||||
s.Nil(err)
|
||||
s.Len(paths, 1)
|
||||
s.Equal(path1, (paths)[0])
|
||||
|
||||
paths, err = s.backend.ListDirectory("")
|
||||
s.Nil(err)
|
||||
|
||||
found1 := false
|
||||
found2 := false
|
||||
for _, path := range paths {
|
||||
if path == "19700101" {
|
||||
found1 = true
|
||||
} else if path == "19800101" {
|
||||
found2 = true
|
||||
}
|
||||
}
|
||||
s.True(found1)
|
||||
s.True(found2)
|
||||
|
||||
s.backend.RemoveFile(path1)
|
||||
s.backend.RemoveFile(path2)
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestRemoveDirectory() {
|
||||
b := []byte("test")
|
||||
|
||||
written, err := s.backend.WriteFile(bytes.NewReader(b), "tests2/foo")
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
|
||||
written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/bar")
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
|
||||
written, err = s.backend.WriteFile(bytes.NewReader(b), "tests2/aaa")
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written, "expected given number of bytes to have been written")
|
||||
|
||||
s.Nil(s.backend.RemoveDirectory("tests2"))
|
||||
|
||||
_, err = s.backend.ReadFile("tests2/foo")
|
||||
s.Error(err)
|
||||
_, err = s.backend.ReadFile("tests2/bar")
|
||||
s.Error(err)
|
||||
_, err = s.backend.ReadFile("tests2/asdf")
|
||||
s.Error(err)
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestAppendFile() {
|
||||
s.Run("should fail if target file is missing", func() {
|
||||
path := "tests/" + randomString()
|
||||
b := make([]byte, 1024)
|
||||
written, err := s.backend.AppendFile(bytes.NewReader(b), path)
|
||||
s.Error(err)
|
||||
s.Zero(written)
|
||||
})
|
||||
|
||||
s.Run("should correctly append the data", func() {
|
||||
// First part needs to be at least 5MB for the S3 implementation to work.
|
||||
size := 5 * 1024 * 1024
|
||||
b := make([]byte, size)
|
||||
for i := range b {
|
||||
b[i] = 'A'
|
||||
}
|
||||
path := "tests/" + randomString()
|
||||
|
||||
written, err := s.backend.WriteFile(bytes.NewReader(b), path)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b), written)
|
||||
defer s.backend.RemoveFile(path)
|
||||
|
||||
b2 := make([]byte, 1024)
|
||||
for i := range b2 {
|
||||
b2[i] = 'B'
|
||||
}
|
||||
|
||||
written, err = s.backend.AppendFile(bytes.NewReader(b2), path)
|
||||
s.Nil(err)
|
||||
s.EqualValues(int64(len(b2)), written)
|
||||
|
||||
read, err := s.backend.ReadFile(path)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b)+len(b2), len(read))
|
||||
s.EqualValues(append(b, b2...), read)
|
||||
|
||||
b3 := make([]byte, 1024)
|
||||
for i := range b3 {
|
||||
b3[i] = 'C'
|
||||
}
|
||||
|
||||
written, err = s.backend.AppendFile(bytes.NewReader(b3), path)
|
||||
s.Nil(err)
|
||||
s.EqualValues(int64(len(b3)), written)
|
||||
|
||||
read, err = s.backend.ReadFile(path)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(b)+len(b2)+len(b3), len(read))
|
||||
s.EqualValues(append(append(b, b2...), b3...), read)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestFileSize() {
|
||||
s.Run("nonexistent file", func() {
|
||||
size, err := s.backend.FileSize("tests/nonexistentfile")
|
||||
s.NotNil(err)
|
||||
s.Zero(size)
|
||||
})
|
||||
|
||||
s.Run("valid file", func() {
|
||||
data := make([]byte, rand.Intn(1024*1024)+1)
|
||||
path := "tests/" + randomString()
|
||||
|
||||
written, err := s.backend.WriteFile(bytes.NewReader(data), path)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(data), written)
|
||||
defer s.backend.RemoveFile(path)
|
||||
|
||||
size, err := s.backend.FileSize(path)
|
||||
s.Nil(err)
|
||||
s.Equal(int64(len(data)), size)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *FileBackendTestSuite) TestFileModTime() {
|
||||
s.Run("nonexistent file", func() {
|
||||
modTime, err := s.backend.FileModTime("tests/nonexistentfile")
|
||||
s.NotNil(err)
|
||||
s.Empty(modTime)
|
||||
})
|
||||
|
||||
s.Run("valid file", func() {
|
||||
path := "tests/" + randomString()
|
||||
data := []byte("some data")
|
||||
|
||||
written, err := s.backend.WriteFile(bytes.NewReader(data), path)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(data), written)
|
||||
defer s.backend.RemoveFile(path)
|
||||
|
||||
modTime, err := s.backend.FileModTime(path)
|
||||
s.Nil(err)
|
||||
s.NotEmpty(modTime)
|
||||
|
||||
// We wait 1 second so that the times will differ enough to be testable.
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
path2 := "tests/" + randomString()
|
||||
written, err = s.backend.WriteFile(bytes.NewReader(data), path2)
|
||||
s.Nil(err)
|
||||
s.EqualValues(len(data), written)
|
||||
defer s.backend.RemoveFile(path2)
|
||||
|
||||
modTime2, err := s.backend.FileModTime(path2)
|
||||
s.Nil(err)
|
||||
s.NotEmpty(modTime2)
|
||||
s.True(modTime2.After(modTime))
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkS3WriteFile(b *testing.B) {
|
||||
settings := FileBackendSettings{
|
||||
DriverName: driverS3,
|
||||
AmazonS3AccessKeyId: "minioaccesskey",
|
||||
AmazonS3SecretAccessKey: "miniosecretkey",
|
||||
AmazonS3Bucket: "mattermost-test",
|
||||
AmazonS3Region: "",
|
||||
AmazonS3Endpoint: "localhost:9000",
|
||||
AmazonS3PathPrefix: "",
|
||||
AmazonS3SSL: false,
|
||||
AmazonS3SSE: false,
|
||||
}
|
||||
|
||||
backend, err := NewFileBackend(settings)
|
||||
require.NoError(b, err)
|
||||
|
||||
// This is needed to create the bucket if it doesn't exist.
|
||||
require.NoError(b, backend.TestConnection())
|
||||
|
||||
path := "tests/" + randomString()
|
||||
size := 1 * 1024 * 1024
|
||||
data := make([]byte, size)
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
written, err := backend.WriteFile(bytes.NewReader(data), path)
|
||||
defer backend.RemoveFile(path)
|
||||
require.NoError(b, err)
|
||||
require.Equal(b, len(data), int(written))
|
||||
}
|
||||
|
||||
b.StopTimer()
|
||||
}
|
||||
@@ -1,211 +0,0 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package filesstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/mattermost/mattermost-server/v5/mlog"
|
||||
)
|
||||
|
||||
const (
|
||||
TestFilePath = "/testfile"
|
||||
)
|
||||
|
||||
type LocalFileBackend struct {
|
||||
directory string
|
||||
}
|
||||
|
||||
// copyFile will copy a file from src path to dst path.
|
||||
// Overwrites any existing files at dst.
|
||||
// Permissions are copied from file at src to the new file at dst.
|
||||
func copyFile(src, dst string) (err error) {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer in.Close()
|
||||
|
||||
if err = os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil {
|
||||
return
|
||||
}
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if e := out.Close(); e != nil {
|
||||
err = e
|
||||
}
|
||||
}()
|
||||
|
||||
_, err = io.Copy(out, in)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
err = out.Sync()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
stat, err := os.Stat(src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = os.Chmod(dst, stat.Mode())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) TestConnection() error {
|
||||
f := bytes.NewReader([]byte("testingwrite"))
|
||||
if _, err := writeFileLocally(f, filepath.Join(b.directory, TestFilePath)); err != nil {
|
||||
return errors.Wrap(err, "unable to write to the local filesystem storage")
|
||||
}
|
||||
os.Remove(filepath.Join(b.directory, TestFilePath))
|
||||
mlog.Debug("Able to write files to local storage.")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) Reader(path string) (ReadCloseSeeker, error) {
|
||||
f, err := os.Open(filepath.Join(b.directory, path))
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "unable to open file %s", path)
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) ReadFile(path string) ([]byte, error) {
|
||||
f, err := ioutil.ReadFile(filepath.Join(b.directory, path))
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "unable to read file %s", path)
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) FileExists(path string) (bool, error) {
|
||||
_, err := os.Stat(filepath.Join(b.directory, path))
|
||||
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return false, errors.Wrapf(err, "unable to know if file %s exists", path)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) FileSize(path string) (int64, error) {
|
||||
info, err := os.Stat(filepath.Join(b.directory, path))
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "unable to get file size for %s", path)
|
||||
}
|
||||
return info.Size(), nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) FileModTime(path string) (time.Time, error) {
|
||||
info, err := os.Stat(filepath.Join(b.directory, path))
|
||||
if err != nil {
|
||||
return time.Time{}, errors.Wrapf(err, "unable to get modification time for file %s", path)
|
||||
}
|
||||
return info.ModTime(), nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) CopyFile(oldPath, newPath string) error {
|
||||
if err := copyFile(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil {
|
||||
return errors.Wrapf(err, "unable to copy file from %s to %s", oldPath, newPath)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) MoveFile(oldPath, newPath string) error {
|
||||
if err := os.MkdirAll(filepath.Dir(filepath.Join(b.directory, newPath)), 0750); err != nil {
|
||||
return errors.Wrapf(err, "unable to create the new destination directory %s", filepath.Dir(newPath))
|
||||
}
|
||||
|
||||
if err := os.Rename(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil {
|
||||
return errors.Wrapf(err, "unable to move the file to %s to the destination directory", newPath)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) WriteFile(fr io.Reader, path string) (int64, error) {
|
||||
return writeFileLocally(fr, filepath.Join(b.directory, path))
|
||||
}
|
||||
|
||||
func writeFileLocally(fr io.Reader, path string) (int64, error) {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil {
|
||||
directory, _ := filepath.Abs(filepath.Dir(path))
|
||||
return 0, errors.Wrapf(err, "unable to create the directory %s for the file %s", directory, path)
|
||||
}
|
||||
fw, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "unable to open the file %s to write the data", path)
|
||||
}
|
||||
defer fw.Close()
|
||||
written, err := io.Copy(fw, fr)
|
||||
if err != nil {
|
||||
return written, errors.Wrapf(err, "unable write the data in the file %s", path)
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) AppendFile(fr io.Reader, path string) (int64, error) {
|
||||
fp := filepath.Join(b.directory, path)
|
||||
if _, err := os.Stat(fp); err != nil {
|
||||
return 0, errors.Wrapf(err, "unable to find the file %s to append the data", path)
|
||||
}
|
||||
fw, err := os.OpenFile(fp, os.O_WRONLY|os.O_APPEND, 0600)
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "unable to open the file %s to append the data", path)
|
||||
}
|
||||
defer fw.Close()
|
||||
written, err := io.Copy(fw, fr)
|
||||
if err != nil {
|
||||
return written, errors.Wrapf(err, "unable append the data in the file %s", path)
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) RemoveFile(path string) error {
|
||||
if err := os.Remove(filepath.Join(b.directory, path)); err != nil {
|
||||
return errors.Wrapf(err, "unable to remove the file %s", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) ListDirectory(path string) ([]string, error) {
|
||||
var paths []string
|
||||
fileInfos, err := ioutil.ReadDir(filepath.Join(b.directory, path))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return paths, nil
|
||||
}
|
||||
return nil, errors.Wrapf(err, "unable to list the directory %s", path)
|
||||
}
|
||||
for _, fileInfo := range fileInfos {
|
||||
paths = append(paths, filepath.Join(path, fileInfo.Name()))
|
||||
}
|
||||
return paths, nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) RemoveDirectory(path string) error {
|
||||
if err := os.RemoveAll(filepath.Join(b.directory, path)); err != nil {
|
||||
return errors.Wrapf(err, "unable to remove the directory %s", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,264 +0,0 @@
|
||||
// Code generated by mockery v1.0.0. DO NOT EDIT.
|
||||
|
||||
// Regenerate this file using `make filesstore-mocks`.
|
||||
|
||||
package mocks
|
||||
|
||||
import (
|
||||
io "io"
|
||||
|
||||
filesstore "github.com/mattermost/mattermost-server/v5/services/filesstore"
|
||||
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
|
||||
time "time"
|
||||
)
|
||||
|
||||
// FileBackend is an autogenerated mock type for the FileBackend type
|
||||
type FileBackend struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
// AppendFile provides a mock function with given fields: fr, path
|
||||
func (_m *FileBackend) AppendFile(fr io.Reader, path string) (int64, error) {
|
||||
ret := _m.Called(fr, path)
|
||||
|
||||
var r0 int64
|
||||
if rf, ok := ret.Get(0).(func(io.Reader, string) int64); ok {
|
||||
r0 = rf(fr, path)
|
||||
} else {
|
||||
r0 = ret.Get(0).(int64)
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(io.Reader, string) error); ok {
|
||||
r1 = rf(fr, path)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// CopyFile provides a mock function with given fields: oldPath, newPath
|
||||
func (_m *FileBackend) CopyFile(oldPath string, newPath string) error {
|
||||
ret := _m.Called(oldPath, newPath)
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(string, string) error); ok {
|
||||
r0 = rf(oldPath, newPath)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// FileExists provides a mock function with given fields: path
|
||||
func (_m *FileBackend) FileExists(path string) (bool, error) {
|
||||
ret := _m.Called(path)
|
||||
|
||||
var r0 bool
|
||||
if rf, ok := ret.Get(0).(func(string) bool); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
r0 = ret.Get(0).(bool)
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(string) error); ok {
|
||||
r1 = rf(path)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// FileModTime provides a mock function with given fields: path
|
||||
func (_m *FileBackend) FileModTime(path string) (time.Time, error) {
|
||||
ret := _m.Called(path)
|
||||
|
||||
var r0 time.Time
|
||||
if rf, ok := ret.Get(0).(func(string) time.Time); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
r0 = ret.Get(0).(time.Time)
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(string) error); ok {
|
||||
r1 = rf(path)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// FileSize provides a mock function with given fields: path
|
||||
func (_m *FileBackend) FileSize(path string) (int64, error) {
|
||||
ret := _m.Called(path)
|
||||
|
||||
var r0 int64
|
||||
if rf, ok := ret.Get(0).(func(string) int64); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
r0 = ret.Get(0).(int64)
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(string) error); ok {
|
||||
r1 = rf(path)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// ListDirectory provides a mock function with given fields: path
|
||||
func (_m *FileBackend) ListDirectory(path string) ([]string, error) {
|
||||
ret := _m.Called(path)
|
||||
|
||||
var r0 []string
|
||||
if rf, ok := ret.Get(0).(func(string) []string); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).([]string)
|
||||
}
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(string) error); ok {
|
||||
r1 = rf(path)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MoveFile provides a mock function with given fields: oldPath, newPath
|
||||
func (_m *FileBackend) MoveFile(oldPath string, newPath string) error {
|
||||
ret := _m.Called(oldPath, newPath)
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(string, string) error); ok {
|
||||
r0 = rf(oldPath, newPath)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// ReadFile provides a mock function with given fields: path
|
||||
func (_m *FileBackend) ReadFile(path string) ([]byte, error) {
|
||||
ret := _m.Called(path)
|
||||
|
||||
var r0 []byte
|
||||
if rf, ok := ret.Get(0).(func(string) []byte); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).([]byte)
|
||||
}
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(string) error); ok {
|
||||
r1 = rf(path)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// Reader provides a mock function with given fields: path
|
||||
func (_m *FileBackend) Reader(path string) (filesstore.ReadCloseSeeker, error) {
|
||||
ret := _m.Called(path)
|
||||
|
||||
var r0 filesstore.ReadCloseSeeker
|
||||
if rf, ok := ret.Get(0).(func(string) filesstore.ReadCloseSeeker); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(filesstore.ReadCloseSeeker)
|
||||
}
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(string) error); ok {
|
||||
r1 = rf(path)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// RemoveDirectory provides a mock function with given fields: path
|
||||
func (_m *FileBackend) RemoveDirectory(path string) error {
|
||||
ret := _m.Called(path)
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(string) error); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// RemoveFile provides a mock function with given fields: path
|
||||
func (_m *FileBackend) RemoveFile(path string) error {
|
||||
ret := _m.Called(path)
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(string) error); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// TestConnection provides a mock function with given fields:
|
||||
func (_m *FileBackend) TestConnection() error {
|
||||
ret := _m.Called()
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func() error); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// WriteFile provides a mock function with given fields: fr, path
|
||||
func (_m *FileBackend) WriteFile(fr io.Reader, path string) (int64, error) {
|
||||
ret := _m.Called(fr, path)
|
||||
|
||||
var r0 int64
|
||||
if rf, ok := ret.Get(0).(func(io.Reader, string) int64); ok {
|
||||
r0 = rf(fr, path)
|
||||
} else {
|
||||
r0 = ret.Get(0).(int64)
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(io.Reader, string) error); ok {
|
||||
r1 = rf(fr, path)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
// Code generated by mockery v1.0.0. DO NOT EDIT.
|
||||
|
||||
// Regenerate this file using `make filesstore-mocks`.
|
||||
|
||||
package mocks
|
||||
|
||||
import mock "github.com/stretchr/testify/mock"
|
||||
|
||||
// ReadCloseSeeker is an autogenerated mock type for the ReadCloseSeeker type
|
||||
type ReadCloseSeeker struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
// Close provides a mock function with given fields:
|
||||
func (_m *ReadCloseSeeker) Close() error {
|
||||
ret := _m.Called()
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func() error); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// Read provides a mock function with given fields: p
|
||||
func (_m *ReadCloseSeeker) Read(p []byte) (int, error) {
|
||||
ret := _m.Called(p)
|
||||
|
||||
var r0 int
|
||||
if rf, ok := ret.Get(0).(func([]byte) int); ok {
|
||||
r0 = rf(p)
|
||||
} else {
|
||||
r0 = ret.Get(0).(int)
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func([]byte) error); ok {
|
||||
r1 = rf(p)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// Seek provides a mock function with given fields: offset, whence
|
||||
func (_m *ReadCloseSeeker) Seek(offset int64, whence int) (int64, error) {
|
||||
ret := _m.Called(offset, whence)
|
||||
|
||||
var r0 int64
|
||||
if rf, ok := ret.Get(0).(func(int64, int) int64); ok {
|
||||
r0 = rf(offset, whence)
|
||||
} else {
|
||||
r0 = ret.Get(0).(int64)
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(int64, int) error); ok {
|
||||
r1 = rf(offset, whence)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package filesstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
// customTransport is used to point the request to a different server.
|
||||
// This is helpful in situations where a different service is handling AWS S3 requests
|
||||
// from multiple Mattermost applications, and the Mattermost service itself does not
|
||||
// have any S3 credentials.
|
||||
type customTransport struct {
|
||||
base http.RoundTripper
|
||||
host string
|
||||
scheme string
|
||||
client http.Client
|
||||
}
|
||||
|
||||
// RoundTrip implements the http.Roundtripper interface.
|
||||
func (t *customTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
// Rountrippers should not modify the original request.
|
||||
newReq := req.Clone(context.Background())
|
||||
*newReq.URL = *req.URL
|
||||
req.URL.Scheme = t.scheme
|
||||
req.URL.Host = t.host
|
||||
return t.client.Do(req)
|
||||
}
|
||||
|
||||
// customProvider is a dummy credentials provider for the minio client to work
|
||||
// without actually providing credentials. This is needed with a custom transport
|
||||
// in cases where the minio client does not actually have credentials with itself,
|
||||
// rather needs responses from another entity.
|
||||
//
|
||||
// It satisfies the credentials.Provider interface.
|
||||
type customProvider struct {
|
||||
isSignV2 bool
|
||||
}
|
||||
|
||||
// Retrieve just returns empty credentials.
|
||||
func (cp customProvider) Retrieve() (credentials.Value, error) {
|
||||
sign := credentials.SignatureV4
|
||||
if cp.isSignV2 {
|
||||
sign = credentials.SignatureV2
|
||||
}
|
||||
return credentials.Value{
|
||||
SignerType: sign,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// IsExpired always returns false.
|
||||
func (cp customProvider) IsExpired() bool { return false }
|
||||
@@ -1,425 +0,0 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package filesstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
s3 "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio-go/v7/pkg/encrypt"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/mattermost/mattermost-server/v5/mlog"
|
||||
)
|
||||
|
||||
// S3FileBackend contains all necessary information to communicate with
|
||||
// an AWS S3 compatible API backend.
|
||||
type S3FileBackend struct {
|
||||
endpoint string
|
||||
accessKey string
|
||||
secretKey string
|
||||
secure bool
|
||||
signV2 bool
|
||||
region string
|
||||
bucket string
|
||||
pathPrefix string
|
||||
encrypt bool
|
||||
trace bool
|
||||
client *s3.Client
|
||||
}
|
||||
|
||||
const (
|
||||
// This is not exported by minio. See: https://github.com/minio/minio-go/issues/1339
|
||||
bucketNotFound = "NoSuchBucket"
|
||||
)
|
||||
|
||||
var (
|
||||
imageExtensions = map[string]bool{".jpg": true, ".jpeg": true, ".gif": true, ".bmp": true, ".png": true, ".tiff": true, "tif": true}
|
||||
imageMimeTypes = map[string]string{".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".gif": "image/gif", ".bmp": "image/bmp", ".png": "image/png", ".tiff": "image/tiff", ".tif": "image/tif"}
|
||||
)
|
||||
|
||||
func isFileExtImage(ext string) bool {
|
||||
ext = strings.ToLower(ext)
|
||||
return imageExtensions[ext]
|
||||
}
|
||||
|
||||
func getImageMimeType(ext string) string {
|
||||
ext = strings.ToLower(ext)
|
||||
if imageMimeTypes[ext] == "" {
|
||||
return "image"
|
||||
}
|
||||
return imageMimeTypes[ext]
|
||||
}
|
||||
|
||||
// NewS3FileBackend returns an instance of an S3FileBackend.
|
||||
func NewS3FileBackend(settings FileBackendSettings) (*S3FileBackend, error) {
|
||||
backend := &S3FileBackend{
|
||||
endpoint: settings.AmazonS3Endpoint,
|
||||
accessKey: settings.AmazonS3AccessKeyId,
|
||||
secretKey: settings.AmazonS3SecretAccessKey,
|
||||
secure: settings.AmazonS3SSL,
|
||||
signV2: settings.AmazonS3SignV2,
|
||||
region: settings.AmazonS3Region,
|
||||
bucket: settings.AmazonS3Bucket,
|
||||
pathPrefix: settings.AmazonS3PathPrefix,
|
||||
encrypt: settings.AmazonS3SSE,
|
||||
trace: settings.AmazonS3Trace,
|
||||
}
|
||||
cli, err := backend.s3New()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
backend.client = cli
|
||||
return backend, nil
|
||||
}
|
||||
|
||||
// Similar to s3.New() but allows initialization of signature v2 or signature v4 client.
|
||||
// If signV2 input is false, function always returns signature v4.
|
||||
//
|
||||
// Additionally this function also takes a user defined region, if set
|
||||
// disables automatic region lookup.
|
||||
func (b *S3FileBackend) s3New() (*s3.Client, error) {
|
||||
var creds *credentials.Credentials
|
||||
|
||||
isCloud := os.Getenv("MM_CLOUD_FILESTORE_BIFROST") != ""
|
||||
if isCloud {
|
||||
creds = credentials.New(customProvider{isSignV2: b.signV2})
|
||||
} else if b.accessKey == "" && b.secretKey == "" {
|
||||
creds = credentials.NewIAM("")
|
||||
} else if b.signV2 {
|
||||
creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV2)
|
||||
} else {
|
||||
creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV4)
|
||||
}
|
||||
|
||||
opts := s3.Options{
|
||||
Creds: creds,
|
||||
Secure: b.secure,
|
||||
Region: b.region,
|
||||
}
|
||||
|
||||
// If this is a cloud installation, we override the default transport.
|
||||
if isCloud {
|
||||
tr, err := s3.DefaultTransport(b.secure)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scheme := "http"
|
||||
if b.secure {
|
||||
scheme = "https"
|
||||
}
|
||||
opts.Transport = &customTransport{
|
||||
base: tr,
|
||||
host: b.endpoint,
|
||||
scheme: scheme,
|
||||
}
|
||||
}
|
||||
|
||||
s3Clnt, err := s3.New(b.endpoint, &opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if b.trace {
|
||||
s3Clnt.TraceOn(os.Stdout)
|
||||
}
|
||||
|
||||
return s3Clnt, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) TestConnection() error {
|
||||
exists := true
|
||||
var err error
|
||||
// If a path prefix is present, we attempt to test the bucket by listing objects under the path
|
||||
// and just checking the first response. This is because the BucketExists call is only at a bucket level
|
||||
// and sometimes the user might only be allowed access to the specified path prefix.
|
||||
if b.pathPrefix != "" {
|
||||
obj := <-b.client.ListObjects(context.Background(), b.bucket, s3.ListObjectsOptions{Prefix: b.pathPrefix})
|
||||
if obj.Err != nil {
|
||||
typedErr := s3.ToErrorResponse(obj.Err)
|
||||
if typedErr.Code != bucketNotFound {
|
||||
return errors.Wrap(err, "unable to list objects in the s3 bucket")
|
||||
}
|
||||
exists = false
|
||||
}
|
||||
} else {
|
||||
exists, err = b.client.BucketExists(context.Background(), b.bucket)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "unable to check if the s3 bucket exists")
|
||||
}
|
||||
}
|
||||
|
||||
if exists {
|
||||
mlog.Debug("Connection to S3 or minio is good. Bucket exists.")
|
||||
} else {
|
||||
mlog.Warn("Bucket specified does not exist. Attempting to create...")
|
||||
err := b.client.MakeBucket(context.Background(), b.bucket, s3.MakeBucketOptions{Region: b.region})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "unable to create the s3 bucket")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Caller must close the first return value
|
||||
func (b *S3FileBackend) Reader(path string) (ReadCloseSeeker, error) {
|
||||
path = filepath.Join(b.pathPrefix, path)
|
||||
minioObject, err := b.client.GetObject(context.Background(), b.bucket, path, s3.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "unable to open file %s", path)
|
||||
}
|
||||
|
||||
return minioObject, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) ReadFile(path string) ([]byte, error) {
|
||||
path = filepath.Join(b.pathPrefix, path)
|
||||
minioObject, err := b.client.GetObject(context.Background(), b.bucket, path, s3.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "unable to open file %s", path)
|
||||
}
|
||||
|
||||
defer minioObject.Close()
|
||||
f, err := ioutil.ReadAll(minioObject)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "unable to read file %s", path)
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) FileExists(path string) (bool, error) {
|
||||
path = filepath.Join(b.pathPrefix, path)
|
||||
|
||||
_, err := b.client.StatObject(context.Background(), b.bucket, path, s3.StatObjectOptions{})
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
var s3Err s3.ErrorResponse
|
||||
if errors.As(err, &s3Err); s3Err.Code == "NoSuchKey" {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return false, errors.Wrapf(err, "unable to know if file %s exists", path)
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) FileSize(path string) (int64, error) {
|
||||
path = filepath.Join(b.pathPrefix, path)
|
||||
|
||||
info, err := b.client.StatObject(context.Background(), b.bucket, path, s3.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "unable to get file size for %s", path)
|
||||
}
|
||||
|
||||
return info.Size, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) FileModTime(path string) (time.Time, error) {
|
||||
path = filepath.Join(b.pathPrefix, path)
|
||||
|
||||
info, err := b.client.StatObject(context.Background(), b.bucket, path, s3.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return time.Time{}, errors.Wrapf(err, "unable to get modification time for file %s", path)
|
||||
}
|
||||
|
||||
return info.LastModified, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) CopyFile(oldPath, newPath string) error {
|
||||
oldPath = filepath.Join(b.pathPrefix, oldPath)
|
||||
newPath = filepath.Join(b.pathPrefix, newPath)
|
||||
srcOpts := s3.CopySrcOptions{
|
||||
Bucket: b.bucket,
|
||||
Object: oldPath,
|
||||
Encryption: encrypt.NewSSE(),
|
||||
}
|
||||
dstOpts := s3.CopyDestOptions{
|
||||
Bucket: b.bucket,
|
||||
Object: newPath,
|
||||
Encryption: encrypt.NewSSE(),
|
||||
}
|
||||
if _, err := b.client.CopyObject(context.Background(), dstOpts, srcOpts); err != nil {
|
||||
return errors.Wrapf(err, "unable to copy file from %s to %s", oldPath, newPath)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) MoveFile(oldPath, newPath string) error {
|
||||
oldPath = filepath.Join(b.pathPrefix, oldPath)
|
||||
newPath = filepath.Join(b.pathPrefix, newPath)
|
||||
srcOpts := s3.CopySrcOptions{
|
||||
Bucket: b.bucket,
|
||||
Object: oldPath,
|
||||
Encryption: encrypt.NewSSE(),
|
||||
}
|
||||
dstOpts := s3.CopyDestOptions{
|
||||
Bucket: b.bucket,
|
||||
Object: newPath,
|
||||
Encryption: encrypt.NewSSE(),
|
||||
}
|
||||
|
||||
if _, err := b.client.CopyObject(context.Background(), dstOpts, srcOpts); err != nil {
|
||||
return errors.Wrapf(err, "unable to copy the file to %s to the new destionation", newPath)
|
||||
}
|
||||
|
||||
if err := b.client.RemoveObject(context.Background(), b.bucket, oldPath, s3.RemoveObjectOptions{}); err != nil {
|
||||
return errors.Wrapf(err, "unable to remove the file old file %s", oldPath)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) WriteFile(fr io.Reader, path string) (int64, error) {
|
||||
var contentType string
|
||||
path = filepath.Join(b.pathPrefix, path)
|
||||
if ext := filepath.Ext(path); isFileExtImage(ext) {
|
||||
contentType = getImageMimeType(ext)
|
||||
} else {
|
||||
contentType = "binary/octet-stream"
|
||||
}
|
||||
|
||||
options := s3PutOptions(b.encrypt, contentType)
|
||||
info, err := b.client.PutObject(context.Background(), b.bucket, path, fr, -1, options)
|
||||
if err != nil {
|
||||
return info.Size, errors.Wrapf(err, "unable write the data in the file %s", path)
|
||||
}
|
||||
|
||||
return info.Size, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) AppendFile(fr io.Reader, path string) (int64, error) {
|
||||
fp := filepath.Join(b.pathPrefix, path)
|
||||
if _, err := b.client.StatObject(context.Background(), b.bucket, fp, s3.StatObjectOptions{}); err != nil {
|
||||
return 0, errors.Wrapf(err, "unable to find the file %s to append the data", path)
|
||||
}
|
||||
|
||||
var contentType string
|
||||
if ext := filepath.Ext(fp); isFileExtImage(ext) {
|
||||
contentType = getImageMimeType(ext)
|
||||
} else {
|
||||
contentType = "binary/octet-stream"
|
||||
}
|
||||
|
||||
options := s3PutOptions(b.encrypt, contentType)
|
||||
sse := options.ServerSideEncryption
|
||||
partName := fp + ".part"
|
||||
info, err := b.client.PutObject(context.Background(), b.bucket, partName, fr, -1, options)
|
||||
defer b.client.RemoveObject(context.Background(), b.bucket, partName, s3.RemoveObjectOptions{})
|
||||
if info.Size > 0 {
|
||||
src1Opts := s3.CopySrcOptions{
|
||||
Bucket: b.bucket,
|
||||
Object: fp,
|
||||
}
|
||||
src2Opts := s3.CopySrcOptions{
|
||||
Bucket: b.bucket,
|
||||
Object: partName,
|
||||
}
|
||||
dstOpts := s3.CopyDestOptions{
|
||||
Bucket: b.bucket,
|
||||
Object: fp,
|
||||
Encryption: sse,
|
||||
}
|
||||
_, err = b.client.ComposeObject(context.Background(), dstOpts, src1Opts, src2Opts)
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "unable append the data in the file %s", path)
|
||||
}
|
||||
return info.Size, nil
|
||||
}
|
||||
|
||||
return 0, errors.Wrapf(err, "unable append the data in the file %s", path)
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) RemoveFile(path string) error {
|
||||
path = filepath.Join(b.pathPrefix, path)
|
||||
if err := b.client.RemoveObject(context.Background(), b.bucket, path, s3.RemoveObjectOptions{}); err != nil {
|
||||
return errors.Wrapf(err, "unable to remove the file %s", path)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan s3.ObjectInfo {
|
||||
out := make(chan s3.ObjectInfo, 1)
|
||||
|
||||
go func() {
|
||||
defer close(out)
|
||||
|
||||
for {
|
||||
info, done := <-in
|
||||
|
||||
if !done {
|
||||
break
|
||||
}
|
||||
|
||||
out <- info
|
||||
}
|
||||
}()
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) ListDirectory(path string) ([]string, error) {
|
||||
path = filepath.Join(b.pathPrefix, path)
|
||||
if !strings.HasSuffix(path, "/") && path != "" {
|
||||
// s3Clnt returns only the path itself when "/" is not present
|
||||
// appending "/" to make it consistent across all filesstores
|
||||
path = path + "/"
|
||||
}
|
||||
|
||||
opts := s3.ListObjectsOptions{
|
||||
Prefix: path,
|
||||
}
|
||||
var paths []string
|
||||
for object := range b.client.ListObjects(context.Background(), b.bucket, opts) {
|
||||
if object.Err != nil {
|
||||
return nil, errors.Wrapf(object.Err, "unable to list the directory %s", path)
|
||||
}
|
||||
// We strip the path prefix that gets applied,
|
||||
// so that it remains transparent to the application.
|
||||
object.Key = strings.TrimPrefix(object.Key, b.pathPrefix)
|
||||
trimmed := strings.Trim(object.Key, "/")
|
||||
if trimmed != "" {
|
||||
paths = append(paths, trimmed)
|
||||
}
|
||||
}
|
||||
|
||||
return paths, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) RemoveDirectory(path string) error {
|
||||
opts := s3.ListObjectsOptions{
|
||||
Prefix: filepath.Join(b.pathPrefix, path),
|
||||
Recursive: true,
|
||||
}
|
||||
list := b.client.ListObjects(context.Background(), b.bucket, opts)
|
||||
objectsCh := b.client.RemoveObjects(context.Background(), b.bucket, getPathsFromObjectInfos(list), s3.RemoveObjectsOptions{})
|
||||
for err := range objectsCh {
|
||||
if err.Err != nil {
|
||||
return errors.Wrapf(err.Err, "unable to remove the directory %s", path)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func s3PutOptions(encrypted bool, contentType string) s3.PutObjectOptions {
|
||||
options := s3.PutObjectOptions{}
|
||||
if encrypted {
|
||||
options.ServerSideEncryption = encrypt.NewSSE()
|
||||
}
|
||||
options.ContentType = contentType
|
||||
// We set the part size to the minimum allowed value of 5MBs
|
||||
// to avoid an excessive allocation in minio.PutObject implementation.
|
||||
options.PartSize = 1024 * 1024 * 5
|
||||
|
||||
return options
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package filesstore
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCheckMandatoryS3Fields(t *testing.T) {
|
||||
cfg := FileBackendSettings{}
|
||||
|
||||
err := cfg.CheckMandatoryS3Fields()
|
||||
require.Error(t, err)
|
||||
require.Equal(t, err.Error(), "missing s3 bucket settings", "should've failed with missing s3 bucket")
|
||||
|
||||
cfg.AmazonS3Bucket = "test-mm"
|
||||
err = cfg.CheckMandatoryS3Fields()
|
||||
require.NoError(t, err)
|
||||
|
||||
cfg.AmazonS3Endpoint = ""
|
||||
err = cfg.CheckMandatoryS3Fields()
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, "s3.amazonaws.com", cfg.AmazonS3Endpoint, "should've set the endpoint to the default")
|
||||
}
|
||||
Ссылка в новой задаче
Block a user