[MM-26812] Add support for resumable file uploads (#15252)

* Implement AppendFile for FileBackend * Split test into subtests * [MM-26812] Add support for resumable file uploads (#15252) * Implement UploadSession * Implement UploadSessionStore * Add error strings * Implement resumable file uploads * Add UploadType * Fix retry layer tests * Regenerate store layers * Fix store error handling * Use base for filename * Prevent concurrent uploads on the same upload session * Fix erroneus error string * Improve error handling Co-authored-by: Mattermod <mattermod@users.noreply.github.com> * Fix translations Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
2020-09-15 21:28:25 +02:00
--- a/services/filesstore/filesstore.go
+++ b/services/filesstore/filesstore.go
@@ -24,6 +24,7 @@ type FileBackend interface {
 	CopyFile(oldPath, newPath string) *model.AppError
 	MoveFile(oldPath, newPath string) *model.AppError
 	WriteFile(fr io.Reader, path string) (int64, *model.AppError)
+	AppendFile(fr io.Reader, path string) (int64, *model.AppError)
 	RemoveFile(path string) *model.AppError

 	ListDirectory(path string) (*[]string, *model.AppError)
--- a/services/filesstore/filesstore_test.go
+++ b/services/filesstore/filesstore_test.go
@@ -308,3 +308,56 @@ func (s *FileBackendTestSuite) TestRemoveDirectory() {
 	_, err = s.backend.ReadFile("tests2/asdf")
 	s.Error(err)
 }
+
+func (s *FileBackendTestSuite) TestAppendFile() {
+	s.Run("should fail if target file is missing", func() {
+		path := "tests/" + model.NewId()
+		b := make([]byte, 1024)
+		written, err := s.backend.AppendFile(bytes.NewReader(b), path)
+		s.Error(err)
+		s.Zero(written)
+	})
+
+	s.Run("should correctly append the data", func() {
+		// First part needs to be at least 5MB for the S3 implementation to work.
+		size := 5 * 1024 * 1024
+		b := make([]byte, size)
+		for i := range b {
+			b[i] = 'A'
+		}
+		path := "tests/" + model.NewId()
+
+		written, err := s.backend.WriteFile(bytes.NewReader(b), path)
+		s.Nil(err)
+		s.EqualValues(len(b), written)
+		defer s.backend.RemoveFile(path)
+
+		b2 := make([]byte, 1024)
+		for i := range b2 {
+			b2[i] = 'B'
+		}
+
+		written, err = s.backend.AppendFile(bytes.NewReader(b2), path)
+		s.Nil(err)
+		s.EqualValues(int64(len(b2)), written)
+
+		read, err := s.backend.ReadFile(path)
+		s.Nil(err)
+		s.EqualValues(len(b)+len(b2), len(read))
+		s.EqualValues(append(b, b2...), read)
+
+		b3 := make([]byte, 1024)
+		for i := range b3 {
+			b3[i] = 'C'
+		}
+
+		written, err = s.backend.AppendFile(bytes.NewReader(b3), path)
+		s.Nil(err)
+		s.EqualValues(int64(len(b3)), written)
+
+		read, err = s.backend.ReadFile(path)
+		s.Nil(err)
+		s.EqualValues(len(b)+len(b2)+len(b3), len(read))
+		s.EqualValues(append(append(b, b2...), b3...), read)
+	})
+}
--- a/services/filesstore/localstore.go
+++ b/services/filesstore/localstore.go
@@ -103,6 +103,23 @@ func writeFileLocally(fr io.Reader, path string) (int64, *model.AppError) {
 	return written, nil
 }

+func (b *LocalFileBackend) AppendFile(fr io.Reader, path string) (int64, *model.AppError) {
+	fp := filepath.Join(b.directory, path)
+	if _, err := os.Stat(fp); err != nil {
+		return 0, model.NewAppError("AppendFile", "api.file.append_file.no_exist.app_error", nil, err.Error(), http.StatusInternalServerError)
+	}
+	fw, err := os.OpenFile(fp, os.O_WRONLY|os.O_APPEND, 0600)
+	if err != nil {
+		return 0, model.NewAppError("AppendFile", "api.file.append_file.opening.app_error", nil, err.Error(), http.StatusInternalServerError)
+	}
+	defer fw.Close()
+	written, err := io.Copy(fw, fr)
+	if err != nil {
+		return written, model.NewAppError("AppendFile", "api.file.append_file.writing.app_error", nil, err.Error(), http.StatusInternalServerError)
+	}
+	return written, nil
+}
+
 func (b *LocalFileBackend) RemoveFile(path string) *model.AppError {
 	if err := os.Remove(filepath.Join(b.directory, path)); err != nil {
 		return model.NewAppError("RemoveFile", "utils.file.remove_file.local.app_error", nil, err.Error(), http.StatusInternalServerError)
--- a/services/filesstore/mocks/FileBackend.go
+++ b/services/filesstore/mocks/FileBackend.go
@@ -19,6 +19,29 @@ type FileBackend struct {
 	mock.Mock
 }

+// AppendFile provides a mock function with given fields: fr, path
+func (_m *FileBackend) AppendFile(fr io.Reader, path string) (int64, *model.AppError) {
+	ret := _m.Called(fr, path)
+
+	var r0 int64
+	if rf, ok := ret.Get(0).(func(io.Reader, string) int64); ok {
+		r0 = rf(fr, path)
+	} else {
+		r0 = ret.Get(0).(int64)
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(io.Reader, string) *model.AppError); ok {
+		r1 = rf(fr, path)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
 // CopyFile provides a mock function with given fields: oldPath, newPath
 func (_m *FileBackend) CopyFile(oldPath string, newPath string) *model.AppError {
 	ret := _m.Called(oldPath, newPath)
--- a/services/filesstore/s3store.go
+++ b/services/filesstore/s3store.go
@@ -4,7 +4,6 @@
 package filesstore

 import (
-	"bytes"
 	"context"
 	"errors"
 	"io"
@@ -218,12 +217,7 @@ func (b *S3FileBackend) WriteFile(fr io.Reader, path string) (int64, *model.AppE
 	}

 	options := s3PutOptions(b.encrypt, contentType)
-	var buf bytes.Buffer
-	_, err = buf.ReadFrom(fr)
-	if err != nil {
-		return 0, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
-	}
-	info, err := s3Clnt.PutObject(context.Background(), b.bucket, path, &buf, int64(buf.Len()), options)
+	info, err := s3Clnt.PutObject(context.Background(), b.bucket, path, fr, -1, options)
 	if err != nil {
 		return info.Size, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
 	}
@@ -231,6 +225,65 @@ func (b *S3FileBackend) WriteFile(fr io.Reader, path string) (int64, *model.AppE
 	return info.Size, nil
 }

+func (b *S3FileBackend) AppendFile(fr io.Reader, path string) (int64, *model.AppError) {
+	s3Clnt, err := b.s3New()
+	if err != nil {
+		return 0, model.NewAppError("AppendFile", "api.file.append_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
+	}
+
+	fp := filepath.Join(b.pathPrefix, path)
+	if _, err = s3Clnt.StatObject(context.Background(), b.bucket, fp, s3.StatObjectOptions{}); err != nil {
+		return 0, model.NewAppError("AppendFile", "api.file.append_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
+	}
+
+	var contentType string
+	if ext := filepath.Ext(fp); model.IsFileExtImage(ext) {
+		contentType = model.GetImageMimeType(ext)
+	} else {
+		contentType = "binary/octet-stream"
+	}
+
+	var sse encrypt.ServerSide
+	if b.encrypt {
+		sse = encrypt.NewSSE()
+	}
+	options := s3.PutObjectOptions{
+		ContentType:          contentType,
+		ServerSideEncryption: sse,
+	}
+
+	partName := fp + ".part"
+	info, err := s3Clnt.PutObject(context.Background(), b.bucket, partName, fr, -1, options)
+	defer s3Clnt.RemoveObject(context.Background(), b.bucket, partName, s3.RemoveObjectOptions{})
+	if info.Size > 0 {
+		src1Opts := s3.CopySrcOptions{
+			Bucket: b.bucket,
+			Object: fp,
+		}
+		src2Opts := s3.CopySrcOptions{
+			Bucket: b.bucket,
+			Object: partName,
+		}
+		dstOpts := s3.CopyDestOptions{
+			Bucket:     b.bucket,
+			Object:     fp,
+			Encryption: sse,
+		}
+		_, err = s3Clnt.ComposeObject(context.Background(), dstOpts, src1Opts, src2Opts)
+		if err != nil {
+			return 0, model.NewAppError("AppendFile", "api.file.append_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
+		}
+		return info.Size, nil
+	}
+
+	var errString string
+	if err != nil {
+		errString = err.Error()
+	}
+
+	return 0, model.NewAppError("AppendFile", "api.file.append_file.s3.app_error", nil, errString, http.StatusInternalServerError)
+}
+
 func (b *S3FileBackend) RemoveFile(path string) *model.AppError {
 	s3Clnt, err := b.s3New()
 	if err != nil {