[MM-26812] Add support for resumable file uploads (#15252)

* Implement AppendFile for FileBackend

* Split test into subtests

* [MM-26812] Add support for resumable file uploads (#15252)

* Implement UploadSession

* Implement UploadSessionStore

* Add error strings

* Implement resumable file uploads

* Add UploadType

* Fix retry layer tests

* Regenerate store layers

* Fix store error handling

* Use base for filename

* Prevent concurrent uploads on the same upload session

* Fix erroneus error string

* Improve error handling

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

* Fix translations

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

model/client4.go

@@ -313,6 +313,14 @@ func (c *Client4) GetFileRoute(fileId string) string {
 	return fmt.Sprintf(c.GetFilesRoute()+"/%v", fileId)
 }
 
+func (c *Client4) GetUploadsRoute() string {
+	return "/uploads"
+}
+
+func (c *Client4) GetUploadRoute(uploadId string) string {
+	return fmt.Sprintf("%s/%s", c.GetUploadsRoute(), uploadId)
+}
+
 func (c *Client4) GetPluginsRoute() string {
 	return "/plugins"
 }
@@ -5531,3 +5539,46 @@ func (c *Client4) CheckIntegrity() ([]IntegrityCheckResult, *Response) {
 	}
 	return results, BuildResponse(r)
 }
+
+// CreateUpload creates a new upload session.
+func (c *Client4) CreateUpload(us *UploadSession) (*UploadSession, *Response) {
+	r, err := c.DoApiPost(c.GetUploadsRoute(), us.ToJson())
+	if err != nil {
+		return nil, BuildErrorResponse(r, err)
+	}
+	defer closeBody(r)
+	return UploadSessionFromJson(r.Body), BuildResponse(r)
+}
+
+// GetUpload returns the upload session for the specified uploadId.
+func (c *Client4) GetUpload(uploadId string) (*UploadSession, *Response) {
+	r, err := c.DoApiGet(c.GetUploadRoute(uploadId), "")
+	if err != nil {
+		return nil, BuildErrorResponse(r, err)
+	}
+	defer closeBody(r)
+	return UploadSessionFromJson(r.Body), BuildResponse(r)
+}
+
+// GetUploadsForUser returns the upload sessions created by the specified
+// userId.
+func (c *Client4) GetUploadsForUser(userId string) ([]*UploadSession, *Response) {
+	r, err := c.DoApiGet(c.GetUserRoute(userId)+"/uploads", "")
+	if err != nil {
+		return nil, BuildErrorResponse(r, err)
+	}
+	defer closeBody(r)
+	return UploadSessionsFromJson(r.Body), BuildResponse(r)
+}
+
+// UploadData performs an upload. On success it returns
+// a FileInfo object.
+func (c *Client4) UploadData(uploadId string, data io.Reader) (*FileInfo, *Response) {
+	url := c.GetUploadRoute(uploadId)
+	r, err := c.doApiRequestReader("POST", c.ApiUrl+url, data, "")
+	if err != nil {
+		return nil, BuildErrorResponse(r, err)
+	}
+	defer closeBody(r)
+	return FileInfoFromJson(r.Body), BuildResponse(r)
+}

model/file_info.go

@@ -4,7 +4,6 @@
 package model
 
 import (
-	"bytes"
 	"encoding/json"
 	"image"
 	"image/gif"
@@ -151,10 +150,10 @@ func NewInfo(name string) *FileInfo {
 	return info
 }
 
-func GetInfoForBytes(name string, data []byte) (*FileInfo, *AppError) {
+func GetInfoForBytes(name string, data io.ReadSeeker, size int) (*FileInfo, *AppError) {
 	info := &FileInfo{
 		Name: name,
-		Size: int64(len(data)),
+		Size: int64(size),
 	}
 	var err *AppError
 
@@ -170,16 +169,17 @@ func GetInfoForBytes(name string, data []byte) (*FileInfo, *AppError) {
 
 	if info.IsImage() {
 		// Only set the width and height if it's actually an image that we can understand
-		if config, _, err := image.DecodeConfig(bytes.NewReader(data)); err == nil {
+		if config, _, err := image.DecodeConfig(data); err == nil {
 			info.Width = config.Width
 			info.Height = config.Height
 
 			if info.MimeType == "image/gif" {
 				// Just show the gif itself instead of a preview image for animated gifs
-				if gifConfig, err := gif.DecodeAll(bytes.NewReader(data)); err != nil {
+				data.Seek(0, io.SeekStart)
+				if gifConfig, err := gif.DecodeAll(data); err != nil {
 					// Still return the rest of the info even though it doesn't appear to be an actual gif
 					info.HasPreviewImage = true
-					return info, NewAppError("GetInfoForBytes", "model.file_info.get.gif.app_error", nil, "name="+name, http.StatusBadRequest)
+					return info, NewAppError("GetInfoForBytes", "model.file_info.get.gif.app_error", nil, err.Error(), http.StatusBadRequest)
 				} else {
 					info.HasPreviewImage = len(gifConfig.Image) == 1
 				}

model/file_info_test.go

@@ -4,6 +4,7 @@
 package model
 
 import (
+	"bytes"
 	"encoding/base64"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -185,7 +186,7 @@ func TestGetInfoForFile(t *testing.T) {
 
 	for _, tc := range ttc {
 		t.Run(tc.testName, func(t *testing.T) {
-			info, errApp := GetInfoForBytes(tc.filename, tc.file)
+			info, errApp := GetInfoForBytes(tc.filename, bytes.NewReader(tc.file), len(tc.file))
 			require.Nil(t, errApp)
 
 			assert.Equalf(t, tc.filename, info.Name, "Got incorrect filename: %v", info.Name)

model/upload_session.go

@@ -0,0 +1,141 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package model
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+)
+
+// UploadType defines the type of an upload.
+type UploadType string
+
+const (
+	UploadTypeAttachment UploadType = "attachment"
+	UploadTypeImport     UploadType = "import"
+)
+
+// UploadSession contains information used to keep track of a file upload.
+type UploadSession struct {
+	// The unique identifier for the session.
+	Id string `json:"id"`
+	// The type of the upload.
+	Type UploadType `json:"type"`
+	// The timestamp of creation.
+	CreateAt int64 `json:"create_at"`
+	// The id of the user performing the upload.
+	UserId string `json:"user_id"`
+	// The id of the channel to upload to.
+	ChannelId string `json:"channel_id"`
+	// The name of the file to upload.
+	Filename string `json:"filename"`
+	// The path where the file is stored.
+	Path string `json:"-"`
+	// The size of the file to upload.
+	FileSize int64 `json:"file_size"`
+	// The amount of received data in bytes. If equal to FileSize it means the
+	// upload has finished.
+	FileOffset int64 `json:"file_offset"`
+}
+
+// ToJson serializes the UploadSession into JSON and returns it as string.
+func (us *UploadSession) ToJson() string {
+	b, _ := json.Marshal(us)
+	return string(b)
+}
+
+// UploadSessionsToJson serializes a list of UploadSession into JSON and
+// returns it as string.
+func UploadSessionsToJson(uss []*UploadSession) string {
+	b, _ := json.Marshal(uss)
+	return string(b)
+}
+
+// UploadSessionsFromJson deserializes a list of UploadSession from JSON data.
+func UploadSessionsFromJson(data io.Reader) []*UploadSession {
+	decoder := json.NewDecoder(data)
+	var uss []*UploadSession
+	if err := decoder.Decode(&uss); err != nil {
+		return nil
+	}
+	return uss
+}
+
+// UploadSessionFromJson deserializes the UploadSession from JSON data.
+func UploadSessionFromJson(data io.Reader) *UploadSession {
+	decoder := json.NewDecoder(data)
+	var us UploadSession
+	if err := decoder.Decode(&us); err != nil {
+		return nil
+	}
+	return &us
+}
+
+// PreSave is a utility function used to fill required information.
+func (us *UploadSession) PreSave() {
+	if us.Id == "" {
+		us.Id = NewId()
+	}
+
+	if us.CreateAt == 0 {
+		us.CreateAt = GetMillis()
+	}
+}
+
+// IsValid validates an UploadType. It returns an error in case of
+// failure.
+func (t UploadType) IsValid() error {
+	switch t {
+	case UploadTypeAttachment:
+		return nil
+	case UploadTypeImport:
+		return nil
+	default:
+	}
+	return fmt.Errorf("invalid UploadType %s", t)
+}
+
+// IsValid validates an UploadSession. It returns an error in case of
+// failure.
+func (us *UploadSession) IsValid() *AppError {
+	if !IsValidId(us.Id) {
+		return NewAppError("UploadSession.IsValid", "model.upload_session.is_valid.id.app_error", nil, "", http.StatusBadRequest)
+	}
+
+	if err := us.Type.IsValid(); err != nil {
+		return NewAppError("UploadSession.IsValid", "model.upload_session.is_valid.type.app_error", nil, err.Error(), http.StatusBadRequest)
+	}
+
+	if !IsValidId(us.UserId) {
+		return NewAppError("UploadSession.IsValid", "model.upload_session.is_valid.user_id.app_error", nil, "id="+us.Id, http.StatusBadRequest)
+	}
+
+	if us.Type == UploadTypeAttachment && !IsValidId(us.ChannelId) {
+		return NewAppError("UploadSession.IsValid", "model.upload_session.is_valid.channel_id.app_error", nil, "id="+us.Id, http.StatusBadRequest)
+	}
+
+	if us.CreateAt == 0 {
+		return NewAppError("UploadSession.IsValid", "model.upload_session.is_valid.create_at.app_error", nil, "id="+us.Id, http.StatusBadRequest)
+	}
+
+	if us.Filename == "" {
+		return NewAppError("UploadSession.IsValid", "model.upload_session.is_valid.filename.app_error", nil, "id="+us.Id, http.StatusBadRequest)
+	}
+
+	if us.FileSize <= 0 {
+		return NewAppError("UploadSession.IsValid", "model.upload_session.is_valid.file_size.app_error", nil, "id="+us.Id, http.StatusBadRequest)
+	}
+
+	if us.FileOffset < 0 || us.FileOffset > us.FileSize {
+		return NewAppError("UploadSession.IsValid", "model.upload_session.is_valid.file_offset.app_error", nil, "id="+us.Id, http.StatusBadRequest)
+	}
+
+	if us.Path == "" {
+		return NewAppError("UploadSession.IsValid", "model.upload_session.is_valid.path.app_error", nil, "id="+us.Id, http.StatusBadRequest)
+	}
+
+	return nil
+}

model/upload_session_test.go

@@ -0,0 +1,125 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package model
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestUploadSessionIsValid(t *testing.T) {
+	var session UploadSession
+
+	t.Run("empty session should fail", func(t *testing.T) {
+		err := session.IsValid()
+		require.NotNil(t, err)
+	})
+
+	t.Run("valid session should succeed", func(t *testing.T) {
+		session = UploadSession{
+			Id:         NewId(),
+			Type:       UploadTypeAttachment,
+			CreateAt:   GetMillis(),
+			UserId:     NewId(),
+			ChannelId:  NewId(),
+			Filename:   "test",
+			Path:       "/tmp/test",
+			FileSize:   1024,
+			FileOffset: 0,
+		}
+		err := session.IsValid()
+		require.Nil(t, err)
+	})
+
+	t.Run("invalid Id should fail", func(t *testing.T) {
+		us := session
+		us.Id = "invalid"
+		err := us.IsValid()
+		require.NotNil(t, err)
+		require.Equal(t, "model.upload_session.is_valid.id.app_error", err.Id)
+	})
+
+	t.Run("invalid type should fail", func(t *testing.T) {
+		us := session
+		us.Type = "invalid"
+		err := us.IsValid()
+		require.NotNil(t, err)
+		require.Equal(t, "model.upload_session.is_valid.type.app_error", err.Id)
+	})
+
+	t.Run("invalid CreateAt should fail", func(t *testing.T) {
+		us := session
+		us.CreateAt = 0
+		err := us.IsValid()
+		require.NotNil(t, err)
+		require.Equal(t, "model.upload_session.is_valid.create_at.app_error", err.Id)
+	})
+
+	t.Run("invalid UserId should fail", func(t *testing.T) {
+		us := session
+		us.UserId = "invalid"
+		err := us.IsValid()
+		require.NotNil(t, err)
+		require.Equal(t, "model.upload_session.is_valid.user_id.app_error", err.Id)
+	})
+
+	t.Run("invalid ChannelId should fail", func(t *testing.T) {
+		us := session
+		us.ChannelId = "invalid"
+		err := us.IsValid()
+		require.NotNil(t, err)
+		require.Equal(t, "model.upload_session.is_valid.channel_id.app_error", err.Id)
+	})
+
+	t.Run("ChannelId is not validated if type is not attachment", func(t *testing.T) {
+		us := session
+		us.ChannelId = ""
+		us.Type = UploadTypeImport
+		err := us.IsValid()
+		require.Nil(t, err)
+	})
+
+	t.Run("invalid Filename should fail", func(t *testing.T) {
+		us := session
+		us.Filename = ""
+		err := us.IsValid()
+		require.NotNil(t, err)
+		require.Equal(t, "model.upload_session.is_valid.filename.app_error", err.Id)
+	})
+
+	t.Run("invalid Path should fail", func(t *testing.T) {
+		us := session
+		us.Path = ""
+		err := us.IsValid()
+		require.NotNil(t, err)
+		require.Equal(t, "model.upload_session.is_valid.path.app_error", err.Id)
+	})
+
+	t.Run("invalid FileSize should fail", func(t *testing.T) {
+		us := session
+		us.FileSize = 0
+		err := us.IsValid()
+		require.NotNil(t, err)
+		require.Equal(t, "model.upload_session.is_valid.file_size.app_error", err.Id)
+
+		us.FileSize = -1
+		err = us.IsValid()
+		require.NotNil(t, err)
+		require.Equal(t, "model.upload_session.is_valid.file_size.app_error", err.Id)
+	})
+
+	t.Run("invalid FileOffset should fail", func(t *testing.T) {
+		us := session
+		us.FileOffset = us.FileSize + 1
+		err := us.IsValid()
+		require.NotNil(t, err)
+		require.Equal(t, "model.upload_session.is_valid.file_offset.app_error", err.Id)
+
+		us.FileOffset = -1
+		err = us.IsValid()
+		require.NotNil(t, err)
+		require.Equal(t, "model.upload_session.is_valid.file_offset.app_error", err.Id)
+	})
+}