[MM-26812] Add support for resumable file uploads (#15252)

* Implement AppendFile for FileBackend

* Split test into subtests

* [MM-26812] Add support for resumable file uploads (#15252)

* Implement UploadSession

* Implement UploadSessionStore

* Add error strings

* Implement resumable file uploads

* Add UploadType

* Fix retry layer tests

* Regenerate store layers

* Fix store error handling

* Use base for filename

* Prevent concurrent uploads on the same upload session

* Fix erroneus error string

* Improve error handling

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

* Fix translations

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

store/sqlstore/store.go

@@ -92,6 +92,7 @@ type SqlStore interface {
 	Emoji() store.EmojiStore
 	Status() store.StatusStore
 	FileInfo() store.FileInfoStore
+	UploadSession() store.UploadSessionStore
 	Reaction() store.ReactionStore
 	Job() store.JobStore
 	Plugin() store.PluginStore

store/sqlstore/supplier.go

@@ -89,6 +89,7 @@ type SqlSupplierStores struct {
 	emoji                store.EmojiStore
 	status               store.StatusStore
 	fileInfo             store.FileInfoStore
+	uploadSession        store.UploadSessionStore
 	reaction             store.ReactionStore
 	job                  store.JobStore
 	userAccessToken      store.UserAccessTokenStore
@@ -157,6 +158,7 @@ func NewSqlSupplier(settings model.SqlSettings, metrics einterfaces.MetricsInter
 	supplier.stores.emoji = newSqlEmojiStore(supplier, metrics)
 	supplier.stores.status = newSqlStatusStore(supplier)
 	supplier.stores.fileInfo = newSqlFileInfoStore(supplier, metrics)
+	supplier.stores.uploadSession = newSqlUploadSessionStore(supplier)
 	supplier.stores.job = newSqlJobStore(supplier)
 	supplier.stores.userAccessToken = newSqlUserAccessTokenStore(supplier)
 	supplier.stores.channelMemberHistory = newSqlChannelMemberHistoryStore(supplier)
@@ -202,6 +204,7 @@ func NewSqlSupplier(settings model.SqlSettings, metrics einterfaces.MetricsInter
 	supplier.stores.emoji.(*SqlEmojiStore).createIndexesIfNotExists()
 	supplier.stores.status.(*SqlStatusStore).createIndexesIfNotExists()
 	supplier.stores.fileInfo.(*SqlFileInfoStore).createIndexesIfNotExists()
+	supplier.stores.uploadSession.(*SqlUploadSessionStore).createIndexesIfNotExists()
 	supplier.stores.job.(*SqlJobStore).createIndexesIfNotExists()
 	supplier.stores.userAccessToken.(*SqlUserAccessTokenStore).createIndexesIfNotExists()
 	supplier.stores.plugin.(*SqlPluginStore).createIndexesIfNotExists()
@@ -1134,6 +1137,10 @@ func (ss *SqlSupplier) FileInfo() store.FileInfoStore {
 	return ss.stores.fileInfo
 }
 
+func (ss *SqlSupplier) UploadSession() store.UploadSessionStore {
+	return ss.stores.uploadSession
+}
+
 func (ss *SqlSupplier) Reaction() store.ReactionStore {
 	return ss.stores.reaction
 }

store/sqlstore/upload_session_store.go

@@ -0,0 +1,133 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sqlstore
+
+import (
+	"database/sql"
+
+	"github.com/pkg/errors"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/store"
+
+	sq "github.com/Masterminds/squirrel"
+)
+
+type SqlUploadSessionStore struct {
+	SqlStore
+}
+
+func newSqlUploadSessionStore(sqlStore SqlStore) store.UploadSessionStore {
+	s := &SqlUploadSessionStore{
+		SqlStore: sqlStore,
+	}
+	for _, db := range sqlStore.GetAllConns() {
+		table := db.AddTableWithName(model.UploadSession{}, "UploadSessions").SetKeys(false, "Id")
+		table.ColMap("Id").SetMaxSize(26)
+		table.ColMap("Type").SetMaxSize(32)
+		table.ColMap("UserId").SetMaxSize(26)
+		table.ColMap("ChannelId").SetMaxSize(26)
+		table.ColMap("Filename").SetMaxSize(256)
+		table.ColMap("Path").SetMaxSize(512)
+	}
+	return s
+}
+
+func (us SqlUploadSessionStore) createIndexesIfNotExists() {
+	us.CreateIndexIfNotExists("idx_uploadsessions_user_id", "UploadSessions", "Type")
+	us.CreateIndexIfNotExists("idx_uploadsessions_create_at", "UploadSessions", "CreateAt")
+	us.CreateIndexIfNotExists("idx_uploadsessions_user_id", "UploadSessions", "UserId")
+}
+
+func (us SqlUploadSessionStore) Save(session *model.UploadSession) (*model.UploadSession, error) {
+	if session == nil {
+		return nil, errors.New("SqlUploadSessionStore.Save: session should not be nil")
+	}
+	session.PreSave()
+	if err := session.IsValid(); err != nil {
+		return nil, errors.Wrap(err, "SqlUploadSessionStore.Save: validation failed")
+	}
+	if err := us.GetMaster().Insert(session); err != nil {
+		return nil, errors.Wrap(err, "SqlUploadSessionStore.Save: failed to insert")
+	}
+	return session, nil
+}
+
+func (us SqlUploadSessionStore) Update(session *model.UploadSession) error {
+	if session == nil {
+		return errors.New("SqlUploadSessionStore.Update: session should not be nil")
+	}
+	if err := session.IsValid(); err != nil {
+		return errors.Wrap(err, "SqlUploadSessionStore.Update: validation failed")
+	}
+	if _, err := us.GetMaster().Update(session); err != nil {
+		if err == sql.ErrNoRows {
+			return store.NewErrNotFound("UploadSession", session.Id)
+		}
+		return errors.Wrapf(err, "SqlUploadSessionStore.Update: failed to update session with id=%s", session.Id)
+	}
+	return nil
+}
+
+func (us SqlUploadSessionStore) Get(id string) (*model.UploadSession, error) {
+	if !model.IsValidId(id) {
+		return nil, errors.New("SqlUploadSessionStore.Get: id is not valid")
+	}
+	query := us.getQueryBuilder().
+		Select("*").
+		From("UploadSessions").
+		Where(sq.Eq{"Id": id})
+	queryString, args, err := query.ToSql()
+	if err != nil {
+		return nil, errors.Wrap(err, "SqlUploadSessionStore.Get: failed to build query")
+	}
+	var session model.UploadSession
+	if err := us.GetReplica().SelectOne(&session, queryString, args...); err != nil {
+		if err == sql.ErrNoRows {
+			return nil, store.NewErrNotFound("UploadSession", id)
+		}
+		return nil, errors.Wrapf(err, "SqlUploadSessionStore.Get: failed to select session with id=%s", id)
+	}
+	return &session, nil
+}
+
+func (us SqlUploadSessionStore) GetForUser(userId string) ([]*model.UploadSession, error) {
+	if !model.IsValidId(userId) {
+		return nil, errors.New("SqlUploadSessionStore.GetForUser: userId is not valid")
+	}
+	query := us.getQueryBuilder().
+		Select("*").
+		From("UploadSessions").
+		Where(sq.Eq{"UserId": userId}).
+		OrderBy("CreateAt ASC")
+	queryString, args, err := query.ToSql()
+	if err != nil {
+		return nil, errors.Wrap(err, "SqlUploadSessionStore.GetForUser: failed to build query")
+	}
+	var sessions []*model.UploadSession
+	if _, err := us.GetReplica().Select(&sessions, queryString, args...); err != nil {
+		return nil, errors.Wrap(err, "SqlUploadSessionStore.GetForUser: failed to select")
+	}
+	return sessions, nil
+}
+
+func (us SqlUploadSessionStore) Delete(id string) error {
+	if !model.IsValidId(id) {
+		return errors.New("SqlUploadSessionStore.Delete: id is not valid")
+	}
+
+	query := us.getQueryBuilder().
+		Delete("UploadSessions").
+		Where(sq.Eq{"Id": id})
+	queryString, args, err := query.ToSql()
+	if err != nil {
+		return errors.Wrap(err, "SqlUploadSessionStore.Delete: failed to build query")
+	}
+
+	if _, err := us.GetMaster().Exec(queryString, args...); err != nil {
+		return errors.Wrap(err, "SqlUploadSessionStore.Delete: failed to delete")
+	}
+
+	return nil
+}

store/sqlstore/upload_session_store_test.go

@@ -0,0 +1,14 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sqlstore
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost-server/v5/store/storetest"
+)
+
+func TestUploadSessionStore(t *testing.T) {
+	StoreTest(t, storetest.TestUploadSessionStore)
+}