[MM-26812] Add support for resumable file uploads (#15252)
* Implement AppendFile for FileBackend * Split test into subtests * [MM-26812] Add support for resumable file uploads (#15252) * Implement UploadSession * Implement UploadSessionStore * Add error strings * Implement resumable file uploads * Add UploadType * Fix retry layer tests * Regenerate store layers * Fix store error handling * Use base for filename * Prevent concurrent uploads on the same upload session * Fix erroneus error string * Improve error handling Co-authored-by: Mattermod <mattermod@users.noreply.github.com> * Fix translations Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
Этот коммит содержится в:
Claudio Costa
коммит произвёл
GitHub
GitHub
родитель
6a58834f34
Коммит
9c272f0b20
@@ -57,6 +57,9 @@ type Routes struct {
|
||||
Files *mux.Router // 'api/v4/files'
|
||||
File *mux.Router // 'api/v4/files/{file_id:[A-Za-z0-9]+}'
|
||||
|
||||
Uploads *mux.Router // 'api/v4/uploads'
|
||||
Upload *mux.Router // 'api/v4/uploads/{upload_id:[A-Za-z0-9]+}'
|
||||
|
||||
Plugins *mux.Router // 'api/v4/plugins'
|
||||
Plugin *mux.Router // 'api/v4/plugins/{plugin_id:[A-Za-z0-9\\_\\-\\.]+}'
|
||||
|
||||
@@ -173,6 +176,9 @@ func Init(configservice configservice.ConfigService, globalOptionsFunc app.AppOp
|
||||
api.BaseRoutes.File = api.BaseRoutes.Files.PathPrefix("/{file_id:[A-Za-z0-9]+}").Subrouter()
|
||||
api.BaseRoutes.PublicFile = api.BaseRoutes.Root.PathPrefix("/files/{file_id:[A-Za-z0-9]+}/public").Subrouter()
|
||||
|
||||
api.BaseRoutes.Uploads = api.BaseRoutes.ApiRoot.PathPrefix("/uploads").Subrouter()
|
||||
api.BaseRoutes.Upload = api.BaseRoutes.Uploads.PathPrefix("/{upload_id:[A-Za-z0-9]+}").Subrouter()
|
||||
|
||||
api.BaseRoutes.Plugins = api.BaseRoutes.ApiRoot.PathPrefix("/plugins").Subrouter()
|
||||
api.BaseRoutes.Plugin = api.BaseRoutes.Plugins.PathPrefix("/{plugin_id:[A-Za-z0-9\\_\\-\\.]+}").Subrouter()
|
||||
|
||||
@@ -227,6 +233,7 @@ func Init(configservice configservice.ConfigService, globalOptionsFunc app.AppOp
|
||||
api.InitChannel()
|
||||
api.InitPost()
|
||||
api.InitFile()
|
||||
api.InitUpload()
|
||||
api.InitSystem()
|
||||
api.InitLicense()
|
||||
api.InitConfig()
|
||||
|
||||
123
api4/upload.go
Обычный файл
123
api4/upload.go
Обычный файл
@@ -0,0 +1,123 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package api4
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/mattermost/mattermost-server/v5/audit"
|
||||
"github.com/mattermost/mattermost-server/v5/model"
|
||||
)
|
||||
|
||||
func (api *API) InitUpload() {
|
||||
api.BaseRoutes.Uploads.Handle("", api.ApiSessionRequired(createUpload)).Methods("POST")
|
||||
api.BaseRoutes.Upload.Handle("", api.ApiSessionRequired(getUpload)).Methods("GET")
|
||||
api.BaseRoutes.Upload.Handle("", api.ApiSessionRequired(uploadData)).Methods("POST")
|
||||
}
|
||||
|
||||
func createUpload(c *Context, w http.ResponseWriter, r *http.Request) {
|
||||
if !*c.App.Config().FileSettings.EnableFileAttachments {
|
||||
c.Err = model.NewAppError("createUpload",
|
||||
"api.file.attachments.disabled.app_error",
|
||||
nil, "", http.StatusNotImplemented)
|
||||
return
|
||||
}
|
||||
|
||||
us := model.UploadSessionFromJson(r.Body)
|
||||
if us == nil {
|
||||
c.SetInvalidParam("upload")
|
||||
return
|
||||
}
|
||||
|
||||
auditRec := c.MakeAuditRecord("createUpload", audit.Fail)
|
||||
defer c.LogAuditRec(auditRec)
|
||||
auditRec.AddMeta("upload", us)
|
||||
|
||||
if !c.App.SessionHasPermissionToChannel(*c.App.Session(), us.ChannelId, model.PERMISSION_UPLOAD_FILE) {
|
||||
c.SetPermissionError(model.PERMISSION_UPLOAD_FILE)
|
||||
return
|
||||
}
|
||||
|
||||
us.Id = model.NewId()
|
||||
us.Type = model.UploadTypeAttachment
|
||||
us.UserId = c.App.Session().UserId
|
||||
us, err := c.App.CreateUploadSession(us)
|
||||
if err != nil {
|
||||
c.Err = err
|
||||
return
|
||||
}
|
||||
|
||||
auditRec.Success()
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
w.Write([]byte(us.ToJson()))
|
||||
}
|
||||
|
||||
func getUpload(c *Context, w http.ResponseWriter, r *http.Request) {
|
||||
c.RequireUploadId()
|
||||
if c.Err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
us, err := c.App.GetUploadSession(c.Params.UploadId)
|
||||
if err != nil {
|
||||
c.Err = err
|
||||
return
|
||||
}
|
||||
|
||||
if us.UserId != c.App.Session().UserId {
|
||||
c.Err = model.NewAppError("getUpload", "api.upload.get_upload.forbidden.app_error", nil, "", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
|
||||
w.Write([]byte(us.ToJson()))
|
||||
}
|
||||
|
||||
func uploadData(c *Context, w http.ResponseWriter, r *http.Request) {
|
||||
if !*c.App.Config().FileSettings.EnableFileAttachments {
|
||||
c.Err = model.NewAppError("uploadData", "api.file.attachments.disabled.app_error",
|
||||
nil, "", http.StatusNotImplemented)
|
||||
return
|
||||
}
|
||||
|
||||
c.RequireUploadId()
|
||||
if c.Err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
auditRec := c.MakeAuditRecord("uploadData", audit.Fail)
|
||||
defer c.LogAuditRec(auditRec)
|
||||
auditRec.AddMeta("upload_id", c.Params.UploadId)
|
||||
|
||||
us, err := c.App.GetUploadSession(c.Params.UploadId)
|
||||
if err != nil {
|
||||
c.Err = err
|
||||
return
|
||||
}
|
||||
|
||||
if r.ContentLength > (us.FileSize - us.FileOffset) {
|
||||
c.Err = model.NewAppError("uploadData", "api.upload.upload_data.invalid_content_length",
|
||||
nil, "", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if us.UserId != c.App.Session().UserId || !c.App.SessionHasPermissionToChannel(*c.App.Session(), us.ChannelId, model.PERMISSION_UPLOAD_FILE) {
|
||||
c.SetPermissionError(model.PERMISSION_UPLOAD_FILE)
|
||||
return
|
||||
}
|
||||
|
||||
info, err := c.App.UploadData(us, r.Body)
|
||||
if err != nil {
|
||||
c.Err = err
|
||||
return
|
||||
}
|
||||
|
||||
auditRec.Success()
|
||||
|
||||
if info == nil {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
|
||||
w.Write([]byte(info.ToJson()))
|
||||
}
|
||||
242
api4/upload_test.go
Обычный файл
242
api4/upload_test.go
Обычный файл
@@ -0,0 +1,242 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package api4
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/mattermost/mattermost-server/v5/model"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCreateUpload(t *testing.T) {
|
||||
th := Setup(t).InitBasic()
|
||||
defer th.TearDown()
|
||||
|
||||
us := &model.UploadSession{
|
||||
ChannelId: th.BasicChannel.Id,
|
||||
Filename: "upload",
|
||||
FileSize: 8 * 1024 * 1024,
|
||||
}
|
||||
|
||||
t.Run("file attachments disabled", func(t *testing.T) {
|
||||
th.App.UpdateConfig(func(cfg *model.Config) { *cfg.FileSettings.EnableFileAttachments = false })
|
||||
defer th.App.UpdateConfig(func(cfg *model.Config) { *cfg.FileSettings.EnableFileAttachments = true })
|
||||
u, resp := th.Client.CreateUpload(us)
|
||||
require.Nil(t, u)
|
||||
require.Error(t, resp.Error)
|
||||
require.Equal(t, "api.file.attachments.disabled.app_error", resp.Error.Id)
|
||||
require.Equal(t, http.StatusNotImplemented, resp.StatusCode)
|
||||
})
|
||||
|
||||
t.Run("no permissions", func(t *testing.T) {
|
||||
us.ChannelId = th.BasicPrivateChannel2.Id
|
||||
u, resp := th.Client.CreateUpload(us)
|
||||
require.Nil(t, u)
|
||||
require.Error(t, resp.Error)
|
||||
require.Equal(t, "api.context.permissions.app_error", resp.Error.Id)
|
||||
require.Equal(t, http.StatusForbidden, resp.StatusCode)
|
||||
})
|
||||
|
||||
t.Run("valid", func(t *testing.T) {
|
||||
us.ChannelId = th.BasicChannel.Id
|
||||
u, resp := th.Client.CreateUpload(us)
|
||||
require.Nil(t, resp.Error)
|
||||
require.NotEmpty(t, u)
|
||||
require.Equal(t, http.StatusCreated, resp.StatusCode)
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetUpload(t *testing.T) {
|
||||
th := Setup(t).InitBasic()
|
||||
defer th.TearDown()
|
||||
|
||||
us := &model.UploadSession{
|
||||
Id: model.NewId(),
|
||||
Type: model.UploadTypeAttachment,
|
||||
CreateAt: model.GetMillis(),
|
||||
UserId: th.BasicUser2.Id,
|
||||
ChannelId: th.BasicChannel.Id,
|
||||
Filename: "upload",
|
||||
FileSize: 8 * 1024 * 1024,
|
||||
}
|
||||
us, err := th.App.CreateUploadSession(us)
|
||||
require.Nil(t, err)
|
||||
require.NotNil(t, us)
|
||||
require.NotEmpty(t, us)
|
||||
|
||||
t.Run("upload not found", func(t *testing.T) {
|
||||
u, resp := th.Client.GetUpload(model.NewId())
|
||||
require.Nil(t, u)
|
||||
require.Error(t, resp.Error)
|
||||
require.Equal(t, "app.upload.get.app_error", resp.Error.Id)
|
||||
require.Equal(t, http.StatusNotFound, resp.StatusCode)
|
||||
})
|
||||
|
||||
t.Run("no permissions", func(t *testing.T) {
|
||||
u, resp := th.Client.GetUpload(us.Id)
|
||||
require.Nil(t, u)
|
||||
require.Error(t, resp.Error)
|
||||
require.Equal(t, "api.upload.get_upload.forbidden.app_error", resp.Error.Id)
|
||||
})
|
||||
|
||||
t.Run("success", func(t *testing.T) {
|
||||
expected, resp := th.Client.CreateUpload(us)
|
||||
require.Nil(t, resp.Error)
|
||||
require.NotEmpty(t, expected)
|
||||
require.Equal(t, http.StatusCreated, resp.StatusCode)
|
||||
|
||||
u, resp := th.Client.GetUpload(expected.Id)
|
||||
require.Nil(t, resp.Error)
|
||||
require.NotEmpty(t, u)
|
||||
require.Equal(t, expected, u)
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetUploadsForUser(t *testing.T) {
|
||||
th := Setup(t).InitBasic()
|
||||
defer th.TearDown()
|
||||
|
||||
t.Run("no permissions", func(t *testing.T) {
|
||||
uss, resp := th.Client.GetUploadsForUser(th.BasicUser2.Id)
|
||||
require.Error(t, resp.Error)
|
||||
require.Equal(t, "api.user.get_uploads_for_user.forbidden.app_error", resp.Error.Id)
|
||||
require.Nil(t, uss)
|
||||
})
|
||||
|
||||
t.Run("empty", func(t *testing.T) {
|
||||
uss, resp := th.Client.GetUploadsForUser(th.BasicUser.Id)
|
||||
require.Nil(t, resp.Error)
|
||||
require.Empty(t, uss)
|
||||
})
|
||||
|
||||
t.Run("success", func(t *testing.T) {
|
||||
uploads := make([]*model.UploadSession, 4)
|
||||
for i := 0; i < len(uploads); i++ {
|
||||
us := &model.UploadSession{
|
||||
Id: model.NewId(),
|
||||
Type: model.UploadTypeAttachment,
|
||||
CreateAt: model.GetMillis(),
|
||||
UserId: th.BasicUser.Id,
|
||||
ChannelId: th.BasicChannel.Id,
|
||||
Filename: "upload",
|
||||
FileSize: 8 * 1024 * 1024,
|
||||
}
|
||||
us, err := th.App.CreateUploadSession(us)
|
||||
require.Nil(t, err)
|
||||
require.NotNil(t, us)
|
||||
require.NotEmpty(t, us)
|
||||
us.Path = ""
|
||||
uploads[i] = us
|
||||
}
|
||||
|
||||
uss, resp := th.Client.GetUploadsForUser(th.BasicUser.Id)
|
||||
require.Nil(t, resp.Error)
|
||||
require.NotEmpty(t, uss)
|
||||
require.Len(t, uss, len(uploads))
|
||||
for i := range uploads {
|
||||
require.Contains(t, uss, uploads[i])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestUploadData(t *testing.T) {
|
||||
th := Setup(t).InitBasic()
|
||||
defer th.TearDown()
|
||||
if *th.App.Config().FileSettings.DriverName == "" {
|
||||
t.Skip("skipping because no file driver is enabled")
|
||||
}
|
||||
|
||||
us := &model.UploadSession{
|
||||
Id: model.NewId(),
|
||||
Type: model.UploadTypeAttachment,
|
||||
CreateAt: model.GetMillis(),
|
||||
UserId: th.BasicUser2.Id,
|
||||
ChannelId: th.BasicChannel.Id,
|
||||
Filename: "upload",
|
||||
FileSize: 8 * 1024 * 1024,
|
||||
}
|
||||
us, err := th.App.CreateUploadSession(us)
|
||||
require.Nil(t, err)
|
||||
require.NotNil(t, us)
|
||||
require.NotEmpty(t, us)
|
||||
|
||||
data := randomBytes(t, int(us.FileSize))
|
||||
|
||||
t.Run("file attachments disabled", func(t *testing.T) {
|
||||
th.App.UpdateConfig(func(cfg *model.Config) { *cfg.FileSettings.EnableFileAttachments = false })
|
||||
defer th.App.UpdateConfig(func(cfg *model.Config) { *cfg.FileSettings.EnableFileAttachments = true })
|
||||
info, resp := th.Client.UploadData(model.NewId(), bytes.NewReader(data))
|
||||
require.Nil(t, info)
|
||||
require.Error(t, resp.Error)
|
||||
require.Equal(t, "api.file.attachments.disabled.app_error", resp.Error.Id)
|
||||
})
|
||||
|
||||
t.Run("upload not found", func(t *testing.T) {
|
||||
info, resp := th.Client.UploadData(model.NewId(), bytes.NewReader(data))
|
||||
require.Nil(t, info)
|
||||
require.Error(t, resp.Error)
|
||||
require.Equal(t, "app.upload.get.app_error", resp.Error.Id)
|
||||
require.Equal(t, http.StatusNotFound, resp.StatusCode)
|
||||
})
|
||||
|
||||
t.Run("no permissions", func(t *testing.T) {
|
||||
info, resp := th.Client.UploadData(us.Id, bytes.NewReader(data))
|
||||
require.Nil(t, info)
|
||||
require.Error(t, resp.Error)
|
||||
require.Equal(t, "api.context.permissions.app_error", resp.Error.Id)
|
||||
})
|
||||
|
||||
t.Run("bad content-length", func(t *testing.T) {
|
||||
info, resp := th.Client.UploadData(us.Id, bytes.NewReader(append(data, 0x00)))
|
||||
require.Nil(t, info)
|
||||
require.Error(t, resp.Error)
|
||||
require.Equal(t, "api.upload.upload_data.invalid_content_length", resp.Error.Id)
|
||||
})
|
||||
|
||||
t.Run("success", func(t *testing.T) {
|
||||
u, resp := th.Client.CreateUpload(us)
|
||||
require.Nil(t, resp.Error)
|
||||
require.NotEmpty(t, u)
|
||||
require.Equal(t, http.StatusCreated, resp.StatusCode)
|
||||
|
||||
info, resp := th.Client.UploadData(u.Id, bytes.NewReader(data))
|
||||
require.Nil(t, resp.Error)
|
||||
require.NotEmpty(t, info)
|
||||
require.Equal(t, u.Filename, info.Name)
|
||||
|
||||
file, resp := th.Client.GetFile(info.Id)
|
||||
require.Nil(t, resp.Error)
|
||||
require.Equal(t, file, data)
|
||||
})
|
||||
|
||||
t.Run("resume success", func(t *testing.T) {
|
||||
u, resp := th.Client.CreateUpload(us)
|
||||
require.Nil(t, resp.Error)
|
||||
require.NotEmpty(t, u)
|
||||
require.Equal(t, http.StatusCreated, resp.StatusCode)
|
||||
|
||||
rd := &io.LimitedReader{
|
||||
R: bytes.NewReader(data),
|
||||
N: 5 * 1024 * 1024,
|
||||
}
|
||||
info, resp := th.Client.UploadData(u.Id, rd)
|
||||
require.Nil(t, resp.Error)
|
||||
require.Nil(t, info)
|
||||
require.Equal(t, http.StatusNoContent, resp.StatusCode)
|
||||
|
||||
info, resp = th.Client.UploadData(u.Id, bytes.NewReader(data[5*1024*1024:]))
|
||||
require.Nil(t, resp.Error)
|
||||
require.NotEmpty(t, info)
|
||||
require.Equal(t, u.Filename, info.Name)
|
||||
|
||||
file, resp := th.Client.GetFile(info.Id)
|
||||
require.Nil(t, resp.Error)
|
||||
require.Equal(t, file, data)
|
||||
})
|
||||
}
|
||||
22
api4/user.go
22
api4/user.go
@@ -89,6 +89,8 @@ func (api *API) InitUser() {
|
||||
|
||||
api.BaseRoutes.Users.Handle("/migrate_auth/ldap", api.ApiSessionRequired(migrateAuthToLDAP)).Methods("POST")
|
||||
api.BaseRoutes.Users.Handle("/migrate_auth/saml", api.ApiSessionRequired(migrateAuthToSaml)).Methods("POST")
|
||||
|
||||
api.BaseRoutes.User.Handle("/uploads", api.ApiSessionRequired(getUploadsForUser)).Methods("GET")
|
||||
}
|
||||
|
||||
func createUser(c *Context, w http.ResponseWriter, r *http.Request) {
|
||||
@@ -2601,6 +2603,26 @@ func convertUserToBot(c *Context, w http.ResponseWriter, r *http.Request) {
|
||||
w.Write(bot.ToJson())
|
||||
}
|
||||
|
||||
func getUploadsForUser(c *Context, w http.ResponseWriter, r *http.Request) {
|
||||
c.RequireUserId()
|
||||
if c.Err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if c.Params.UserId != c.App.Session().UserId {
|
||||
c.Err = model.NewAppError("getUploadsForUser", "api.user.get_uploads_for_user.forbidden.app_error", nil, "", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
|
||||
uss, err := c.App.GetUploadSessionsForUser(c.Params.UserId)
|
||||
if err != nil {
|
||||
c.Err = err
|
||||
return
|
||||
}
|
||||
|
||||
w.Write([]byte(model.UploadSessionsToJson(uss)))
|
||||
}
|
||||
|
||||
func migrateAuthToLDAP(c *Context, w http.ResponseWriter, r *http.Request) {
|
||||
props := model.StringInterfaceFromJson(r.Body)
|
||||
from, ok := props["from"].(string)
|
||||
|
||||
Ссылка в новой задаче
Block a user