[MM-28422] Enable processing of import files through API (#16062)

* Implement unzip function

* Implement FileSize method

* Implement path rewriting for bulk import

* Small improvements

* Add ImportSettings to config

* Implement ListImports API endpoint

* Enable uploading import files

* Implement import process job

* Add missing license headers

* Address reviews

* Make path sanitization a bit smarter

* Clean path before calculating Dir

* [MM-30008] Add mmctl support for file imports (#16301)

* Add mmctl support for import files

* Improve test

* Remove unnecessary handlers

* Use th.TestForSystemAdminAndLocal

* Make nouser id a constant

api4/api.go

@@ -123,6 +123,8 @@ type Routes struct {
 	Groups         *mux.Router // 'api/v4/groups'
 
 	Cloud *mux.Router // 'api/v4/cloud'
+
+	Imports *mux.Router // 'api/v4/imports'
 }
 
 type API struct {
@@ -235,6 +237,8 @@ func Init(configservice configservice.ConfigService, globalOptionsFunc app.AppOp
 
 	api.BaseRoutes.Cloud = api.BaseRoutes.ApiRoot.PathPrefix("/cloud").Subrouter()
 
+	api.BaseRoutes.Imports = api.BaseRoutes.ApiRoot.PathPrefix("/imports").Subrouter()
+
 	api.InitUser()
 	api.InitBot()
 	api.InitTeam()
@@ -271,6 +275,7 @@ func Init(configservice configservice.ConfigService, globalOptionsFunc app.AppOp
 	api.InitGroup()
 	api.InitAction()
 	api.InitCloud()
+	api.InitImport()
 
 	root.Handle("/api/v4/{anything:.*}", http.HandlerFunc(api.Handle404))
 
@@ -335,6 +340,13 @@ func InitLocal(configservice configservice.ConfigService, globalOptionsFunc app.
 
 	api.BaseRoutes.Roles = api.BaseRoutes.ApiRoot.PathPrefix("/roles").Subrouter()
 
+	api.BaseRoutes.Uploads = api.BaseRoutes.ApiRoot.PathPrefix("/uploads").Subrouter()
+	api.BaseRoutes.Upload = api.BaseRoutes.Uploads.PathPrefix("/{upload_id:[A-Za-z0-9]+}").Subrouter()
+
+	api.BaseRoutes.Imports = api.BaseRoutes.ApiRoot.PathPrefix("/imports").Subrouter()
+
+	api.BaseRoutes.Jobs = api.BaseRoutes.ApiRoot.PathPrefix("/jobs").Subrouter()
+
 	api.InitUserLocal()
 	api.InitTeamLocal()
 	api.InitChannelLocal()
@@ -349,6 +361,9 @@ func InitLocal(configservice configservice.ConfigService, globalOptionsFunc app.
 	api.InitSystemLocal()
 	api.InitPostLocal()
 	api.InitRoleLocal()
+	api.InitUploadLocal()
+	api.InitImportLocal()
+	api.InitJobLocal()
 
 	root.Handle("/api/v4/{anything:.*}", http.HandlerFunc(api.Handle404))
 

api4/import.go

@@ -0,0 +1,36 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package api4
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+)
+
+func (api *API) InitImport() {
+	api.BaseRoutes.Imports.Handle("", api.ApiSessionRequired(listImports)).Methods("GET")
+}
+
+func listImports(c *Context, w http.ResponseWriter, r *http.Request) {
+	if !c.IsSystemAdmin() {
+		c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM)
+		return
+	}
+
+	imports, appErr := c.App.ListImports()
+	if appErr != nil {
+		c.Err = appErr
+		return
+	}
+
+	data, err := json.Marshal(imports)
+	if err != nil {
+		c.Err = model.NewAppError("listImports", "app.import.marshal.app_error", nil, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	w.Write(data)
+}

api4/import_local.go

@@ -0,0 +1,8 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package api4
+
+func (api *API) InitImportLocal() {
+	api.BaseRoutes.Imports.Handle("", api.ApiLocal(listImports)).Methods("GET")
+}

api4/import_test.go

@@ -0,0 +1,106 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package api4
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/utils/fileutils"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestListImports(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	testsDir, _ := fileutils.FindDir("tests")
+	require.NotEmpty(t, testsDir)
+
+	uploadNewImport := func(c *model.Client4, t *testing.T) string {
+		file, err := os.Open(testsDir + "/import_test.zip")
+		require.Nil(t, err)
+
+		info, err := file.Stat()
+		require.Nil(t, err)
+
+		us := &model.UploadSession{
+			Filename: info.Name(),
+			FileSize: info.Size(),
+			Type:     model.UploadTypeImport,
+		}
+
+		if c == th.LocalClient {
+			us.UserId = model.UploadNoUserID
+		}
+
+		u, resp := c.CreateUpload(us)
+		require.Nil(t, resp.Error)
+		require.NotNil(t, u)
+
+		finfo, resp := c.UploadData(u.Id, file)
+		require.Nil(t, resp.Error)
+		require.NotNil(t, finfo)
+
+		return u.Id
+	}
+
+	t.Run("no permissions", func(t *testing.T) {
+		imports, resp := th.Client.ListImports()
+		require.Error(t, resp.Error)
+		require.Equal(t, "api.context.permissions.app_error", resp.Error.Id)
+		require.Nil(t, imports)
+	})
+
+	dataDir, found := fileutils.FindDir("data")
+	require.True(t, found)
+
+	th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) {
+		imports, resp := c.ListImports()
+		require.Nil(t, resp.Error)
+		require.Empty(t, imports)
+	}, "no imports")
+
+	th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) {
+		id := uploadNewImport(c, t)
+		id2 := uploadNewImport(c, t)
+
+		importDir := filepath.Join(dataDir, "import")
+		f, err := os.Create(filepath.Join(importDir, "import.zip.tmp"))
+		require.Nil(t, err)
+		f.Close()
+
+		imports, resp := c.ListImports()
+		require.Nil(t, resp.Error)
+		require.NotEmpty(t, imports)
+		require.Len(t, imports, 2)
+		require.Contains(t, imports, id+"_import_test.zip")
+		require.Contains(t, imports, id2+"_import_test.zip")
+
+		require.Nil(t, os.RemoveAll(importDir))
+	}, "expected imports")
+
+	th.TestForSystemAdminAndLocal(t, func(t *testing.T, c *model.Client4) {
+		th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ImportSettings.Directory = "import_new" })
+		defer th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ImportSettings.Directory = "import" })
+
+		importDir := filepath.Join(dataDir, "import_new")
+
+		imports, resp := c.ListImports()
+		require.Nil(t, resp.Error)
+		require.Empty(t, imports)
+
+		id := uploadNewImport(c, t)
+		imports, resp = c.ListImports()
+		require.Nil(t, resp.Error)
+		require.NotEmpty(t, imports)
+		require.Len(t, imports, 1)
+		require.Equal(t, id+"_import_test.zip", imports[0])
+
+		require.Nil(t, os.RemoveAll(importDir))
+	}, "change import directory")
+}

api4/job_local.go

@@ -0,0 +1,12 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package api4
+
+func (api *API) InitJobLocal() {
+	api.BaseRoutes.Jobs.Handle("", api.ApiLocal(getJobs)).Methods("GET")
+	api.BaseRoutes.Jobs.Handle("", api.ApiLocal(createJob)).Methods("POST")
+	api.BaseRoutes.Jobs.Handle("/{job_id:[A-Za-z0-9]+}", api.ApiLocal(getJob)).Methods("GET")
+	api.BaseRoutes.Jobs.Handle("/{job_id:[A-Za-z0-9]+}/cancel", api.ApiLocal(cancelJob)).Methods("POST")
+	api.BaseRoutes.Jobs.Handle("/type/{job_type:[A-Za-z0-9_-]+}", api.ApiLocal(getJobsByType)).Methods("GET")
+}

api4/upload.go

@@ -37,14 +37,23 @@ func createUpload(c *Context, w http.ResponseWriter, r *http.Request) {
 	defer c.LogAuditRec(auditRec)
 	auditRec.AddMeta("upload", us)
 
-	if !c.App.SessionHasPermissionToChannel(*c.App.Session(), us.ChannelId, model.PERMISSION_UPLOAD_FILE) {
-		c.SetPermissionError(model.PERMISSION_UPLOAD_FILE)
-		return
+	if us.Type == model.UploadTypeImport {
+		if !c.IsSystemAdmin() {
+			c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM)
+			return
+		}
+	} else {
+		if !c.App.SessionHasPermissionToChannel(*c.App.Session(), us.ChannelId, model.PERMISSION_UPLOAD_FILE) {
+			c.SetPermissionError(model.PERMISSION_UPLOAD_FILE)
+			return
+		}
+		us.Type = model.UploadTypeAttachment
 	}
 
 	us.Id = model.NewId()
-	us.Type = model.UploadTypeAttachment
-	us.UserId = c.App.Session().UserId
+	if c.App.Session().UserId != "" {
+		us.UserId = c.App.Session().UserId
+	}
 	us, err := c.App.CreateUploadSession(us)
 	if err != nil {
 		c.Err = err
@@ -68,7 +77,7 @@ func getUpload(c *Context, w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if us.UserId != c.App.Session().UserId {
+	if us.UserId != c.App.Session().UserId && !c.IsSystemAdmin() {
 		c.Err = model.NewAppError("getUpload", "api.upload.get_upload.forbidden.app_error", nil, "", http.StatusForbidden)
 		return
 	}
@@ -98,9 +107,16 @@ func uploadData(c *Context, w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if us.UserId != c.App.Session().UserId || !c.App.SessionHasPermissionToChannel(*c.App.Session(), us.ChannelId, model.PERMISSION_UPLOAD_FILE) {
-		c.SetPermissionError(model.PERMISSION_UPLOAD_FILE)
-		return
+	if us.Type == model.UploadTypeImport {
+		if !c.IsSystemAdmin() {
+			c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM)
+			return
+		}
+	} else {
+		if us.UserId != c.App.Session().UserId || !c.App.SessionHasPermissionToChannel(*c.App.Session(), us.ChannelId, model.PERMISSION_UPLOAD_FILE) {
+			c.SetPermissionError(model.PERMISSION_UPLOAD_FILE)
+			return
+		}
 	}
 
 	boundary, parseErr := parseMultipartRequestHeader(r)

api4/upload_local.go

@@ -0,0 +1,10 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package api4
+
+func (api *API) InitUploadLocal() {
+	api.BaseRoutes.Uploads.Handle("", api.ApiLocal(createUpload)).Methods("POST")
+	api.BaseRoutes.Upload.Handle("", api.ApiLocal(getUpload)).Methods("GET")
+	api.BaseRoutes.Upload.Handle("", api.ApiLocal(uploadData)).Methods("POST")
+}

api4/upload_test.go

@@ -8,9 +8,11 @@ import (
 	"io"
 	"mime/multipart"
 	"net/http"
+	"os"
 	"testing"
 
 	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/utils/fileutils"
 
 	"github.com/stretchr/testify/require"
 )
@@ -51,6 +53,41 @@ func TestCreateUpload(t *testing.T) {
 		require.NotEmpty(t, u)
 		require.Equal(t, http.StatusCreated, resp.StatusCode)
 	})
+
+	t.Run("import file", func(t *testing.T) {
+		testsDir, _ := fileutils.FindDir("tests")
+
+		importFile, err := os.Open(testsDir + "/import_test.zip")
+		require.Nil(t, err)
+		defer importFile.Close()
+
+		info, err := importFile.Stat()
+		require.Nil(t, err)
+
+		t.Run("permissions error", func(t *testing.T) {
+			us := &model.UploadSession{
+				Filename: info.Name(),
+				FileSize: info.Size(),
+				Type:     model.UploadTypeImport,
+			}
+			u, resp := th.Client.CreateUpload(us)
+			require.Nil(t, u)
+			require.Error(t, resp.Error)
+			require.Equal(t, "api.context.permissions.app_error", resp.Error.Id)
+			require.Equal(t, http.StatusForbidden, resp.StatusCode)
+		})
+
+		t.Run("success", func(t *testing.T) {
+			us := &model.UploadSession{
+				Filename: info.Name(),
+				FileSize: info.Size(),
+				Type:     model.UploadTypeImport,
+			}
+			u, resp := th.SystemAdminClient.CreateUpload(us)
+			require.Nil(t, resp.Error)
+			require.NotEmpty(t, u)
+		})
+	})
 }
 
 func TestGetUpload(t *testing.T) {

api4/user_local.go

@@ -38,6 +38,8 @@ func (api *API) InitUserLocal() {
 
 	api.BaseRoutes.Users.Handle("/migrate_auth/ldap", api.ApiLocal(migrateAuthToLDAP)).Methods("POST")
 	api.BaseRoutes.Users.Handle("/migrate_auth/saml", api.ApiLocal(migrateAuthToSaml)).Methods("POST")
+
+	api.BaseRoutes.User.Handle("/uploads", api.ApiLocal(localGetUploadsForUser)).Methods("GET")
 }
 
 func localGetUsers(c *Context, w http.ResponseWriter, r *http.Request) {
@@ -315,3 +317,13 @@ func localGetUserByEmail(c *Context, w http.ResponseWriter, r *http.Request) {
 	w.Header().Set(model.HEADER_ETAG_SERVER, etag)
 	w.Write([]byte(user.ToJson()))
 }
+
+func localGetUploadsForUser(c *Context, w http.ResponseWriter, r *http.Request) {
+	uss, err := c.App.GetUploadSessionsForUser(c.Params.UserId)
+	if err != nil {
+		c.Err = err
+		return
+	}
+
+	w.Write([]byte(model.UploadSessionsToJson(uss)))
+}