[MM-28422] Enable processing of import files through API (#16062)

* Implement unzip function

* Implement FileSize method

* Implement path rewriting for bulk import

* Small improvements

* Add ImportSettings to config

* Implement ListImports API endpoint

* Enable uploading import files

* Implement import process job

* Add missing license headers

* Address reviews

* Make path sanitization a bit smarter

* Clean path before calculating Dir

* [MM-30008] Add mmctl support for file imports (#16301)

* Add mmctl support for import files

* Improve test

* Remove unnecessary handlers

* Use th.TestForSystemAdminAndLocal

* Make nouser id a constant

app/upload.go

@@ -19,6 +19,7 @@ import (
 )
 
 const minFirstPartSize = 5 * 1024 * 1024 // 5MB
+const incompleteUploadSuffix = ".tmp"
 
 func (a *App) runPluginsHook(info *model.FileInfo, file io.Reader) *model.AppError {
 	pluginsEnvironment := a.GetPluginsEnvironment()
@@ -108,19 +109,25 @@ func (a *App) CreateUploadSession(us *model.UploadSession) (*model.UploadSession
 	us.FileOffset = 0
 	now := time.Now()
 	us.CreateAt = model.GetMillisForTime(now)
-	us.Path = now.Format("20060102") + "/teams/noteam/channels/" + us.ChannelId + "/users/" + us.UserId + "/" + us.Id + "/" + filepath.Base(us.Filename)
+	if us.Type == model.UploadTypeAttachment {
+		us.Path = now.Format("20060102") + "/teams/noteam/channels/" + us.ChannelId + "/users/" + us.UserId + "/" + us.Id + "/" + filepath.Base(us.Filename)
+	} else if us.Type == model.UploadTypeImport {
+		us.Path = *a.Config().ImportSettings.Directory + "/" + us.Id + "_" + filepath.Base(us.Filename)
+	}
 	if err := us.IsValid(); err != nil {
 		return nil, err
 	}
 
-	channel, err := a.GetChannel(us.ChannelId)
-	if err != nil {
-		return nil, model.NewAppError("CreateUploadSession", "app.upload.create.incorrect_channel_id.app_error",
-			map[string]interface{}{"channelId": us.ChannelId}, "", http.StatusBadRequest)
-	}
-	if channel.DeleteAt != 0 {
-		return nil, model.NewAppError("CreateUploadSession", "app.upload.create.cannot_upload_to_deleted_channel.app_error",
-			map[string]interface{}{"channelId": us.ChannelId}, "", http.StatusBadRequest)
+	if us.Type == model.UploadTypeAttachment {
+		channel, err := a.GetChannel(us.ChannelId)
+		if err != nil {
+			return nil, model.NewAppError("CreateUploadSession", "app.upload.create.incorrect_channel_id.app_error",
+				map[string]interface{}{"channelId": us.ChannelId}, "", http.StatusBadRequest)
+		}
+		if channel.DeleteAt != 0 {
+			return nil, model.NewAppError("CreateUploadSession", "app.upload.create.cannot_upload_to_deleted_channel.app_error",
+				map[string]interface{}{"channelId": us.ChannelId}, "", http.StatusBadRequest)
+		}
 	}
 
 	us, storeErr := a.Srv().Store.UploadSession().Save(us)
@@ -186,6 +193,11 @@ func (a *App) UploadData(us *model.UploadSession, rd io.Reader) (*model.FileInfo
 			nil, "FileOffset mismatch", http.StatusBadRequest)
 	}
 
+	uploadPath := us.Path
+	if us.Type == model.UploadTypeImport {
+		uploadPath += incompleteUploadSuffix
+	}
+
 	// make sure it's not possible to upload more data than what is expected.
 	lr := &io.LimitedReader{
 		R: rd,
@@ -195,12 +207,12 @@ func (a *App) UploadData(us *model.UploadSession, rd io.Reader) (*model.FileInfo
 	var written int64
 	if us.FileOffset == 0 {
 		// new upload
-		written, err = a.WriteFile(lr, us.Path)
+		written, err = a.WriteFile(lr, uploadPath)
 		if err != nil && written == 0 {
 			return nil, err
 		}
 		if written < minFirstPartSize && written != us.FileSize {
-			a.RemoveFile(us.Path)
+			a.RemoveFile(uploadPath)
 			var errStr string
 			if err != nil {
 				errStr = err.Error()
@@ -210,7 +222,7 @@ func (a *App) UploadData(us *model.UploadSession, rd io.Reader) (*model.FileInfo
 		}
 	} else if us.FileOffset < us.FileSize {
 		// resume upload
-		written, err = a.AppendFile(lr, us.Path)
+		written, err = a.AppendFile(lr, uploadPath)
 	}
 	if written > 0 {
 		us.FileOffset += written
@@ -228,7 +240,7 @@ func (a *App) UploadData(us *model.UploadSession, rd io.Reader) (*model.FileInfo
 	}
 
 	// upload is done, create FileInfo
-	file, err := a.FileReader(us.Path)
+	file, err := a.FileReader(uploadPath)
 	if err != nil {
 		return nil, model.NewAppError("UploadData", "app.upload.upload_data.read_file.app_error", nil, err.Error(), http.StatusInternalServerError)
 	}
@@ -260,13 +272,19 @@ func (a *App) UploadData(us *model.UploadSession, rd io.Reader) (*model.FileInfo
 		nameWithoutExtension := info.Name[:strings.LastIndex(info.Name, ".")]
 		info.PreviewPath = filepath.Dir(info.Path) + "/" + nameWithoutExtension + "_preview.jpg"
 		info.ThumbnailPath = filepath.Dir(info.Path) + "/" + nameWithoutExtension + "_thumb.jpg"
-		imgData, fileErr := a.ReadFile(us.Path)
+		imgData, fileErr := a.ReadFile(uploadPath)
 		if fileErr != nil {
 			return nil, fileErr
 		}
 		a.HandleImages([]string{info.PreviewPath}, []string{info.ThumbnailPath}, [][]byte{imgData})
 	}
 
+	if us.Type == model.UploadTypeImport {
+		if err := a.MoveFile(uploadPath, us.Path); err != nil {
+			return nil, model.NewAppError("UploadData", "app.upload.upload_data.move_file.app_error", nil, err.Error(), http.StatusInternalServerError)
+		}
+	}
+
 	var storeErr error
 	if info, storeErr = a.Srv().Store.FileInfo().Save(info); storeErr != nil {
 		var appErr *model.AppError