[MM-28422] Enable processing of import files through API (#16062)

* Implement unzip function

* Implement FileSize method

* Implement path rewriting for bulk import

* Small improvements

* Add ImportSettings to config

* Implement ListImports API endpoint

* Enable uploading import files

* Implement import process job

* Add missing license headers

* Address reviews

* Make path sanitization a bit smarter

* Clean path before calculating Dir

* [MM-30008] Add mmctl support for file imports (#16301)

* Add mmctl support for import files

* Improve test

* Remove unnecessary handlers

* Use th.TestForSystemAdminAndLocal

* Make nouser id a constant

app/import.go

@@ -5,9 +5,11 @@ package app
 
 import (
 	"bufio"
+	"bytes"
 	"encoding/json"
 	"io"
 	"net/http"
+	"path/filepath"
 	"strings"
 	"sync"
 
@@ -29,6 +31,46 @@ func stopOnError(err LineImportWorkerError) bool {
 	return true
 }
 
+func rewriteAttachmentPaths(files *[]AttachmentImportData, basePath string) {
+	if files == nil {
+		return
+	}
+	for _, f := range *files {
+		if f.Path != nil {
+			*f.Path = filepath.Join(basePath, *f.Path)
+		}
+	}
+}
+
+func rewriteFilePaths(line *LineImportData, basePath string) {
+	switch line.Type {
+	case "post", "direct_post":
+		var replies []ReplyImportData
+		if line.Type == "direct_post" {
+			rewriteAttachmentPaths(line.DirectPost.Attachments, basePath)
+			if line.DirectPost.Replies != nil {
+				replies = *line.DirectPost.Replies
+			}
+		} else {
+			rewriteAttachmentPaths(line.Post.Attachments, basePath)
+			if line.Post.Replies != nil {
+				replies = *line.Post.Replies
+			}
+		}
+		for _, reply := range replies {
+			rewriteAttachmentPaths(reply.Attachments, basePath)
+		}
+	case "user":
+		if line.User.ProfileImage != nil {
+			*line.User.ProfileImage = filepath.Join(basePath, *line.User.ProfileImage)
+		}
+	case "emoji":
+		if line.Emoji.Image != nil {
+			*line.Emoji.Image = filepath.Join(basePath, *line.Emoji.Image)
+		}
+	}
+}
+
 func (a *App) bulkImportWorker(dryRun bool, wg *sync.WaitGroup, lines <-chan LineImportWorkerData, errors chan<- LineImportWorkerError) {
 	postLines := []LineImportWorkerData{}
 	directPostLines := []LineImportWorkerData{}
@@ -77,6 +119,14 @@ func (a *App) bulkImportWorker(dryRun bool, wg *sync.WaitGroup, lines <-chan Lin
 }
 
 func (a *App) BulkImport(fileReader io.Reader, dryRun bool, workers int) (*model.AppError, int) {
+	return a.bulkImport(fileReader, dryRun, workers, "")
+}
+
+func (a *App) BulkImportWithPath(fileReader io.Reader, dryRun bool, workers int, importPath string) (*model.AppError, int) {
+	return a.bulkImport(fileReader, dryRun, workers, importPath)
+}
+
+func (a *App) bulkImport(fileReader io.Reader, dryRun bool, workers int, importPath string) (*model.AppError, int) {
 	scanner := bufio.NewScanner(fileReader)
 	buf := make([]byte, 0, 64*1024)
 	scanner.Buffer(buf, maxScanTokenSize)
@@ -92,7 +142,7 @@ func (a *App) BulkImport(fileReader io.Reader, dryRun bool, workers int) (*model
 	lastLineType := ""
 
 	for scanner.Scan() {
-		decoder := json.NewDecoder(strings.NewReader(scanner.Text()))
+		decoder := json.NewDecoder(bytes.NewReader(scanner.Bytes()))
 		lineNumber++
 
 		var line LineImportData
@@ -100,6 +150,10 @@ func (a *App) BulkImport(fileReader io.Reader, dryRun bool, workers int) (*model
 			return model.NewAppError("BulkImport", "app.import.bulk_import.json_decode.error", nil, err.Error(), http.StatusBadRequest), lineNumber
 		}
 
+		if importPath != "" {
+			rewriteFilePaths(&line, importPath)
+		}
+
 		if lineNumber == 1 {
 			importDataFileVersion, appErr := processImportDataFileVersionLine(line)
 			if appErr != nil {
@@ -214,3 +268,20 @@ func (a *App) importLine(line LineImportData, dryRun bool) *model.AppError {
 		return model.NewAppError("BulkImport", "app.import.import_line.unknown_line_type.error", map[string]interface{}{"Type": line.Type}, "", http.StatusBadRequest)
 	}
 }
+
+func (a *App) ListImports() ([]string, *model.AppError) {
+	imports, appErr := a.ListDirectory(*a.Config().ImportSettings.Directory)
+	if appErr != nil {
+		return nil, appErr
+	}
+
+	results := make([]string, 0, len(imports))
+	for i := 0; i < len(imports); i++ {
+		filename := filepath.Base(imports[i])
+		if !strings.HasSuffix(filename, incompleteUploadSuffix) {
+			results = append(results, filename)
+		}
+	}
+
+	return results, nil
+}