[MM-28422] Enable processing of import files through API (#16062)

* Implement unzip function

* Implement FileSize method

* Implement path rewriting for bulk import

* Small improvements

* Add ImportSettings to config

* Implement ListImports API endpoint

* Enable uploading import files

* Implement import process job

* Add missing license headers

* Address reviews

* Make path sanitization a bit smarter

* Clean path before calculating Dir

* [MM-30008] Add mmctl support for file imports (#16301)

* Add mmctl support for import files

* Improve test

* Remove unnecessary handlers

* Use th.TestForSystemAdminAndLocal

* Make nouser id a constant

services/filesstore/filesstore.go

@@ -21,6 +21,7 @@ type FileBackend interface {
 	Reader(path string) (ReadCloseSeeker, *model.AppError)
 	ReadFile(path string) ([]byte, *model.AppError)
 	FileExists(path string) (bool, *model.AppError)
+	FileSize(path string) (int64, *model.AppError)
 	CopyFile(oldPath, newPath string) *model.AppError
 	MoveFile(oldPath, newPath string) *model.AppError
 	WriteFile(fr io.Reader, path string) (int64, *model.AppError)

services/filesstore/filesstore_test.go

@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"os"
 	"testing"
 
@@ -365,6 +366,28 @@ func (s *FileBackendTestSuite) TestAppendFile() {
 	})
 }
 
+func (s *FileBackendTestSuite) TestFileSize() {
+	s.Run("nonexistent file", func() {
+		size, err := s.backend.FileSize("tests/nonexistentfile")
+		s.NotNil(err)
+		s.Zero(size)
+	})
+
+	s.Run("valid file", func() {
+		data := make([]byte, rand.Intn(1024*1024)+1)
+		path := "tests/" + model.NewId()
+
+		written, err := s.backend.WriteFile(bytes.NewReader(data), path)
+		s.Nil(err)
+		s.EqualValues(len(data), written)
+		defer s.backend.RemoveFile(path)
+
+		size, err := s.backend.FileSize(path)
+		s.Nil(err)
+		s.Equal(int64(len(data)), size)
+	})
+}
+
 func BenchmarkS3WriteFile(b *testing.B) {
 	utils.TranslationsPreInit()
 

services/filesstore/localstore.go

@@ -63,6 +63,14 @@ func (b *LocalFileBackend) FileExists(path string) (bool, *model.AppError) {
 	return true, nil
 }
 
+func (b *LocalFileBackend) FileSize(path string) (int64, *model.AppError) {
+	info, err := os.Stat(filepath.Join(b.directory, path))
+	if err != nil {
+		return 0, model.NewAppError("FileSize", "api.file.file_size.local.app_error", nil, err.Error(), http.StatusInternalServerError)
+	}
+	return info.Size(), nil
+}
+
 func (b *LocalFileBackend) CopyFile(oldPath, newPath string) *model.AppError {
 	if err := utils.CopyFile(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil {
 		return model.NewAppError("copyFile", "api.file.move_file.rename.app_error", nil, err.Error(), http.StatusInternalServerError)

services/filesstore/s3store.go

@@ -195,6 +195,17 @@ func (b *S3FileBackend) FileExists(path string) (bool, *model.AppError) {
 	return false, model.NewAppError("FileExists", "api.file.file_exists.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
 }
 
+func (b *S3FileBackend) FileSize(path string) (int64, *model.AppError) {
+	path = filepath.Join(b.pathPrefix, path)
+
+	info, err := b.client.StatObject(context.Background(), b.bucket, path, s3.StatObjectOptions{})
+	if err != nil {
+		return 0, model.NewAppError("FileSize", "api.file.file_size.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
+	}
+
+	return info.Size, nil
+}
+
 func (b *S3FileBackend) CopyFile(oldPath, newPath string) *model.AppError {
 	oldPath = filepath.Join(b.pathPrefix, oldPath)
 	newPath = filepath.Join(b.pathPrefix, newPath)