[MM-28422] Enable processing of import files through API (#16062)

* Implement unzip function

* Implement FileSize method

* Implement path rewriting for bulk import

* Small improvements

* Add ImportSettings to config

* Implement ListImports API endpoint

* Enable uploading import files

* Implement import process job

* Add missing license headers

* Address reviews

* Make path sanitization a bit smarter

* Clean path before calculating Dir

* [MM-30008] Add mmctl support for file imports (#16301)

* Add mmctl support for import files

* Improve test

* Remove unnecessary handlers

* Use th.TestForSystemAdminAndLocal

* Make nouser id a constant

utils/archive.go

@@ -0,0 +1,65 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package utils
+
+import (
+	"archive/zip"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func sanitizePath(p string) string {
+	dir := strings.ReplaceAll(filepath.Dir(filepath.Clean(p)), "..", "")
+	base := filepath.Base(p)
+	if strings.Count(base, ".") == len(base) {
+		return ""
+	}
+	return filepath.Join(dir, base)
+}
+
+// UnzipToPath extracts a given zip archive into a given path.
+// It returns a list of extracted paths.
+func UnzipToPath(zipFile io.ReaderAt, size int64, outPath string) ([]string, error) {
+	rd, err := zip.NewReader(zipFile, size)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create reader: %w", err)
+	}
+
+	paths := make([]string, len(rd.File))
+	for i, f := range rd.File {
+		filePath := sanitizePath(f.Name)
+		if filePath == "" {
+			return nil, fmt.Errorf("invalid filepath `%s`", f.Name)
+		}
+		path := filepath.Join(outPath, filePath)
+		paths[i] = path
+		if f.FileInfo().IsDir() {
+			if err := os.Mkdir(path, 0744); err != nil {
+				return nil, fmt.Errorf("failed to create directory: %w", err)
+			}
+			continue
+		}
+
+		outFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0600)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create file: %w", err)
+		}
+		defer outFile.Close()
+
+		file, err := f.Open()
+		if err != nil {
+			return nil, fmt.Errorf("failed to open file: %w", err)
+		}
+		defer file.Close()
+
+		if _, err := io.Copy(outFile, file); err != nil {
+			return nil, fmt.Errorf("failed to write to file: %w", err)
+		}
+	}
+
+	return paths, nil
+}

utils/archive_test.go

@@ -0,0 +1,149 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package utils
+
+import (
+	"archive/zip"
+	"errors"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/mattermost/mattermost-server/v5/utils/fileutils"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSanitizePath(t *testing.T) {
+	cases := []struct {
+		input    string
+		expected string
+	}{
+		{
+			".",
+			"",
+		},
+		{
+			"../",
+			"",
+		},
+		{
+			"...",
+			"",
+		},
+		{
+			"..//.",
+			"",
+		},
+		{
+			"/../",
+			"",
+		},
+		{
+			"/path/...../to/file",
+			"/path/to/file",
+		},
+		{
+			"/path/to/file...",
+			"/path/to/file...",
+		},
+		{
+			"/path/to/../../../file",
+			"/file",
+		},
+		{
+			"../../../../file",
+			"/file",
+		},
+		{
+			"/path/to/file..ext",
+			"/path/to/file..ext",
+		},
+		{
+			"/path/to/...file..ext",
+			"/path/to/...file..ext",
+		},
+		{
+			"./path/to/...file..ext",
+			"path/to/...file..ext",
+		},
+		{
+			"./...file",
+			"...file",
+		},
+		{
+			"path/",
+			"path",
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.input, func(t *testing.T) {
+			require.Equal(t, c.expected, sanitizePath(c.input))
+		})
+	}
+}
+
+func TestUnzipToPath(t *testing.T) {
+	testDir, _ := fileutils.FindDir("tests")
+	require.NotEmpty(t, testDir)
+
+	dir, err := ioutil.TempDir("", "unzip")
+	require.Nil(t, err)
+	defer os.RemoveAll(dir)
+
+	t.Run("invalid archive", func(t *testing.T) {
+		file, err := os.Open(testDir + "/testplugin.tar.gz")
+		require.Nil(t, err)
+		defer file.Close()
+
+		info, err := file.Stat()
+		require.Nil(t, err)
+
+		paths, err := UnzipToPath(file, info.Size(), dir)
+		require.NotNil(t, err)
+		require.True(t, errors.Is(err, zip.ErrFormat))
+		require.Nil(t, paths)
+	})
+
+	t.Run("valid archive", func(t *testing.T) {
+		file, err := os.Open(testDir + "/testarchive.zip")
+		require.Nil(t, err)
+		defer file.Close()
+
+		info, err := file.Stat()
+		require.Nil(t, err)
+
+		paths, err := UnzipToPath(file, info.Size(), dir)
+		require.Nil(t, err)
+		require.NotEmpty(t, paths)
+
+		expectedFiles := map[string]int64{
+			dir + "/testfile.txt":           446,
+			dir + "/testdir/testfile2.txt":  866,
+			dir + "/testdir2/testfile3.txt": 845,
+		}
+
+		expectedDirs := []string{
+			dir + "/testdir",
+			dir + "/testdir2",
+		}
+
+		err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+			require.Nil(t, err)
+			if path == dir {
+				return nil
+			}
+			require.Contains(t, paths, path)
+			if info.IsDir() {
+				require.Contains(t, expectedDirs, path)
+			} else {
+				require.Equal(t, expectedFiles[path], info.Size())
+			}
+			return nil
+		})
+		require.Nil(t, err)
+	})
+}