[MM-28422] Enable processing of import files through API (#16062)

* Implement unzip function

* Implement FileSize method

* Implement path rewriting for bulk import

* Small improvements

* Add ImportSettings to config

* Implement ListImports API endpoint

* Enable uploading import files

* Implement import process job

* Add missing license headers

* Address reviews

* Make path sanitization a bit smarter

* Clean path before calculating Dir

* [MM-30008] Add mmctl support for file imports (#16301)

* Add mmctl support for import files

* Improve test

* Remove unnecessary handlers

* Use th.TestForSystemAdminAndLocal

* Make nouser id a constant

model/client4.go

@@ -543,6 +543,10 @@ func (c *Client4) GetGroupSyncablesRoute(groupID string, syncableType GroupSynca
 	return fmt.Sprintf("%s/%ss", c.GetGroupRoute(groupID), strings.ToLower(syncableType.String()))
 }
 
+func (c *Client4) GetImportsRoute() string {
+	return "/imports"
+}
+
 func (c *Client4) DoApiGet(url string, etag string) (*http.Response, *AppError) {
 	return c.DoApiRequest(http.MethodGet, c.ApiUrl+url, "", etag)
 }
@@ -5754,6 +5758,15 @@ func (c *Client4) UpdateCloudCustomerAddress(address *Address) (*CloudCustomer,
 	return customer, BuildResponse(r)
 }
 
+func (c *Client4) ListImports() ([]string, *Response) {
+	r, err := c.DoApiGet(c.GetImportsRoute(), "")
+	if err != nil {
+		return nil, BuildErrorResponse(r, err)
+	}
+	defer closeBody(r)
+	return ArrayFromJson(r.Body), BuildResponse(r)
+}
+
 func (c *Client4) GetUserThreads(userId string, options GetUserThreadsOpts) (*Threads, *Response) {
 	v := url.Values{}
 	if options.Since != 0 {

model/config.go

@@ -117,6 +117,9 @@ const (
 
 	FILE_SETTINGS_DEFAULT_DIRECTORY = "./data/"
 
+	IMPORT_SETTINGS_DEFAULT_DIRECTORY      = "./import"
+	IMPORT_SETTINGS_DEFAULT_RETENTION_DAYS = 30
+
 	EMAIL_SETTINGS_DEFAULT_FEEDBACK_ORGANIZATION = ""
 
 	SUPPORT_SETTINGS_DEFAULT_TERMS_OF_SERVICE_LINK = "https://about.mattermost.com/default-terms/"
@@ -2854,6 +2857,37 @@ func (s *ImageProxySettings) SetDefaults(ss ServiceSettings) {
 	}
 }
 
+// ImportSettings defines configuration settings for file imports.
+type ImportSettings struct {
+	// The directory where to store the imported files.
+	Directory *string
+	// The number of days to retain the imported files before deleting them.
+	RetentionDays *int
+}
+
+func (s *ImportSettings) isValid() *AppError {
+	if *s.Directory == "" {
+		return NewAppError("Config.IsValid", "model.config.is_valid.import.directory.app_error", nil, "", http.StatusBadRequest)
+	}
+
+	if *s.RetentionDays <= 0 {
+		return NewAppError("Config.IsValid", "model.config.is_valid.import.retention_days_too_low.app_error", nil, "", http.StatusBadRequest)
+	}
+
+	return nil
+}
+
+// SetDefaults applies the default settings to the struct.
+func (s *ImportSettings) SetDefaults() {
+	if s.Directory == nil || *s.Directory == "" {
+		s.Directory = NewString(IMPORT_SETTINGS_DEFAULT_DIRECTORY)
+	}
+
+	if s.RetentionDays == nil {
+		s.RetentionDays = NewInt(IMPORT_SETTINGS_DEFAULT_RETENTION_DAYS)
+	}
+}
+
 type ConfigFunc func() *Config
 
 const ConfigAccessTagType = "access"
@@ -2929,6 +2963,7 @@ type Config struct {
 	ImageProxySettings        ImageProxySettings
 	CloudSettings             CloudSettings
 	FeatureFlags              *FeatureFlags `json:",omitempty"`
+	ImportSettings            ImportSettings
 }
 
 func (o *Config) Clone() *Config {
@@ -3032,6 +3067,7 @@ func (o *Config) SetDefaults() {
 		o.FeatureFlags = &FeatureFlags{}
 		o.FeatureFlags.SetDefaults()
 	}
+	o.ImportSettings.SetDefaults()
 }
 
 func (o *Config) IsValid() *AppError {
@@ -3110,6 +3146,10 @@ func (o *Config) IsValid() *AppError {
 	if err := o.ImageProxySettings.isValid(); err != nil {
 		return err
 	}
+
+	if err := o.ImportSettings.isValid(); err != nil {
+		return err
+	}
 	return nil
 }
 

model/config_test.go

@@ -1389,3 +1389,31 @@ func TestSetDefaultFeatureFlagBehaviour(t *testing.T) {
 	require.Equal(t, "somevalue", cfg.FeatureFlags.TestFeature)
 
 }
+
+func TestConfigImportSettingsDefaults(t *testing.T) {
+	cfg := Config{}
+	cfg.SetDefaults()
+
+	require.Equal(t, "./import", *cfg.ImportSettings.Directory)
+	require.Equal(t, 30, *cfg.ImportSettings.RetentionDays)
+}
+
+func TestConfigImportSettingsIsValid(t *testing.T) {
+	cfg := Config{}
+	cfg.SetDefaults()
+
+	err := cfg.ImportSettings.isValid()
+	require.Nil(t, err)
+
+	*cfg.ImportSettings.Directory = ""
+	err = cfg.ImportSettings.isValid()
+	require.NotNil(t, err)
+	require.Equal(t, "model.config.is_valid.import.directory.app_error", err.Id)
+
+	cfg.SetDefaults()
+
+	*cfg.ImportSettings.RetentionDays = 0
+	err = cfg.ImportSettings.isValid()
+	require.NotNil(t, err)
+	require.Equal(t, "model.config.is_valid.import.retention_days_too_low.app_error", err.Id)
+}

model/job.go

@@ -22,6 +22,7 @@ const (
 	JOB_TYPE_EXPIRY_NOTIFY                  = "expiry_notify"
 	JOB_TYPE_PRODUCT_NOTICES                = "product_notices"
 	JOB_TYPE_ACTIVE_USERS                   = "active_users"
+	JOB_TYPE_IMPORT_PROCESS                 = "import_process"
 	JOB_TYPE_CLOUD                          = "cloud"
 
 	JOB_STATUS_PENDING          = "pending"
@@ -66,6 +67,7 @@ func (j *Job) IsValid() *AppError {
 	case JOB_TYPE_PRODUCT_NOTICES:
 	case JOB_TYPE_EXPIRY_NOTIFY:
 	case JOB_TYPE_ACTIVE_USERS:
+	case JOB_TYPE_IMPORT_PROCESS:
 	case JOB_TYPE_CLOUD:
 	default:
 		return NewAppError("Job.IsValid", "model.job.is_valid.type.app_error", nil, "id="+j.Id, http.StatusBadRequest)

model/upload_session.go

@@ -18,6 +18,9 @@ const (
 	UploadTypeImport     UploadType = "import"
 )
 
+// UploadNoUserID is a "fake" user id used by the API layer when in local mode.
+const UploadNoUserID = "nouser"
+
 // UploadSession contains information used to keep track of a file upload.
 type UploadSession struct {
 	// The unique identifier for the session.
@@ -29,7 +32,7 @@ type UploadSession struct {
 	// The id of the user performing the upload.
 	UserId string `json:"user_id"`
 	// The id of the channel to upload to.
-	ChannelId string `json:"channel_id"`
+	ChannelId string `json:"channel_id,omitempty"`
 	// The name of the file to upload.
 	Filename string `json:"filename"`
 	// The path where the file is stored.
@@ -109,7 +112,7 @@ func (us *UploadSession) IsValid() *AppError {
 		return NewAppError("UploadSession.IsValid", "model.upload_session.is_valid.type.app_error", nil, err.Error(), http.StatusBadRequest)
 	}
 
-	if !IsValidId(us.UserId) {
+	if !IsValidId(us.UserId) && us.UserId != UploadNoUserID {
 		return NewAppError("UploadSession.IsValid", "model.upload_session.is_valid.user_id.app_error", nil, "id="+us.Id, http.StatusBadRequest)
 	}