[MM-31247] Add support for compressed export files with attachments (#16614)

* Include filepaths for post attachments * Cleanup * Enable exporting file attachments * Fix file import * Enable zip export * Support creating missing directories when unzipping * Add test * Add translations * Export direct channel posts attachments * Fix returned values order Remove pointer to slice in return * [MM-31597] Implement export process job (#16626) * Implement export process job * Add translations * Remove unused value * [MM-31249] Add /exports API endpoint (#16633) * Implement API endpoints to list, download and delete export files * Add endpoint for single resource * Update i18n/en.json Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com> * Update i18n/en.json Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com> Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com> * Fix var name Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com>
2021-02-09 11:58:31 +01:00
--- a/model/client4.go
+++ b/model/client4.go
@@ -34,6 +34,7 @@ const (
 	HEADER_CLOUD_TOKEN        = "X-Cloud-Token"
 	HEADER_REQUESTED_WITH     = "X-Requested-With"
 	HEADER_REQUESTED_WITH_XML = "XMLHttpRequest"
+	HEADER_RANGE              = "Range"
 	STATUS                    = "status"
 	STATUS_OK                 = "OK"
 	STATUS_FAIL               = "FAIL"
@@ -550,6 +551,14 @@ func (c *Client4) GetImportsRoute() string {
 	return "/imports"
 }

+func (c *Client4) GetExportsRoute() string {
+	return "/exports"
+}
+
+func (c *Client4) GetExportRoute(name string) string {
+	return fmt.Sprintf(c.GetExportsRoute()+"/%v", name)
+}
+
 func (c *Client4) DoApiGet(url string, etag string) (*http.Response, *AppError) {
 	return c.DoApiRequest(http.MethodGet, c.ApiUrl+url, "", etag)
 }
@@ -575,21 +584,25 @@ func (c *Client4) DoApiDelete(url string) (*http.Response, *AppError) {
 }

 func (c *Client4) DoApiRequest(method, url, data, etag string) (*http.Response, *AppError) {
-	return c.doApiRequestReader(method, url, strings.NewReader(data), etag)
+	return c.doApiRequestReader(method, url, strings.NewReader(data), map[string]string{HEADER_ETAG_CLIENT: etag})
+}
+
+func (c *Client4) DoApiRequestWithHeaders(method, url, data string, headers map[string]string) (*http.Response, *AppError) {
+	return c.doApiRequestReader(method, url, strings.NewReader(data), headers)
 }

 func (c *Client4) doApiRequestBytes(method, url string, data []byte, etag string) (*http.Response, *AppError) {
-	return c.doApiRequestReader(method, url, bytes.NewReader(data), etag)
+	return c.doApiRequestReader(method, url, bytes.NewReader(data), map[string]string{HEADER_ETAG_CLIENT: etag})
 }

-func (c *Client4) doApiRequestReader(method, url string, data io.Reader, etag string) (*http.Response, *AppError) {
+func (c *Client4) doApiRequestReader(method, url string, data io.Reader, headers map[string]string) (*http.Response, *AppError) {
 	rq, err := http.NewRequest(method, url, data)
 	if err != nil {
 		return nil, NewAppError(url, "model.client.connecting.app_error", nil, err.Error(), http.StatusBadRequest)
 	}

-	if etag != "" {
-		rq.Header.Set(HEADER_ETAG_CLIENT, etag)
+	for k, v := range headers {
+		rq.Header.Set(k, v)
 	}

 	if c.AuthToken != "" {
@@ -5668,7 +5681,7 @@ func (c *Client4) GetUploadsForUser(userId string) ([]*UploadSession, *Response)
 // a FileInfo object.
 func (c *Client4) UploadData(uploadId string, data io.Reader) (*FileInfo, *Response) {
 	url := c.GetUploadRoute(uploadId)
-	r, err := c.doApiRequestReader("POST", c.ApiUrl+url, data, "")
+	r, err := c.doApiRequestReader("POST", c.ApiUrl+url, data, nil)
 	if err != nil {
 		return nil, BuildErrorResponse(r, err)
 	}
@@ -5816,6 +5829,43 @@ func (c *Client4) ListImports() ([]string, *Response) {
 	return ArrayFromJson(r.Body), BuildResponse(r)
 }

+func (c *Client4) ListExports() ([]string, *Response) {
+	r, err := c.DoApiGet(c.GetExportsRoute(), "")
+	if err != nil {
+		return nil, BuildErrorResponse(r, err)
+	}
+	defer closeBody(r)
+	return ArrayFromJson(r.Body), BuildResponse(r)
+}
+
+func (c *Client4) DeleteExport(name string) (bool, *Response) {
+	r, err := c.DoApiDelete(c.GetExportRoute(name))
+	if err != nil {
+		return false, BuildErrorResponse(r, err)
+	}
+	defer closeBody(r)
+	return CheckStatusOK(r), BuildResponse(r)
+}
+
+func (c *Client4) DownloadExport(name string, wr io.Writer, offset int64) (int64, *Response) {
+	var headers map[string]string
+	if offset > 0 {
+		headers = map[string]string{
+			HEADER_RANGE: fmt.Sprintf("bytes=%d-", offset),
+		}
+	}
+	r, appErr := c.DoApiRequestWithHeaders(http.MethodGet, c.ApiUrl+c.GetExportRoute(name), "", headers)
+	if appErr != nil {
+		return 0, BuildErrorResponse(r, appErr)
+	}
+	defer closeBody(r)
+	n, err := io.Copy(wr, r.Body)
+	if err != nil {
+		return n, BuildErrorResponse(r, NewAppError("DownloadExport", "model.client.copy.app_error", nil, err.Error(), r.StatusCode))
+	}
+	return n, BuildResponse(r)
+}
+
 func (c *Client4) GetThreadMentionsForUserPerChannel(userId, teamId string) (map[string]int64, *Response) {
 	url := c.GetUserThreadsRoute(userId, teamId)
 	r, appErr := c.DoApiGet(url+"/mention_counts", "")
--- a/model/config.go
+++ b/model/config.go
@@ -122,6 +122,9 @@ const (
 	IMPORT_SETTINGS_DEFAULT_DIRECTORY      = "./import"
 	IMPORT_SETTINGS_DEFAULT_RETENTION_DAYS = 30

+	EXPORT_SETTINGS_DEFAULT_DIRECTORY      = "./export"
+	EXPORT_SETTINGS_DEFAULT_RETENTION_DAYS = 30
+
 	EMAIL_SETTINGS_DEFAULT_FEEDBACK_ORGANIZATION = ""

 	SUPPORT_SETTINGS_DEFAULT_TERMS_OF_SERVICE_LINK = "https://about.mattermost.com/default-terms/"
@@ -2936,6 +2939,37 @@ func (s *ImportSettings) SetDefaults() {
 	}
 }

+// ExportSettings defines configuration settings for file exports.
+type ExportSettings struct {
+	// The directory where to store the exported files.
+	Directory *string
+	// The number of days to retain the exported files before deleting them.
+	RetentionDays *int
+}
+
+func (s *ExportSettings) isValid() *AppError {
+	if *s.Directory == "" {
+		return NewAppError("Config.IsValid", "model.config.is_valid.export.directory.app_error", nil, "", http.StatusBadRequest)
+	}
+
+	if *s.RetentionDays <= 0 {
+		return NewAppError("Config.IsValid", "model.config.is_valid.export.retention_days_too_low.app_error", nil, "", http.StatusBadRequest)
+	}
+
+	return nil
+}
+
+// SetDefaults applies the default settings to the struct.
+func (s *ExportSettings) SetDefaults() {
+	if s.Directory == nil || *s.Directory == "" {
+		s.Directory = NewString(EXPORT_SETTINGS_DEFAULT_DIRECTORY)
+	}
+
+	if s.RetentionDays == nil {
+		s.RetentionDays = NewInt(EXPORT_SETTINGS_DEFAULT_RETENTION_DAYS)
+	}
+}
+
 type ConfigFunc func() *Config

 const ConfigAccessTagType = "access"
@@ -3013,6 +3047,7 @@ type Config struct {
 	CloudSettings             CloudSettings
 	FeatureFlags              *FeatureFlags `json:",omitempty"`
 	ImportSettings            ImportSettings
+	ExportSettings            ExportSettings
 }

 func (o *Config) Clone() *Config {
@@ -3121,6 +3156,7 @@ func (o *Config) SetDefaults() {
 		o.FeatureFlags.SetDefaults()
 	}
 	o.ImportSettings.SetDefaults()
+	o.ExportSettings.SetDefaults()
 }

 func (o *Config) IsValid() *AppError {
--- a/model/config_test.go
+++ b/model/config_test.go
@@ -1439,3 +1439,31 @@ func TestConfigImportSettingsIsValid(t *testing.T) {
 	require.NotNil(t, err)
 	require.Equal(t, "model.config.is_valid.import.retention_days_too_low.app_error", err.Id)
 }
+
+func TestConfigExportSettingsDefaults(t *testing.T) {
+	cfg := Config{}
+	cfg.SetDefaults()
+
+	require.Equal(t, "./export", *cfg.ExportSettings.Directory)
+	require.Equal(t, 30, *cfg.ExportSettings.RetentionDays)
+}
+
+func TestConfigExportSettingsIsValid(t *testing.T) {
+	cfg := Config{}
+	cfg.SetDefaults()
+
+	err := cfg.ExportSettings.isValid()
+	require.Nil(t, err)
+
+	*cfg.ExportSettings.Directory = ""
+	err = cfg.ExportSettings.isValid()
+	require.NotNil(t, err)
+	require.Equal(t, "model.config.is_valid.export.directory.app_error", err.Id)
+
+	cfg.SetDefaults()
+
+	*cfg.ExportSettings.RetentionDays = 0
+	err = cfg.ExportSettings.isValid()
+	require.NotNil(t, err)
+	require.Equal(t, "model.config.is_valid.export.retention_days_too_low.app_error", err.Id)
+}
--- a/model/job.go
+++ b/model/job.go
@@ -24,6 +24,7 @@ const (
 	JOB_TYPE_ACTIVE_USERS                   = "active_users"
 	JOB_TYPE_IMPORT_PROCESS                 = "import_process"
 	JOB_TYPE_IMPORT_DELETE                  = "import_delete"
+	JOB_TYPE_EXPORT_PROCESS                 = "export_process"
 	JOB_TYPE_CLOUD                          = "cloud"

 	JOB_STATUS_PENDING          = "pending"
@@ -70,6 +71,7 @@ func (j *Job) IsValid() *AppError {
 	case JOB_TYPE_ACTIVE_USERS:
 	case JOB_TYPE_IMPORT_PROCESS:
 	case JOB_TYPE_IMPORT_DELETE:
+	case JOB_TYPE_EXPORT_PROCESS:
 	case JOB_TYPE_CLOUD:
 	default:
 		return NewAppError("Job.IsValid", "model.job.is_valid.type.app_error", nil, "id="+j.Id, http.StatusBadRequest)