[MM-19640] Retrieve Compliance files from the System Console (#14976)

* add query param for downloading file * Address PR comments * sanitize error messages * revert error santization * address PR comments * Log Error * Comment for clarification * Add test + Opt In Option * Fix typo * Remove line number * Check is downloadable in server * Don't use constants * Check for downloadExportResults in API * make actiance export zip * make i18n strings * Remove translations * Add translations * Revert "Add translations" This reverts commit adc5d28b3e7712f51cd3eff565e30fb8ddfd6f65. Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
2020-07-27 08:37:04 -04:00
--- a/api4/job.go
+++ b/api4/job.go
@@ -4,9 +4,13 @@
 package api4

 import (
+	"fmt"
 	"net/http"
+	"strconv"
+	"time"

 	"github.com/mattermost/mattermost-server/v5/audit"
+	"github.com/mattermost/mattermost-server/v5/mlog"
 	"github.com/mattermost/mattermost-server/v5/model"
 )

@@ -14,6 +18,7 @@ func (api *API) InitJob() {
 	api.BaseRoutes.Jobs.Handle("", api.ApiSessionRequired(getJobs)).Methods("GET")
 	api.BaseRoutes.Jobs.Handle("", api.ApiSessionRequired(createJob)).Methods("POST")
 	api.BaseRoutes.Jobs.Handle("/{job_id:[A-Za-z0-9]+}", api.ApiSessionRequired(getJob)).Methods("GET")
+	api.BaseRoutes.Jobs.Handle("/{job_id:[A-Za-z0-9]+}/download", api.ApiSessionRequiredTrustRequester(downloadJob)).Methods("GET")
 	api.BaseRoutes.Jobs.Handle("/{job_id:[A-Za-z0-9]+}/cancel", api.ApiSessionRequired(cancelJob)).Methods("POST")
 	api.BaseRoutes.Jobs.Handle("/type/{job_type:[A-Za-z0-9_-]+}", api.ApiSessionRequired(getJobsByType)).Methods("GET")
 }
@@ -38,6 +43,68 @@ func getJob(c *Context, w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte(job.ToJson()))
 }

+func downloadJob(c *Context, w http.ResponseWriter, r *http.Request) {
+	config := c.App.Config()
+	const FILE_PATH = "export/%s/%s"
+	fileInfo := map[string]map[string]string{
+		"csv": {
+			"fileName": "csv_export.zip",
+			"fileMime": "application/zip",
+		},
+		"actiance": {
+			"fileName": "actiance_export.zip",
+			"fileMime": "application/zip",
+		},
+	}
+
+	c.RequireJobId()
+	if c.Err != nil {
+		return
+	}
+
+	if !*config.MessageExportSettings.DownloadExportResults {
+		c.Err = model.NewAppError("downloadExportResultsNotEnabled", "app.job.download_export_results_not_enabled", nil, "", http.StatusNotImplemented)
+		return
+	}
+
+	if !c.App.SessionHasPermissionTo(*c.App.Session(), model.PERMISSION_MANAGE_JOBS) {
+		c.SetPermissionError(model.PERMISSION_MANAGE_JOBS)
+		return
+	}
+
+	job, err := c.App.GetJob(c.Params.JobId)
+	if err != nil {
+		mlog.Error(err.Error())
+		c.Err = err
+		return
+	}
+
+	isDownloadable, _ := strconv.ParseBool(job.Data["is_downloadable"])
+	if !isDownloadable {
+		c.Err = model.NewAppError("unableToDownloadJob", "api.job.unable_to_download_job", nil, "", http.StatusBadRequest)
+		return
+	}
+
+	filePath := fmt.Sprintf(FILE_PATH, job.Id, fileInfo[job.Data["export_type"]]["fileName"])
+	fileReader, err := c.App.FileReader(filePath)
+	if err != nil {
+		mlog.Error(err.Error())
+		c.Err = err
+		c.Err.StatusCode = http.StatusNotFound
+		return
+	}
+	defer fileReader.Close()
+
+	// We are able to pass 0 for content size due to the fact that Golang's serveContent (https://golang.org/src/net/http/fs.go)
+	// already sets that for us
+	err = writeFileResponse(fileInfo[job.Data["export_type"]]["fileName"], fileInfo[job.Data["export_type"]]["fileMime"], 0, time.Unix(0, job.LastActivityAt*int64(1000*1000)), *c.App.Config().ServiceSettings.WebserverMode, fileReader, true, w, r)
+	if err != nil {
+		mlog.Error(err.Error())
+		c.Err = err
+		return
+	}
+}
+
 func createJob(c *Context, w http.ResponseWriter, r *http.Request) {
 	job := model.JobFromJson(r.Body)
 	if job == nil {
--- a/api4/job_test.go
+++ b/api4/job_test.go
@@ -4,6 +4,8 @@
 package api4

 import (
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"

@@ -172,6 +174,68 @@ func TestGetJobsByType(t *testing.T) {
 	CheckForbiddenStatus(t, resp)
 }

+func TestDownloadJob(t *testing.T) {
+	th := Setup(t).InitBasic()
+	defer th.TearDown()
+	jobName := model.NewId()
+	job := &model.Job{
+		Id:   jobName,
+		Type: model.JOB_TYPE_MESSAGE_EXPORT,
+		Data: map[string]string{
+			"export_type": "csv",
+		},
+		Status: model.JOB_STATUS_SUCCESS,
+	}
+
+	// DownloadExportResults is not set to true so we should get a not implemented error status
+	_, resp := th.Client.DownloadJob(job.Id)
+	CheckNotImplementedStatus(t, resp)
+
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.MessageExportSettings.DownloadExportResults = true
+	})
+
+	// Normal user cannot download the results of these job (Doesn't have permission)
+	_, resp = th.Client.DownloadJob(job.Id)
+	CheckForbiddenStatus(t, resp)
+
+	// System admin trying to download the results of a non-existant job
+	_, resp = th.SystemAdminClient.DownloadJob(job.Id)
+	CheckNotFoundStatus(t, resp)
+
+	// Here we have a job that exist in our database but the results do not exist therefore when we try to download the results
+	// as a system admin, we should get a not found status.
+	_, err := th.App.Srv().Store.Job().Save(job)
+	require.Nil(t, err)
+	defer th.App.Srv().Store.Job().Delete(job.Id)
+
+	filePath := "./data/export/" + job.Id + "/testdat.txt"
+	mkdirAllErr := os.MkdirAll(filepath.Dir(filePath), 0770)
+	require.Nil(t, mkdirAllErr)
+	os.Create(filePath)
+
+	_, resp = th.SystemAdminClient.DownloadJob(job.Id)
+	CheckBadRequestStatus(t, resp)
+
+	job.Data["is_downloadable"] = "true"
+	updateStatus, err := th.App.Srv().Store.Job().UpdateOptimistically(job, model.JOB_STATUS_SUCCESS)
+	require.True(t, updateStatus)
+	require.Nil(t, err)
+
+	_, resp = th.SystemAdminClient.DownloadJob(job.Id)
+	CheckNotFoundStatus(t, resp)
+
+	// Now we stub the results of the job into the same directory and try to download it again
+	// This time we should successfully retrieve the results without any error
+	filePath = "./data/export/" + job.Id + "/csv_export.zip"
+	mkdirAllErr = os.MkdirAll(filepath.Dir(filePath), 0770)
+	require.Nil(t, mkdirAllErr)
+	os.Create(filePath)
+
+	_, resp = th.SystemAdminClient.DownloadJob(job.Id)
+	require.Nil(t, resp.Error)
+}
+
 func TestCancelJob(t *testing.T) {
 	th := Setup(t)
 	defer th.TearDown()
--- a/i18n/en.json
+++ b/i18n/en.json
@@ -1452,6 +1452,10 @@
    "id": "api.io_error",
    "translation": "input/output error"
  },
+  {
+    "id": "api.job.unable_to_download_job",
+    "translation": "Unable to download this job"
+  },
  {
    "id": "api.ldap_group.not_found",
    "translation": "ldap group not found"
@@ -3954,6 +3958,10 @@
    "id": "app.import.validate_user_teams_import_data.team_name_missing.error",
    "translation": "Team name missing from User's Team Membership."
  },
+  {
+    "id": "app.job.download_export_results_not_enabled",
+    "translation": "DownloadExportResults in config.json is false. Please set this to true to download the results of this job."
+  },
  {
    "id": "app.notification.body.intro.direct.full",
    "translation": "You have a new Direct Message."
--- a/model/client4.go
+++ b/model/client4.go
@@ -4698,6 +4698,21 @@ func (c *Client4) CancelJob(jobId string) (bool, *Response) {
 	return CheckStatusOK(r), BuildResponse(r)
 }

+// DownloadJob downloads the results of the job
+func (c *Client4) DownloadJob(jobId string) ([]byte, *Response) {
+	r, appErr := c.DoApiGet(c.GetJobsRoute()+fmt.Sprintf("/%v/download", jobId), "")
+	if appErr != nil {
+		return nil, BuildErrorResponse(r, appErr)
+	}
+	defer closeBody(r)
+
+	data, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		return nil, BuildErrorResponse(r, NewAppError("GetFile", "model.client.read_job_result_file.app_error", nil, err.Error(), r.StatusCode))
+	}
+	return data, BuildResponse(r)
+}
+
 // Roles Section

 // GetRole gets a single role by ID.
--- a/model/config.go
+++ b/model/config.go
@@ -2615,11 +2615,12 @@ func (s *GlobalRelayMessageExportSettings) SetDefaults() {
 }

 type MessageExportSettings struct {
-	EnableExport        *bool
-	ExportFormat        *string
-	DailyRunTime        *string
-	ExportFromTimestamp *int64
-	BatchSize           *int
+	EnableExport          *bool
+	DownloadExportResults *bool
+	ExportFormat          *string
+	DailyRunTime          *string
+	ExportFromTimestamp   *int64
+	BatchSize             *int

 	// formatter-specific settings - these are only expected to be non-nil if ExportFormat is set to the associated format
 	GlobalRelaySettings *GlobalRelayMessageExportSettings
@@ -2630,6 +2631,10 @@ func (s *MessageExportSettings) SetDefaults() {
 		s.EnableExport = NewBool(false)
 	}

+	if s.DownloadExportResults == nil {
+		s.DownloadExportResults = NewBool(false)
+	}
+
 	if s.ExportFormat == nil {
 		s.ExportFormat = NewString(COMPLIANCE_EXPORT_TYPE_ACTIANCE)
 	}