MM-60115, MM-62436, MM-62493: Compliance export downloads from local and s3; e2e tests (#29806)

* add ZipReader method to filestore and file/s3 backends

to merge

to merge

* add WriteStreamResponse as an alternative to WriteFileResponse

* add generated layers

* enable download link; download job endpoint now streams export dir zips

* fix MM-62493

* re-enable e2e tests--we have download links, folks

* Add tests for ZipReader in filestore and s3store

* remove unnecessary error return on ZipReader

* little cleanup

* improve tests; some refactoring: s.Nil(err) -> s.NoError(err)

* blank commit

* backwards compatability for pre-10.5 job downloads

* compress file response; better errors; better comments; PR comments

* update generated app layers

* improve/widen tests; improve comments; simplify localstore ZipReader

* regenerate layers

* follow GoDoc conventions

* update generated layers

* remove unnecessary comment

* in jobs/job-id/download, clean exportDir before sending to ZipReader

* better comments; add an error return on ZipReader

* improve file permissions

* adjust tests for new error returns

* linting

* i18n

server/channels/api4/job.go

@@ -6,10 +6,13 @@ package api4
 import (
 	"encoding/json"
 	"net/http"
+	"path"
 	"path/filepath"
 	"strconv"
 	"time"
 
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+
 	"github.com/mattermost/mattermost/server/public/model"
 	"github.com/mattermost/mattermost/server/public/shared/mlog"
 	"github.com/mattermost/mattermost/server/v8/channels/audit"
@@ -55,7 +58,7 @@ func getJob(c *Context, w http.ResponseWriter, r *http.Request) {
 
 func downloadJob(c *Context, w http.ResponseWriter, r *http.Request) {
 	config := c.App.Config()
-	const FilePath = "export"
+	const oldFilePath = "export"
 	const FileMime = "application/zip"
 
 	c.RequireJobId()
@@ -90,19 +93,53 @@ func downloadJob(c *Context, w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	fileName := job.Id + ".zip"
-	filePath := filepath.Join(FilePath, fileName)
-	fileReader, err := c.App.FileReader(filePath)
-	if err != nil {
-		c.Err = err
-		c.Err.StatusCode = http.StatusNotFound
+	exportDir, ok := job.Data["export_dir"]
+	fileName := path.Base(exportDir)
+	if !ok || exportDir == "" || fileName == "/" || fileName == "." {
+		// Could be a pre-overhaul job. Try the old method:
+		fileName = job.Id + ".zip"
+		filePath := filepath.Join(oldFilePath, fileName)
+		var fileReader filestore.ReadCloseSeeker
+		fileReader, err = c.App.FileReader(filePath)
+		if err != nil {
+			c.Err = model.NewAppError("unableToDownloadJob", "api.job.unable_to_download_job", nil,
+				"job.Data did not include export_dir, export_dir was malformed, or jobId.zip wasn't found",
+				http.StatusNotFound).Wrap(err)
+			return
+		}
+		defer fileReader.Close()
+
+		// We are able to pass 0 for content size due to the fact that Golang's serveContent (https://golang.org/src/net/http/fs.go)
+		// already sets that for us
+		web.WriteFileResponse(fileName, FileMime, 0, time.UnixMilli(job.LastActivityAt), *c.App.Config().ServiceSettings.WebserverMode, fileReader, true, w, r)
 		return
 	}
-	defer fileReader.Close()
 
-	// We are able to pass 0 for content size due to the fact that Golang's serveContent (https://golang.org/src/net/http/fs.go)
-	// already sets that for us
-	web.WriteFileResponse(fileName, FileMime, 0, time.Unix(0, job.LastActivityAt*int64(1000*1000)), *c.App.Config().ServiceSettings.WebserverMode, fileReader, true, w, r)
+	// We have a base directory, we're using that as the exported filename:
+	fileName += ".zip"
+
+	cleanedExportDir := filepath.Clean(exportDir)
+	if !filepath.IsLocal(cleanedExportDir) {
+		c.Err = model.NewAppError("unableToDownloadJob", "api.job.unable_to_download_job", nil,
+			"job.Data did not include export_dir, export_dir was malformed, or jobId.zip wasn't found",
+			http.StatusNotFound).Wrap(err)
+		return
+	}
+
+	zipReader, err := c.App.ZipReader(cleanedExportDir, false)
+	if err != nil {
+		c.Err = model.NewAppError("unableToDownloadJob", "api.job.unable_to_download_job", nil,
+			"error creating zip reader", http.StatusNotFound).Wrap(err)
+		return
+	}
+	defer zipReader.Close()
+
+	if err := web.WriteStreamResponse(w, zipReader, fileName, FileMime, true); err != nil {
+		c.Err = model.NewAppError("unableToDownloadJob", "api.job.unable_to_download_job", nil,
+			"failure to WriteStreamResponse", http.StatusInternalServerError).
+			Wrap(err)
+		return
+	}
 }
 
 func createJob(c *Context, w http.ResponseWriter, r *http.Request) {