MM-60115, MM-62436, MM-62493: Compliance export downloads from local and s3; e2e tests (#29806)

* add ZipReader method to filestore and file/s3 backends

to merge

to merge

* add WriteStreamResponse as an alternative to WriteFileResponse

* add generated layers

* enable download link; download job endpoint now streams export dir zips

* fix MM-62493

* re-enable e2e tests--we have download links, folks

* Add tests for ZipReader in filestore and s3store

* remove unnecessary error return on ZipReader

* little cleanup

* improve tests; some refactoring: s.Nil(err) -> s.NoError(err)

* blank commit

* backwards compatability for pre-10.5 job downloads

* compress file response; better errors; better comments; PR comments

* update generated app layers

* improve/widen tests; improve comments; simplify localstore ZipReader

* regenerate layers

* follow GoDoc conventions

* update generated layers

* remove unnecessary comment

* in jobs/job-id/download, clean exportDir before sending to ZipReader

* better comments; add an error return on ZipReader

* improve file permissions

* adjust tests for new error returns

* linting

* i18n

server/platform/shared/web/files.go

@@ -10,6 +10,8 @@ import (
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/pkg/errors"
 )
 
 var UnsafeContentTypes = [...]string{
@@ -35,9 +37,10 @@ var MediaContentTypes = [...]string{
 	"audio/wav",
 }
 
+// WriteFileResponse copies the io.ReadSeeker `fileReader` to the ResponseWriter `w`. Use this when you have a
+// ReadSeeker.
 func WriteFileResponse(filename string, contentType string, contentSize int64, lastModification time.Time, webserverMode string, fileReader io.ReadSeeker, forceDownload bool, w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Cache-Control", "private, max-age=86400")
-	w.Header().Set("X-Content-Type-Options", "nosniff")
+	setHeaders(w, contentType, forceDownload, filename)
 
 	if contentSize > 0 {
 		contentSizeStr := strconv.Itoa(int(contentSize))
@@ -48,6 +51,25 @@ func WriteFileResponse(filename string, contentType string, contentSize int64, l
 		}
 	}
 
+	http.ServeContent(w, r, filename, lastModification, fileReader)
+}
+
+// WriteStreamResponse copies the ReadCloser `r` to the ResponseWriter `w`. Use this when you need to stream a response
+// to the client that will appear as a file `filename` of type `contentType`.
+func WriteStreamResponse(w http.ResponseWriter, r io.ReadCloser, filename string, contentType string, forceDownload bool) error {
+	setHeaders(w, contentType, forceDownload, filename)
+
+	if _, err := io.Copy(w, r); err != nil {
+		return errors.Wrap(err, "error streaming ReadCloser")
+	}
+
+	return nil
+}
+
+func setHeaders(w http.ResponseWriter, contentType string, forceDownload bool, filename string) {
+	w.Header().Set("Cache-Control", "private, max-age=86400")
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+
 	if contentType == "" {
 		contentType = "application/octet-stream"
 	} else {
@@ -66,14 +88,12 @@ func WriteFileResponse(filename string, contentType string, contentSize int64, l
 		toDownload = true
 	} else {
 		isMediaType := false
-
 		for _, mediaContentType := range MediaContentTypes {
 			if strings.HasPrefix(contentType, mediaContentType) {
 				isMediaType = true
 				break
 			}
 		}
-
 		toDownload = !isMediaType
 	}
 
@@ -88,6 +108,4 @@ func WriteFileResponse(filename string, contentType string, contentSize int64, l
 	// prevent file links from being embedded in iframes
 	w.Header().Set("X-Frame-Options", "DENY")
 	w.Header().Set("Content-Security-Policy", "Frame-ancestors 'none'")
-
-	http.ServeContent(w, r, filename, lastModification, fileReader)
 }