MM-25095: Handles static JS and CSS using Brotli encoding. (#14524)

* MM-25095: Handles static JS and CSS files using Brotli.

* MM-25095: Linting fix.

* MM-25095: Adds missing license.

* MM-25095: Moves initialization of slice.

* MM-25095: Moves initialization of slice.

api4/handlers.go

@@ -6,8 +6,8 @@ package api4
 import (
 	"net/http"
 
-	"github.com/NYTimes/gziphandler"
 	"github.com/mattermost/mattermost-server/v5/web"
+	"github.com/mkraft/gziphandler"
 )
 
 type Context = web.Context

go.mod

@@ -4,7 +4,6 @@ go 1.14
 
 require (
 	github.com/Masterminds/squirrel v1.2.0
-	github.com/NYTimes/gziphandler v1.1.1
 	github.com/armon/go-metrics v0.3.0 // indirect
 	github.com/avct/uasurfer v0.0.0-20191028135549-26b5daa857f1
 	github.com/beevik/etree v1.1.0 // indirect
@@ -63,6 +62,7 @@ require (
 	github.com/minio/minio-go/v6 v6.0.55
 	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
 	github.com/mitchellh/mapstructure v1.2.3 // indirect
+	github.com/mkraft/gziphandler v1.1.2-0.20200509175700-73dc64f3ad90
 	github.com/muesli/smartcrop v0.3.0 // indirect
 	github.com/oklog/run v1.1.0 // indirect
 	github.com/olekukonko/tablewriter v0.0.4 // indirect
@@ -75,7 +75,7 @@ require (
 	github.com/prometheus/procfs v0.0.11 // indirect
 	github.com/rs/cors v1.7.0
 	github.com/rudderlabs/analytics-go v3.2.1+incompatible
-	github.com/russellhaering/goxmldsig v0.0.0-20180430223755-7acd5e4a6ef7 // indirect
+	github.com/russellhaering/goxmldsig v0.0.0-20180430223755-7acd5e4a6ef7
 	github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd
 	github.com/segmentio/analytics-go v3.1.0+incompatible
 	github.com/segmentio/backo-go v0.0.0-20200129164019-23eae7c10bd3 // indirect

go.sum

@@ -20,8 +20,6 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0
 github.com/Masterminds/squirrel v1.2.0 h1:K1NhbTO21BWG47IVR0OnIZuE0LZcXAYqywrC3Ko53KI=
 github.com/Masterminds/squirrel v1.2.0/go.mod h1:yaPeOnPG5ZRwL9oKdTsO/prlkPbXWZlRVMQ/gGlzIuA=
 github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
-github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
-github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PaulARoy/azurestoragecache v0.0.0-20170906084534-3c249a3ba788/go.mod h1:lY1dZd8HBzJ10eqKERHn3CU59tfhzcAVb2c0ZhIWSOk=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
@@ -358,6 +356,12 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.2.3 h1:f/MjBEBDLttYCGfRaKBbKSRVF5aV2O6fnBpzknuE3jU=
 github.com/mitchellh/mapstructure v1.2.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mkraft/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
+github.com/mkraft/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
+github.com/mkraft/gziphandler v1.1.2-0.20200509170533-f387ff5f65bc h1:S6AKguh0+X1d0bW9+catK3n+PMX/dke1lkL4uxxukd8=
+github.com/mkraft/gziphandler v1.1.2-0.20200509170533-f387ff5f65bc/go.mod h1:gG8WEPb2aI5MHdmHv83au7bk3molRSZiAjdxYrEMJdQ=
+github.com/mkraft/gziphandler v1.1.2-0.20200509175700-73dc64f3ad90 h1:qEm+0lDAcdszJkCnYSuP1oITiQf0TzDIAmeD2PlBtZU=
+github.com/mkraft/gziphandler v1.1.2-0.20200509175700-73dc64f3ad90/go.mod h1:gG8WEPb2aI5MHdmHv83au7bk3molRSZiAjdxYrEMJdQ=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=

vendor/github.com/mkraft/gziphandler/go.mod

@@ -1,4 +1,4 @@
-module github.com/NYTimes/gziphandler
+module github.com/mkraft/gziphandler
 
 go 1.11
 

vendor/github.com/mkraft/gziphandler/gzip.go

@@ -1,4 +1,4 @@
-package gziphandler // import "github.com/NYTimes/gziphandler"
+package gziphandler // import "github.com/mkraft/gziphandler"
 
 import (
 	"bufio"
@@ -81,11 +81,13 @@ type GzipResponseWriter struct {
 
 	code int // Saves the WriteHeader value.
 
-	minSize int    // Specifed the minimum response size to gzip. If the response length is bigger than this value, it is compressed.
+	minSize int    // Specifies the minimum response size to gzip. If the response length is bigger than this value, it is compressed.
 	buf     []byte // Holds the first part of the write before reaching the minSize or the end of the write.
 	ignore  bool   // If true, then we immediately passthru writes to the underlying ResponseWriter.
 
 	contentTypes []parsedContentType // Only compress if the response is one of these content-types. All are accepted if empty.
+
+	contentTypeExceptions []parsedContentType // Only compress if the response is not one of these content-types. All are accepted if empty.
 }
 
 type GzipResponseWriterWithCloseNotify struct {
@@ -118,7 +120,7 @@ func (w *GzipResponseWriter) Write(b []byte) (int, error) {
 		ce    = w.Header().Get(contentEncoding)
 	)
 	// Only continue if they didn't already choose an encoding or a known unhandled content length or type.
-	if ce == "" && (cl == 0 || cl >= w.minSize) && (ct == "" || handleContentType(w.contentTypes, ct)) {
+	if ce == "" && (cl == 0 || cl >= w.minSize) && (ct == "" || handleContentType(w.contentTypes, w.contentTypeExceptions, ct)) {
 		// If the current buffer is less than minSize and a Content-Length isn't set, then wait until we have more data.
 		if len(w.buf) < w.minSize && cl == 0 {
 			return len(b), nil
@@ -131,7 +133,7 @@ func (w *GzipResponseWriter) Write(b []byte) (int, error) {
 				w.Header().Set(contentType, ct)
 			}
 			// If the Content-Type is acceptable to GZIP, initialize the GZIP writer.
-			if handleContentType(w.contentTypes, ct) {
+			if handleContentType(w.contentTypes, w.contentTypeExceptions, ct) {
 				if err := w.startGzip(); err != nil {
 					return 0, err
 				}
@@ -324,10 +326,11 @@ func GzipHandlerWithOpts(opts ...option) (func(http.Handler) http.Handler, error
 			w.Header().Add(vary, acceptEncoding)
 			if acceptsGzip(r) {
 				gw := &GzipResponseWriter{
-					ResponseWriter: w,
-					index:          index,
-					minSize:        c.minSize,
-					contentTypes:   c.contentTypes,
+					ResponseWriter:        w,
+					index:                 index,
+					minSize:               c.minSize,
+					contentTypes:          c.contentTypes,
+					contentTypeExceptions: c.contentTypeExceptions,
 				}
 				defer gw.Close()
 
@@ -376,9 +379,10 @@ func (pct parsedContentType) equals(mediaType string, params map[string]string)
 
 // Used for functional configuration.
 type config struct {
-	minSize      int
-	level        int
-	contentTypes []parsedContentType
+	minSize               int
+	level                 int
+	contentTypes          []parsedContentType
+	contentTypeExceptions []parsedContentType
 }
 
 func (c *config) validate() error {
@@ -386,6 +390,10 @@ func (c *config) validate() error {
 		return fmt.Errorf("invalid compression level requested: %d", c.level)
 	}
 
+	if len(c.contentTypes) > 0 && len(c.contentTypeExceptions) > 0 {
+		return fmt.Errorf("ContentTypes and ContentTypeExceptions are mutually exclusive")
+	}
+
 	if c.minSize < 0 {
 		return fmt.Errorf("minimum size must be more than zero")
 	}
@@ -411,6 +419,9 @@ func CompressionLevel(level int) option {
 // the Content-Type header to before compressing. If none
 // match, the response will be returned as-is.
 //
+// ContentTypes cannot be used with ContentTypeExceptions, the options
+// are mutually exclusive.
+//
 // Content types are compared in a case-insensitive, whitespace-ignored
 // manner.
 //
@@ -437,6 +448,39 @@ func ContentTypes(types []string) option {
 	}
 }
 
+// ContentTypeExceptions specifies a list of content types to compare
+// the Content-Type header to before compressing. If any
+// match, the response will be returned as-is.
+//
+// Content types are compared in a case-insensitive, whitespace-ignored
+// manner.
+//
+// ContentTypeExceptions cannot be used with ContentTypes, the options
+// are mutually exclusive.
+//
+// A MIME type without any other directive will match a content type
+// that has the same MIME type, regardless of that content type's other
+// directives. I.e., "text/html" will match both "text/html" and
+// "text/html; charset=utf-8".
+//
+// A MIME type with any other directive will only match a content type
+// that has the same MIME type and other directives. I.e.,
+// "text/html; charset=utf-8" will only match "text/html; charset=utf-8".
+//
+// By default, responses are gzipped regardless of
+// Content-Type.
+func ContentTypeExceptions(types []string) option {
+	return func(c *config) {
+		c.contentTypeExceptions = []parsedContentType{}
+		for _, v := range types {
+			mediaType, params, err := mime.ParseMediaType(v)
+			if err == nil {
+				c.contentTypeExceptions = append(c.contentTypeExceptions, parsedContentType{mediaType, params})
+			}
+		}
+	}
+}
+
 // GzipHandler wraps an HTTP handler, to transparently gzip the response body if
 // the client supports it (via the Accept-Encoding header). This will compress at
 // the default compression level.
@@ -453,9 +497,11 @@ func acceptsGzip(r *http.Request) bool {
 }
 
 // returns true if we've been configured to compress the specific content type.
-func handleContentType(contentTypes []parsedContentType, ct string) bool {
-	// If contentTypes is empty we handle all content types.
-	if len(contentTypes) == 0 {
+func handleContentType(whitelist, blacklist []parsedContentType, ct string) bool {
+	// If whitelist and blacklist are empty we handle all content types.
+	whiteLen := len(whitelist)
+	blackLen := len(blacklist)
+	if whiteLen == 0 && blackLen == 0 {
 		return true
 	}
 
@@ -464,13 +510,24 @@ func handleContentType(contentTypes []parsedContentType, ct string) bool {
 		return false
 	}
 
-	for _, c := range contentTypes {
+	var listToCheck []parsedContentType
+	var whitelistMode bool
+
+	if whiteLen > 0 {
+		whitelistMode = true
+		listToCheck = whitelist
+	} else {
+		listToCheck = blacklist
+	}
+
+	var isInList bool
+	for _, c := range listToCheck {
 		if c.equals(mediaType, params) {
-			return true
+			isInList = true
 		}
 	}
 
-	return false
+	return (whitelistMode && isInList) || (!whitelistMode && !isInList)
 }
 
 // parseEncodings attempts to parse a list of codings, per RFC 2616, as might

vendor/modules.txt

@@ -1,9 +1,6 @@
 # github.com/Masterminds/squirrel v1.2.0
 ## explicit
 github.com/Masterminds/squirrel
-# github.com/NYTimes/gziphandler v1.1.1
-## explicit
-github.com/NYTimes/gziphandler
 # github.com/armon/go-metrics v0.3.0
 ## explicit
 github.com/armon/go-metrics
@@ -239,6 +236,9 @@ github.com/mitchellh/go-testing-interface
 # github.com/mitchellh/mapstructure v1.2.3
 ## explicit
 github.com/mitchellh/mapstructure
+# github.com/mkraft/gziphandler v1.1.2-0.20200509175700-73dc64f3ad90
+## explicit
+github.com/mkraft/gziphandler
 # github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd
 github.com/modern-go/concurrent
 # github.com/modern-go/reflect2 v1.0.1

web/handlers.go

@@ -14,7 +14,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/NYTimes/gziphandler"
+	"github.com/mkraft/gziphandler"
 	"github.com/opentracing/opentracing-go"
 	"github.com/opentracing/opentracing-go/ext"
 	spanlog "github.com/opentracing/opentracing-go/log"

web/static.go

@@ -9,16 +9,23 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/NYTimes/gziphandler"
-
 	"github.com/mattermost/mattermost-server/v5/mlog"
 	"github.com/mattermost/mattermost-server/v5/model"
 	"github.com/mattermost/mattermost-server/v5/utils"
 	"github.com/mattermost/mattermost-server/v5/utils/fileutils"
+	"github.com/mkraft/gziphandler"
 )
 
 var robotsTxt = []byte("User-agent: *\nDisallow: /\n")
 
+// the static content types that are Brotli encoded rather than gzipped.
+var brotliEncodedContent = map[string]string{
+	"js":  "application/javascript",
+	"css": "text/css",
+}
+
+var brotliContentTypes []string
+
 func (w *Web) InitStatic() {
 	if *w.ConfigService.Config().ServiceSettings.WebserverMode != "disabled" {
 		if err := utils.UpdateAssetsSubpathFromConfig(w.ConfigService.Config()); err != nil {
@@ -30,11 +37,20 @@ func (w *Web) InitStatic() {
 
 		subpath, _ := utils.GetSubpathFromConfig(w.ConfigService.Config())
 
-		staticHandler := staticFilesHandler(http.StripPrefix(path.Join(subpath, "static"), http.FileServer(http.Dir(staticDir))))
+		staticHandler := brotliFilesHandler(staticFilesHandler(http.StripPrefix(path.Join(subpath, "static"), http.FileServer(http.Dir(staticDir)))))
 		pluginHandler := staticFilesHandler(http.StripPrefix(path.Join(subpath, "static", "plugins"), http.FileServer(http.Dir(*w.ConfigService.Config().PluginSettings.ClientDirectory))))
 
 		if *w.ConfigService.Config().ServiceSettings.WebserverMode == "gzip" {
-			staticHandler = gziphandler.GzipHandler(staticHandler)
+			for _, ct := range brotliEncodedContent {
+				brotliContentTypes = append(brotliContentTypes, ct)
+			}
+
+			everythingExceptBrotliGzipHandler, err := gziphandler.GzipHandlerWithOpts(gziphandler.ContentTypeExceptions(brotliContentTypes))
+			if err != nil {
+				mlog.Error("Failed to initialize gziphandler", mlog.Err(err))
+			}
+
+			staticHandler = everythingExceptBrotliGzipHandler(staticHandler)
 			pluginHandler = gziphandler.GzipHandler(pluginHandler)
 		}
 
@@ -72,6 +88,26 @@ func root(c *Context, w http.ResponseWriter, r *http.Request) {
 	http.ServeFile(w, r, filepath.Join(staticDir, "root.html"))
 }
 
+func acceptsEncodingBrotli(r *http.Request) bool {
+	directives := strings.Fields(r.Header.Get("Accept-Encoding"))
+	for _, directive := range directives {
+		if strings.ToLower(directive) == "br" {
+			return true
+		}
+	}
+	return false
+}
+
+func requestingBrotliFileExtension(r *http.Request) (bool, string) {
+	extension := r.URL.Path[strings.LastIndex(r.URL.Path, ".")+1:]
+	for bx, ct := range brotliEncodedContent {
+		if bx == extension {
+			return true, ct
+		}
+	}
+	return false, ""
+}
+
 func staticFilesHandler(handler http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		//wrap our ResponseWriter with our no-cache 404-handler
@@ -83,6 +119,20 @@ func staticFilesHandler(handler http.Handler) http.Handler {
 			http.NotFound(w, r)
 			return
 		}
+
+		handler.ServeHTTP(w, r)
+	})
+}
+
+func brotliFilesHandler(handler http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		isRequestingBrotliFile, contentType := requestingBrotliFileExtension(r)
+		if isRequestingBrotliFile && acceptsEncodingBrotli(r) {
+			r.URL.Path = r.URL.Path + ".br"
+			w.Header().Set("Content-Encoding", "br")
+			w.Header().Set("Content-Type", contentType)
+		}
+
 		handler.ServeHTTP(w, r)
 	})
 }

web/static_test.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package web
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+var tests = []struct {
+	requestURL            string
+	requestContentType    string
+	requestAcceptEncoding []string
+	expectBrotli          bool
+}{
+	{"http://test.com/foo.js", "application/javascript", []string{"br"}, true},
+	{"http://test.com/foo.css", "text/css", []string{"br"}, true},
+	{"http://test.com/foo.jss", "text/plain; charset=utf-8", []string{"gzip"}, false},
+	{"http://test.com/foo.css", "text/plain; charset=utf-8", []string{"gzip"}, false},
+	{"http://test.com/foo.jsx", "text/plain; charset=utf-8", []string{"br"}, false},
+	{"http://test.com/foo.xcss", "text/plain; charset=utf-8", []string{"gzip"}, false},
+}
+
+type mockHandler struct{}
+
+func (mh mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	fmt.Fprintf(w, "hello")
+}
+
+func TestBrotliFilesHandler(t *testing.T) {
+	for _, tt := range tests {
+		t.Run(fmt.Sprintf("%v", tt), func(t *testing.T) {
+
+			req := httptest.NewRequest("GET", tt.requestURL, nil)
+			req.Header.Set("Accept-Encoding", strings.Join(tt.requestAcceptEncoding, ", "))
+			w := httptest.NewRecorder()
+
+			handler := brotliFilesHandler(mockHandler{})
+			handler.ServeHTTP(w, req)
+
+			resp := w.Result()
+
+			require.Equal(t, tt.expectBrotli, resp.Header.Get("Content-Encoding") == "br")
+			if tt.expectBrotli {
+				require.Equal(t, tt.requestURL+".br", req.URL.String())
+			} else {
+				require.Equal(t, tt.requestURL, req.URL.String())
+			}
+			require.Equal(t, tt.requestContentType, resp.Header.Get("Content-Type"))
+		})
+	}
+}