diff --git a/api4/handlers.go b/api4/handlers.go index 942d851d93..17486b0228 100644 --- a/api4/handlers.go +++ b/api4/handlers.go @@ -6,8 +6,8 @@ package api4 import ( "net/http" - "github.com/NYTimes/gziphandler" "github.com/mattermost/mattermost-server/v5/web" + "github.com/mkraft/gziphandler" ) type Context = web.Context diff --git a/go.mod b/go.mod index bb0eeaa77f..b964c3300c 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.14 require ( github.com/Masterminds/squirrel v1.2.0 - github.com/NYTimes/gziphandler v1.1.1 github.com/armon/go-metrics v0.3.0 // indirect github.com/avct/uasurfer v0.0.0-20191028135549-26b5daa857f1 github.com/beevik/etree v1.1.0 // indirect @@ -63,6 +62,7 @@ require ( github.com/minio/minio-go/v6 v6.0.55 github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/mapstructure v1.2.3 // indirect + github.com/mkraft/gziphandler v1.1.2-0.20200509175700-73dc64f3ad90 github.com/muesli/smartcrop v0.3.0 // indirect github.com/oklog/run v1.1.0 // indirect github.com/olekukonko/tablewriter v0.0.4 // indirect @@ -75,7 +75,7 @@ require ( github.com/prometheus/procfs v0.0.11 // indirect github.com/rs/cors v1.7.0 github.com/rudderlabs/analytics-go v3.2.1+incompatible - github.com/russellhaering/goxmldsig v0.0.0-20180430223755-7acd5e4a6ef7 // indirect + github.com/russellhaering/goxmldsig v0.0.0-20180430223755-7acd5e4a6ef7 github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd github.com/segmentio/analytics-go v3.1.0+incompatible github.com/segmentio/backo-go v0.0.0-20200129164019-23eae7c10bd3 // indirect diff --git a/go.sum b/go.sum index 8f7b999645..53815d3748 100644 --- a/go.sum +++ b/go.sum @@ -20,8 +20,6 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0 github.com/Masterminds/squirrel v1.2.0 h1:K1NhbTO21BWG47IVR0OnIZuE0LZcXAYqywrC3Ko53KI= github.com/Masterminds/squirrel v1.2.0/go.mod h1:yaPeOnPG5ZRwL9oKdTsO/prlkPbXWZlRVMQ/gGlzIuA= github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= -github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PaulARoy/azurestoragecache v0.0.0-20170906084534-3c249a3ba788/go.mod h1:lY1dZd8HBzJ10eqKERHn3CU59tfhzcAVb2c0ZhIWSOk= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= @@ -358,6 +356,12 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.2.3 h1:f/MjBEBDLttYCGfRaKBbKSRVF5aV2O6fnBpzknuE3jU= github.com/mitchellh/mapstructure v1.2.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mkraft/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= +github.com/mkraft/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/mkraft/gziphandler v1.1.2-0.20200509170533-f387ff5f65bc h1:S6AKguh0+X1d0bW9+catK3n+PMX/dke1lkL4uxxukd8= +github.com/mkraft/gziphandler v1.1.2-0.20200509170533-f387ff5f65bc/go.mod h1:gG8WEPb2aI5MHdmHv83au7bk3molRSZiAjdxYrEMJdQ= +github.com/mkraft/gziphandler v1.1.2-0.20200509175700-73dc64f3ad90 h1:qEm+0lDAcdszJkCnYSuP1oITiQf0TzDIAmeD2PlBtZU= +github.com/mkraft/gziphandler v1.1.2-0.20200509175700-73dc64f3ad90/go.mod h1:gG8WEPb2aI5MHdmHv83au7bk3molRSZiAjdxYrEMJdQ= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= diff --git a/vendor/github.com/NYTimes/gziphandler/.gitignore b/vendor/github.com/mkraft/gziphandler/.gitignore similarity index 100% rename from vendor/github.com/NYTimes/gziphandler/.gitignore rename to vendor/github.com/mkraft/gziphandler/.gitignore diff --git a/vendor/github.com/NYTimes/gziphandler/.travis.yml b/vendor/github.com/mkraft/gziphandler/.travis.yml similarity index 100% rename from vendor/github.com/NYTimes/gziphandler/.travis.yml rename to vendor/github.com/mkraft/gziphandler/.travis.yml diff --git a/vendor/github.com/NYTimes/gziphandler/CODE_OF_CONDUCT.md b/vendor/github.com/mkraft/gziphandler/CODE_OF_CONDUCT.md similarity index 100% rename from vendor/github.com/NYTimes/gziphandler/CODE_OF_CONDUCT.md rename to vendor/github.com/mkraft/gziphandler/CODE_OF_CONDUCT.md diff --git a/vendor/github.com/NYTimes/gziphandler/CONTRIBUTING.md b/vendor/github.com/mkraft/gziphandler/CONTRIBUTING.md similarity index 100% rename from vendor/github.com/NYTimes/gziphandler/CONTRIBUTING.md rename to vendor/github.com/mkraft/gziphandler/CONTRIBUTING.md diff --git a/vendor/github.com/NYTimes/gziphandler/LICENSE b/vendor/github.com/mkraft/gziphandler/LICENSE similarity index 100% rename from vendor/github.com/NYTimes/gziphandler/LICENSE rename to vendor/github.com/mkraft/gziphandler/LICENSE diff --git a/vendor/github.com/NYTimes/gziphandler/README.md b/vendor/github.com/mkraft/gziphandler/README.md similarity index 100% rename from vendor/github.com/NYTimes/gziphandler/README.md rename to vendor/github.com/mkraft/gziphandler/README.md diff --git a/vendor/github.com/NYTimes/gziphandler/go.mod b/vendor/github.com/mkraft/gziphandler/go.mod similarity index 58% rename from vendor/github.com/NYTimes/gziphandler/go.mod rename to vendor/github.com/mkraft/gziphandler/go.mod index 8019012742..359f88db3d 100644 --- a/vendor/github.com/NYTimes/gziphandler/go.mod +++ b/vendor/github.com/mkraft/gziphandler/go.mod @@ -1,4 +1,4 @@ -module github.com/NYTimes/gziphandler +module github.com/mkraft/gziphandler go 1.11 diff --git a/vendor/github.com/NYTimes/gziphandler/go.sum b/vendor/github.com/mkraft/gziphandler/go.sum similarity index 100% rename from vendor/github.com/NYTimes/gziphandler/go.sum rename to vendor/github.com/mkraft/gziphandler/go.sum diff --git a/vendor/github.com/NYTimes/gziphandler/gzip.go b/vendor/github.com/mkraft/gziphandler/gzip.go similarity index 84% rename from vendor/github.com/NYTimes/gziphandler/gzip.go rename to vendor/github.com/mkraft/gziphandler/gzip.go index c112bbdf81..99afebc98f 100644 --- a/vendor/github.com/NYTimes/gziphandler/gzip.go +++ b/vendor/github.com/mkraft/gziphandler/gzip.go @@ -1,4 +1,4 @@ -package gziphandler // import "github.com/NYTimes/gziphandler" +package gziphandler // import "github.com/mkraft/gziphandler" import ( "bufio" @@ -81,11 +81,13 @@ type GzipResponseWriter struct { code int // Saves the WriteHeader value. - minSize int // Specifed the minimum response size to gzip. If the response length is bigger than this value, it is compressed. + minSize int // Specifies the minimum response size to gzip. If the response length is bigger than this value, it is compressed. buf []byte // Holds the first part of the write before reaching the minSize or the end of the write. ignore bool // If true, then we immediately passthru writes to the underlying ResponseWriter. contentTypes []parsedContentType // Only compress if the response is one of these content-types. All are accepted if empty. + + contentTypeExceptions []parsedContentType // Only compress if the response is not one of these content-types. All are accepted if empty. } type GzipResponseWriterWithCloseNotify struct { @@ -118,7 +120,7 @@ func (w *GzipResponseWriter) Write(b []byte) (int, error) { ce = w.Header().Get(contentEncoding) ) // Only continue if they didn't already choose an encoding or a known unhandled content length or type. - if ce == "" && (cl == 0 || cl >= w.minSize) && (ct == "" || handleContentType(w.contentTypes, ct)) { + if ce == "" && (cl == 0 || cl >= w.minSize) && (ct == "" || handleContentType(w.contentTypes, w.contentTypeExceptions, ct)) { // If the current buffer is less than minSize and a Content-Length isn't set, then wait until we have more data. if len(w.buf) < w.minSize && cl == 0 { return len(b), nil @@ -131,7 +133,7 @@ func (w *GzipResponseWriter) Write(b []byte) (int, error) { w.Header().Set(contentType, ct) } // If the Content-Type is acceptable to GZIP, initialize the GZIP writer. - if handleContentType(w.contentTypes, ct) { + if handleContentType(w.contentTypes, w.contentTypeExceptions, ct) { if err := w.startGzip(); err != nil { return 0, err } @@ -324,10 +326,11 @@ func GzipHandlerWithOpts(opts ...option) (func(http.Handler) http.Handler, error w.Header().Add(vary, acceptEncoding) if acceptsGzip(r) { gw := &GzipResponseWriter{ - ResponseWriter: w, - index: index, - minSize: c.minSize, - contentTypes: c.contentTypes, + ResponseWriter: w, + index: index, + minSize: c.minSize, + contentTypes: c.contentTypes, + contentTypeExceptions: c.contentTypeExceptions, } defer gw.Close() @@ -376,9 +379,10 @@ func (pct parsedContentType) equals(mediaType string, params map[string]string) // Used for functional configuration. type config struct { - minSize int - level int - contentTypes []parsedContentType + minSize int + level int + contentTypes []parsedContentType + contentTypeExceptions []parsedContentType } func (c *config) validate() error { @@ -386,6 +390,10 @@ func (c *config) validate() error { return fmt.Errorf("invalid compression level requested: %d", c.level) } + if len(c.contentTypes) > 0 && len(c.contentTypeExceptions) > 0 { + return fmt.Errorf("ContentTypes and ContentTypeExceptions are mutually exclusive") + } + if c.minSize < 0 { return fmt.Errorf("minimum size must be more than zero") } @@ -411,6 +419,9 @@ func CompressionLevel(level int) option { // the Content-Type header to before compressing. If none // match, the response will be returned as-is. // +// ContentTypes cannot be used with ContentTypeExceptions, the options +// are mutually exclusive. +// // Content types are compared in a case-insensitive, whitespace-ignored // manner. // @@ -437,6 +448,39 @@ func ContentTypes(types []string) option { } } +// ContentTypeExceptions specifies a list of content types to compare +// the Content-Type header to before compressing. If any +// match, the response will be returned as-is. +// +// Content types are compared in a case-insensitive, whitespace-ignored +// manner. +// +// ContentTypeExceptions cannot be used with ContentTypes, the options +// are mutually exclusive. +// +// A MIME type without any other directive will match a content type +// that has the same MIME type, regardless of that content type's other +// directives. I.e., "text/html" will match both "text/html" and +// "text/html; charset=utf-8". +// +// A MIME type with any other directive will only match a content type +// that has the same MIME type and other directives. I.e., +// "text/html; charset=utf-8" will only match "text/html; charset=utf-8". +// +// By default, responses are gzipped regardless of +// Content-Type. +func ContentTypeExceptions(types []string) option { + return func(c *config) { + c.contentTypeExceptions = []parsedContentType{} + for _, v := range types { + mediaType, params, err := mime.ParseMediaType(v) + if err == nil { + c.contentTypeExceptions = append(c.contentTypeExceptions, parsedContentType{mediaType, params}) + } + } + } +} + // GzipHandler wraps an HTTP handler, to transparently gzip the response body if // the client supports it (via the Accept-Encoding header). This will compress at // the default compression level. @@ -453,9 +497,11 @@ func acceptsGzip(r *http.Request) bool { } // returns true if we've been configured to compress the specific content type. -func handleContentType(contentTypes []parsedContentType, ct string) bool { - // If contentTypes is empty we handle all content types. - if len(contentTypes) == 0 { +func handleContentType(whitelist, blacklist []parsedContentType, ct string) bool { + // If whitelist and blacklist are empty we handle all content types. + whiteLen := len(whitelist) + blackLen := len(blacklist) + if whiteLen == 0 && blackLen == 0 { return true } @@ -464,13 +510,24 @@ func handleContentType(contentTypes []parsedContentType, ct string) bool { return false } - for _, c := range contentTypes { + var listToCheck []parsedContentType + var whitelistMode bool + + if whiteLen > 0 { + whitelistMode = true + listToCheck = whitelist + } else { + listToCheck = blacklist + } + + var isInList bool + for _, c := range listToCheck { if c.equals(mediaType, params) { - return true + isInList = true } } - return false + return (whitelistMode && isInList) || (!whitelistMode && !isInList) } // parseEncodings attempts to parse a list of codings, per RFC 2616, as might diff --git a/vendor/github.com/NYTimes/gziphandler/gzip_go18.go b/vendor/github.com/mkraft/gziphandler/gzip_go18.go similarity index 100% rename from vendor/github.com/NYTimes/gziphandler/gzip_go18.go rename to vendor/github.com/mkraft/gziphandler/gzip_go18.go diff --git a/vendor/modules.txt b/vendor/modules.txt index 1053487f85..54b6e4c780 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,9 +1,6 @@ # github.com/Masterminds/squirrel v1.2.0 ## explicit github.com/Masterminds/squirrel -# github.com/NYTimes/gziphandler v1.1.1 -## explicit -github.com/NYTimes/gziphandler # github.com/armon/go-metrics v0.3.0 ## explicit github.com/armon/go-metrics @@ -239,6 +236,9 @@ github.com/mitchellh/go-testing-interface # github.com/mitchellh/mapstructure v1.2.3 ## explicit github.com/mitchellh/mapstructure +# github.com/mkraft/gziphandler v1.1.2-0.20200509175700-73dc64f3ad90 +## explicit +github.com/mkraft/gziphandler # github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd github.com/modern-go/concurrent # github.com/modern-go/reflect2 v1.0.1 diff --git a/web/handlers.go b/web/handlers.go index 702120756f..4c09286504 100644 --- a/web/handlers.go +++ b/web/handlers.go @@ -14,7 +14,7 @@ import ( "strings" "time" - "github.com/NYTimes/gziphandler" + "github.com/mkraft/gziphandler" "github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go/ext" spanlog "github.com/opentracing/opentracing-go/log" diff --git a/web/static.go b/web/static.go index ae41c8319c..2c9120cbda 100644 --- a/web/static.go +++ b/web/static.go @@ -9,16 +9,23 @@ import ( "path/filepath" "strings" - "github.com/NYTimes/gziphandler" - "github.com/mattermost/mattermost-server/v5/mlog" "github.com/mattermost/mattermost-server/v5/model" "github.com/mattermost/mattermost-server/v5/utils" "github.com/mattermost/mattermost-server/v5/utils/fileutils" + "github.com/mkraft/gziphandler" ) var robotsTxt = []byte("User-agent: *\nDisallow: /\n") +// the static content types that are Brotli encoded rather than gzipped. +var brotliEncodedContent = map[string]string{ + "js": "application/javascript", + "css": "text/css", +} + +var brotliContentTypes []string + func (w *Web) InitStatic() { if *w.ConfigService.Config().ServiceSettings.WebserverMode != "disabled" { if err := utils.UpdateAssetsSubpathFromConfig(w.ConfigService.Config()); err != nil { @@ -30,11 +37,20 @@ func (w *Web) InitStatic() { subpath, _ := utils.GetSubpathFromConfig(w.ConfigService.Config()) - staticHandler := staticFilesHandler(http.StripPrefix(path.Join(subpath, "static"), http.FileServer(http.Dir(staticDir)))) + staticHandler := brotliFilesHandler(staticFilesHandler(http.StripPrefix(path.Join(subpath, "static"), http.FileServer(http.Dir(staticDir))))) pluginHandler := staticFilesHandler(http.StripPrefix(path.Join(subpath, "static", "plugins"), http.FileServer(http.Dir(*w.ConfigService.Config().PluginSettings.ClientDirectory)))) if *w.ConfigService.Config().ServiceSettings.WebserverMode == "gzip" { - staticHandler = gziphandler.GzipHandler(staticHandler) + for _, ct := range brotliEncodedContent { + brotliContentTypes = append(brotliContentTypes, ct) + } + + everythingExceptBrotliGzipHandler, err := gziphandler.GzipHandlerWithOpts(gziphandler.ContentTypeExceptions(brotliContentTypes)) + if err != nil { + mlog.Error("Failed to initialize gziphandler", mlog.Err(err)) + } + + staticHandler = everythingExceptBrotliGzipHandler(staticHandler) pluginHandler = gziphandler.GzipHandler(pluginHandler) } @@ -72,6 +88,26 @@ func root(c *Context, w http.ResponseWriter, r *http.Request) { http.ServeFile(w, r, filepath.Join(staticDir, "root.html")) } +func acceptsEncodingBrotli(r *http.Request) bool { + directives := strings.Fields(r.Header.Get("Accept-Encoding")) + for _, directive := range directives { + if strings.ToLower(directive) == "br" { + return true + } + } + return false +} + +func requestingBrotliFileExtension(r *http.Request) (bool, string) { + extension := r.URL.Path[strings.LastIndex(r.URL.Path, ".")+1:] + for bx, ct := range brotliEncodedContent { + if bx == extension { + return true, ct + } + } + return false, "" +} + func staticFilesHandler(handler http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { //wrap our ResponseWriter with our no-cache 404-handler @@ -83,6 +119,20 @@ func staticFilesHandler(handler http.Handler) http.Handler { http.NotFound(w, r) return } + + handler.ServeHTTP(w, r) + }) +} + +func brotliFilesHandler(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + isRequestingBrotliFile, contentType := requestingBrotliFileExtension(r) + if isRequestingBrotliFile && acceptsEncodingBrotli(r) { + r.URL.Path = r.URL.Path + ".br" + w.Header().Set("Content-Encoding", "br") + w.Header().Set("Content-Type", contentType) + } + handler.ServeHTTP(w, r) }) } diff --git a/web/static_test.go b/web/static_test.go new file mode 100644 index 0000000000..12c9f994bd --- /dev/null +++ b/web/static_test.go @@ -0,0 +1,58 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package web + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +var tests = []struct { + requestURL string + requestContentType string + requestAcceptEncoding []string + expectBrotli bool +}{ + {"http://test.com/foo.js", "application/javascript", []string{"br"}, true}, + {"http://test.com/foo.css", "text/css", []string{"br"}, true}, + {"http://test.com/foo.jss", "text/plain; charset=utf-8", []string{"gzip"}, false}, + {"http://test.com/foo.css", "text/plain; charset=utf-8", []string{"gzip"}, false}, + {"http://test.com/foo.jsx", "text/plain; charset=utf-8", []string{"br"}, false}, + {"http://test.com/foo.xcss", "text/plain; charset=utf-8", []string{"gzip"}, false}, +} + +type mockHandler struct{} + +func (mh mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "hello") +} + +func TestBrotliFilesHandler(t *testing.T) { + for _, tt := range tests { + t.Run(fmt.Sprintf("%v", tt), func(t *testing.T) { + + req := httptest.NewRequest("GET", tt.requestURL, nil) + req.Header.Set("Accept-Encoding", strings.Join(tt.requestAcceptEncoding, ", ")) + w := httptest.NewRecorder() + + handler := brotliFilesHandler(mockHandler{}) + handler.ServeHTTP(w, req) + + resp := w.Result() + + require.Equal(t, tt.expectBrotli, resp.Header.Get("Content-Encoding") == "br") + if tt.expectBrotli { + require.Equal(t, tt.requestURL+".br", req.URL.String()) + } else { + require.Equal(t, tt.requestURL, req.URL.String()) + } + require.Equal(t, tt.requestContentType, resp.Header.Get("Content-Type")) + }) + } +}