Document extractor service (#15665)

* Document extractor service

* Fixing vendor modules

* Addressing PR Review comments

* Some small simplifications

* Fixing a linter complain

* simplifying a bit the code using package variables

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

vendor/github.com/JalfResi/justext/.gitignore

@@ -0,0 +1,32 @@
+
+src/_go_.6
+
+src/_obj/ourscienceistight/gojustext.a
+
+src/example/_go_.6
+
+src/example/gojustext
+
+src/main.go_old
+
+src/.DS_Store
+
+src/stoplists/.DS_Store
+
+src/example/*.html
+
+src/example/*.htm
+
+src/stoplists/*.txt
+
+src/example/hp-envy-14-spectre-review
+
+src/example/t.txt
+
+src/example/t2.txt
+
+gojustext
+
+gojustext.sublime-project
+
+gojustext.sublime-workspace

vendor/github.com/JalfResi/justext/README.md

@@ -0,0 +1,48 @@
+justext
+=======
+
+A Go package that implements the JusText boilerplate removal algorithm (http://code.google.com/p/justext/)
+
+## Install
+
+    go get github.com/JalfResi/justext
+
+And import:
+
+    import "github.com/JalfResi/justext"
+
+## Usage
+
+Supports all stoplist files available at http://code.google.com/p/justext/source/browse/#svn%2Ftrunk%2Fjustext%2Fstoplists
+
+Justext expects valid HTML; it is your responsability to ensure that valid HTML is passed to Justext. To make things easier 
+I have written a CGO wrapper around libtidy which you can find here: [github.com/JalfResi/GoTidy](https://github.com/JalfResi/GoTidy)
+In the future, once exp/html is part of the standard packages I will refactor JusText to accept only valid HTML documents/strings.
+
+Justext use the reader-writer idiom, alowing you to setup the reader with a common configuration and just pump out 
+articles to the writer.
+
+Example usage:
+
+    // Create a justext reader from another reader
+    reader := justext.NewReader(os.Stdin)
+    
+    // Configure the reader
+    reader.LengthLow = 70
+    reader.LengthHigh = 200
+    reader.Stoplist = stoplist // The stoplist map[string]bool
+    reader.StopwordsLow = 0.3
+    reader.StopwordsHigh = 0.32
+    reader.MaxLinkDensity = 0.2
+    reader.MaxHeadingDistance = 200
+    reader.NoHeadings = false
+    
+    // Read from the reader to generate a paragraph set
+    paragraphSet, _ := reader.ReadAll()
+    
+    // Create a writer from another writer
+    writer := justext.NewWriter(os.Stdout)
+    // Write the paragraph set to the writer
+    writer.WriteAll(paragraphSet)
+    
+    

vendor/github.com/JalfResi/justext/TODO.txt

@@ -0,0 +1,3 @@
+TODO
+====
+- Stoplists need to be separtae subpackages (use the init() package method of registration)

vendor/github.com/JalfResi/justext/classifyParagraphs.go

@@ -0,0 +1,66 @@
+package justext
+
+import (
+	"regexp"
+	"strings"
+)
+
+var findHeadings *regexp.Regexp = regexp.MustCompile("(^h[123456]|.h[123456])")
+var copyrightChar *regexp.Regexp = regexp.MustCompile("(\u0161|&copy)")
+var findSelect *regexp.Regexp = regexp.MustCompile("(^select|.select)")
+
+func classifyParagraphs(paragraphs []*Paragraph, stoplist map[string]bool, lengthLow int, lengthHigh int, stopwordsLow float64, stopwordsHigh float64, maxLinkDensity float64, noHeadings bool) {
+	for _, paragraph := range paragraphs {
+		var length int = len(paragraph.Text)
+		var stopwordCount int = 0
+		for _, word := range strings.Split(paragraph.Text, " ") {
+			if _, ok := stoplist[word]; ok {
+				stopwordCount += 1
+			}
+		}
+
+		var stopwordDensity float64 = 0.0
+		var linkDensity float64 = 0.0
+		var wordCount int = paragraph.WordCount
+
+		if wordCount > 0 {
+			stopwordDensity = 1.0 * float64(stopwordCount) / float64(wordCount)
+			linkDensity = float64(paragraph.LinkedCharCount) / float64(length)
+		}
+
+		paragraph.StopwordCount = stopwordCount
+		paragraph.StopwordDensity = stopwordDensity
+		paragraph.LinkDensity = linkDensity
+		paragraph.Heading = bool(!noHeadings && findHeadings.MatchString(paragraph.DomPath))
+
+		if linkDensity > maxLinkDensity {
+			paragraph.CfClass = "bad"
+		} else if copyrightChar.MatchString(paragraph.Text) {
+			paragraph.CfClass = "bad"
+		} else if findSelect.MatchString(paragraph.DomPath) {
+			paragraph.CfClass = "bad"
+		} else {
+			if length < lengthLow {
+				if paragraph.LinkedCharCount > 0 {
+					paragraph.CfClass = "bad"
+				} else {
+					paragraph.CfClass = "short"
+				}
+			} else {
+				if stopwordDensity >= stopwordsHigh {
+					if length > lengthHigh {
+						paragraph.CfClass = "good"
+					} else {
+						paragraph.CfClass = "neargood"
+					}
+				} else {
+					if stopwordDensity >= stopwordsLow {
+						paragraph.CfClass = "neargood"
+					} else {
+						paragraph.CfClass = "bad"
+					}
+				}
+			}
+		}
+	}
+}

vendor/github.com/JalfResi/justext/defaultTemplate.go

@@ -0,0 +1,50 @@
+package justext
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"reflect"
+	"unsafe"
+)
+
+var _DefaultTemplate = "" +
+	"\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x74\x90\x3d\x6e\xc4\x20" +
+	"\x10\x46\x6b\xfb\x14\xc8\x4a\x8d\xfb\x08\x53\x24\x45\x92\x26\x8a" +
+	"\x14\x5f\x60\x6c\x88\x41\xc2\x80\x80\x22\x2b\xc4\xdd\x17\xc3\x7a" +
+	"\xff\xb4\x5b\x81\xde\x3c\xe6\x1b\x86\x88\xb0\x2a\xda\x92\xc9\xb0" +
+	"\x03\x6d\x63\x7c\xd1\xe6\xcd\x48\xc5\x9d\x55\x10\x38\x7a\x1d\x10" +
+	"\xfe\xbe\x26\x29\x65\xc9\x81\x5e\x38\xc2\x3f\xe0\x60\x71\x60\x85" +
+	"\xcf\xb4\x89\x51\xfe\xa1\x2f\xff\x61\x0c\x43\xf8\x5d\x81\x2f\xb4" +
+	"\x62\xfc\xc9\x81\x49\xbd\x14\xd2\x10\x41\x63\x1c\x9d\x5c\x7f\x2d" +
+	"\xcc\xb9\xcf\xc8\xff\x43\x4a\xa4\x17\xb4\xf8\x5c\x79\x7e\x12\xed" +
+	"\x63\xd1\xd2\x5c\xac\xae\x66\x35\xfb\xfc\xa8\xe4\x69\x13\xd0\xed" +
+	"\x4f\xf6\x86\x68\xde\x26\x1b\xba\xe9\x52\xea\x9e\x85\xdc\x25\x94" +
+	"\xcb\x7e\x92\xbe\x6e\x2c\x4f\xbd\x2d\xf0\x18\x00\x00\xff\xff\x2c" +
+	"\xc5\xf5\x5d\x47\x01\x00\x00"
+
+// DefaultTemplate returns the binary data for a given file.
+func DefaultTemplate() []byte {
+	// This bit of black magic ensures we do not get
+	// unneccesary memcpy's and can read directly from
+	// the .rodata section.
+	var empty [0]byte
+	sx := (*reflect.StringHeader)(unsafe.Pointer(&_DefaultTemplate))
+	b := empty[:]
+	bx := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	bx.Data = sx.Data
+	bx.Len = len(_DefaultTemplate)
+	bx.Cap = bx.Len
+
+	gz, err := gzip.NewReader(bytes.NewBuffer(b))
+
+	if err != nil {
+		panic("Decompression failed: " + err.Error())
+	}
+
+	var buf bytes.Buffer
+	io.Copy(&buf, gz)
+	gz.Close()
+
+	return buf.Bytes()
+}

vendor/github.com/JalfResi/justext/detailedTemplate.go

@@ -0,0 +1,181 @@
+package justext
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"reflect"
+	"unsafe"
+)
+
+var _DetailedTemplate = "" +
+	"\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\xe4\x59\x59\x73\xe2\x3a" +
+	"\xf6\x7f\x4e\x3e\x85\xff\xfc\x6f\x4d\x27\x45\x9a\x1d\x02\x09\x64" +
+	"\xc6\x18\x13\x76\x02\x06\x02\x99\x9a\xea\x12\xb6\x6c\x14\xaf\xb1" +
+	"\x65\x8c\xc9\xf0\xdd\x47\x32\x66\x0b\x24\x4d\xcf\x43\xdf\x5b\x35" +
+	"\xbc\xd8\x3e\x3a\xfb\xf9\x49\x3a\x12\xc5\x19\xd6\xb5\x87\xcb\xa2" +
+	"\x83\x7d\x0d\x3e\x5c\x5e\xfc\x03\xe9\x96\x69\x63\xc6\xb5\xb5\xab" +
+	"\x19\xc6\xd6\x5d\x3c\x2e\x9b\x06\x76\x62\x8a\x69\x2a\x1a\x04\x16" +
+	"\x72\x62\xa2\xa9\xc7\x45\xc7\xf9\xbb\x0c\x74\xa4\xf9\xa5\xae\x05" +
+	"\x8d\xa8\x00\x0c\xe7\x2e\x93\x48\xdc\xa4\x13\x89\xbf\x39\xee\xd4" +
+	"\x81\xb8\xa4\x01\x8c\x8c\x1b\xd1\xb7\x91\xa6\x21\xf1\x3b\x5c\xe0" +
+	"\xed\xc7\x8d\x62\x43\xa8\x06\xa4\x39\x82\xd8\x00\x3a\x74\xe0\x4d" +
+	"\x20\x40\x89\xd7\xf7\x97\x17\x97\x17\x53\x53\xf2\x2f\x2f\xde\x2f" +
+	"\x2f\x2e\xa8\x0b\xdf\xd7\xe6\xee\x98\x6f\xd4\x20\x43\x0d\x7e\xbb" +
+	"\x61\x1c\xf2\xf8\xee\x40\x1b\xc9\xf7\x1b\x3e\x0f\x22\x65\x86\xef" +
+	"\x18\xe2\xc9\x3d\x43\x88\x3a\xb0\x15\xa2\x56\x83\x32\x21\x02\x17" +
+	"\x9b\xf7\x3b\xa2\xbd\x66\xdd\x50\x3d\x24\xe1\xd9\x1d\x93\x4f\x24" +
+	"\xac\x05\xf9\x5e\x5d\x5e\x5e\x38\x16\x30\x62\x0e\x36\x2d\xcf\xb4" +
+	"\x25\x86\x3a\x83\x89\x83\xdf\x25\x28\x9a\x36\x71\xd7\x34\xee\x18" +
+	"\xd7\x90\xa0\xad\x21\x03\x86\x22\x16\x4d\x96\x74\xc3\x58\xb1\x19" +
+	"\x04\x12\x32\x14\xfa\x3a\x05\x81\x34\x43\x7e\xa1\x95\x6c\x76\x6d" +
+	"\x85\x92\x02\x9d\x40\x43\x0a\x51\xf7\xea\x3a\x18\xc9\x7e\x38\x12" +
+	"\x3a\x4a\x1c\xb8\x63\x52\x5b\xfe\x90\x3a\x35\x31\x36\xf5\xfd\x01" +
+	"\x0b\x48\xd4\x62\x18\x6d\xfa\x88\x1e\x06\x9c\xde\xc6\x77\xe4\xec" +
+	"\xc6\xcb\x29\x10\x55\xc5\x36\x49\x70\xdf\x45\x53\x33\xed\x3b\xe6" +
+	"\xff\x45\x59\xdc\x4a\xed\x71\x1f\xa6\x3d\x13\xa6\x3d\x20\x3a\x68" +
+	"\x09\xef\x98\x64\x2c\x0d\xf5\xad\xe4\x5e\x26\x4e\xd8\x90\x25\x29" +
+	"\xe4\x94\xd0\x3c\x66\x01\x1b\x28\x36\xb0\x66\x3f\x24\x88\x01\xd2" +
+	"\x9c\x0f\x39\x4c\x65\x76\x39\x94\x35\x13\x10\xfb\x41\x84\x21\x49" +
+	"\x42\x8e\xa5\x01\x82\x19\xc3\xdc\x56\xe7\xb4\x5a\x0c\xa6\x1a\xdc" +
+	"\xba\x45\x4a\x0d\x6d\xea\x92\x06\x2c\x87\x04\xb0\x79\x3b\x43\x05" +
+	"\xde\x06\x77\x58\x8a\xe3\x12\x85\xa5\xd8\x0d\xac\xcd\x92\x74\x59" +
+	"\x0b\xc6\x31\x35\x24\x91\x6c\xc8\xf2\x57\x36\x83\xda\xad\x0d\xdf" +
+	"\x7c\xc6\xb2\x29\xd4\x89\x08\x0f\x4c\x15\xe4\xc2\x97\xa6\x68\xd9" +
+	"\x7e\xa6\x43\x06\xe0\x4c\x77\x19\x6c\x9f\xe9\x31\xb6\xbf\x40\xcb" +
+	"\xf9\x4e\x7f\xa9\x66\xe7\x77\x0c\xfa\xd0\x24\xeb\x4b\xb8\xee\xac" +
+	"\x21\x45\x2b\x78\xff\x71\x25\xf9\x9e\x24\x40\xdf\x11\x83\x09\xba" +
+	"\x9e\x56\x17\x5b\xd8\x4d\x35\x53\x54\xf7\x16\x96\x74\x72\xcd\x30" +
+	"\x0b\xe7\x4a\x2a\xb3\xfe\xde\x73\xc9\x86\x16\xa4\x36\x0d\x33\x7c" +
+	"\xfd\x30\x8e\x74\xa0\x10\x44\xd2\xf5\x59\x02\x18\xdc\x05\xdf\x71" +
+	"\xcb\x50\xee\xa7\xc0\x81\xb9\xcc\x0d\x1a\x95\xbb\x7d\x2f\xd1\x7c" +
+	"\x54\x4c\x96\xfc\x3a\xc2\x70\xc6\x0f\x15\xf2\x56\xce\xd3\xef\x3e" +
+	"\xc7\x4e\xe8\x53\xb3\x6a\x52\x83\x52\x1b\x09\x8d\xef\x8d\xfa\x99" +
+	"\xd4\x5b\x6a\x34\xee\xa5\x2b\x6d\xd6\xe7\x17\x3e\xdb\xe3\xde\xf8" +
+	"\x1e\xab\xf3\xc1\x93\x43\x7c\xaf\x8c\xc2\x77\x87\xef\xb5\x5f\x0b" +
+	"\x62\xa3\x8d\x9b\x53\x0c\x52\x42\x27\x5a\x49\xe4\x07\xc6\xb8\x76" +
+	"\x9b\x03\xcd\x54\x35\x31\x9d\x98\xcf\x03\x55\x74\xb3\xf5\x45\x4b" +
+	"\x5d\xf0\x6d\xae\x96\xb5\xa6\x3d\x76\x24\x69\x28\xa3\x2d\x13\xc3" +
+	"\x7a\xaf\xde\x86\xd9\xa7\xbe\xe8\xf3\x6d\x1f\xa4\x9f\xd2\xec\x5b" +
+	"\x53\x10\xdc\xe9\xe8\x09\x18\x8b\xaa\x66\xc1\x16\x18\xa3\x7e\x79" +
+	"\xf8\x72\xcb\xab\x13\x7b\xde\x19\xbd\xb1\xd5\x41\x6e\x22\x8d\x27" +
+	"\xed\xc9\x68\x24\x24\xeb\x8f\x2f\xc9\x49\x33\xca\xc7\x1f\xcb\xe2" +
+	"\x04\xd5\x52\xb5\xea\xf8\x35\x5b\xad\x8d\xbb\x99\x6a\x66\x91\x6b" +
+	"\x45\x93\xaf\xb7\xac\x50\x43\x9e\x33\x28\x3c\xb5\x66\xfd\xf9\x4b" +
+	"\x27\x6a\x44\x0b\x2e\x1a\x62\x81\xef\xcb\x4e\x74\xdc\x99\xb0\x9c" +
+	"\x97\xcc\x0f\xa7\xdd\x64\xe7\xcd\x86\x6a\xf2\x79\x6e\xcd\xa3\x65" +
+	"\xf1\x55\x4d\x34\x52\x95\xd6\x44\x28\x24\x6a\xe5\x6a\x05\x60\x2e" +
+	"\xaf\xf8\x8b\xac\x06\xe2\x51\x3b\x5d\x4b\xb3\x03\xbe\xda\x7d\x92" +
+	"\x2b\xad\xaa\xfb\xda\xcd\x65\xdc\xe6\xa0\xba\x30\x3b\xe9\x27\xcb" +
+	"\xc8\x38\x6f\x78\xc9\xdd\xbe\xe9\xa3\x7c\x42\x97\x04\x77\xe0\x4d" +
+	"\xbb\x59\x3d\xe5\x8e\x86\x80\xeb\x75\xc1\x70\x2c\x88\x42\x83\x2b" +
+	"\x14\x16\x8e\x5d\x90\x0a\xf5\x42\x2a\xee\x2f\x9b\x8b\x81\xee\xb9" +
+	"\x43\x0f\x3e\xaa\xb5\x78\x5a\x2d\x64\x66\x15\x29\xe5\x2f\xec\x11" +
+	"\x6c\x69\x62\xa7\xc7\x06\xa5\xe9\x0f\xb3\xbc\xad\x36\x14\x45\x29" +
+	"\x95\xae\xf7\x10\x2a\x6a\xa6\x03\xa5\xf3\x31\xfa\x57\x06\xe4\x24" +
+	"\x04\x24\xf7\xe6\x83\x32\xa5\x4e\xf9\x10\x90\x76\xea\xb9\x22\xa6" +
+	"\xf9\x3a\xeb\x3f\x2e\x3d\x5e\x68\xe8\xbc\xa0\xd6\x39\x5e\x18\xb2" +
+	"\x1c\x6f\xb1\x1e\x37\x44\x93\x19\xaf\x4f\xca\xdc\xc0\x01\x8b\x2e" +
+	"\x28\x74\x32\x6c\x34\x8e\x53\x8d\xc7\xbc\x5b\x4e\xfa\xb2\x66\xbd" +
+	"\xf8\x46\x3a\x2d\xc7\x93\xc3\x44\xed\x51\x9d\x73\x76\xaa\x30\xcd" +
+	"\xb3\xe3\x72\xbf\x10\x2d\xeb\x70\xc2\xa2\x42\x2d\x9a\xcb\x03\x9c" +
+	"\xeb\xb3\x48\xf7\xf3\xcd\xc6\x30\xdb\xcf\x9a\x23\xb9\x65\x56\xfb" +
+	"\x39\x76\x32\x07\xe5\x7c\xc3\x7a\x75\xf9\xa1\x1d\x9d\x22\x24\xa4" +
+	"\x9b\x29\xce\x8c\x3e\xce\xbd\x6a\x45\x89\xf2\x26\xa8\x44\xf9\xc4" +
+	"\xb2\xae\xbe\x2e\x15\xd9\x06\x99\xda\xeb\x74\xb1\xe4\xf8\xe9\x22" +
+	"\xa5\xd4\xb3\xea\x20\xd1\x2b\xa8\xcf\x0d\xdb\xee\x8d\x9e\x0c\x61" +
+	"\xf6\xfa\xe2\xc5\xdd\x4e\x4e\xe8\xcc\xf3\x8e\xe1\xd7\x6e\xcb\x09" +
+	"\xa9\x2b\xd8\x04\x65\xec\x52\xe8\x69\xfa\x14\x5a\x4b\x47\xf4\xa2" +
+	"\x65\xde\x29\xf4\x58\xac\x76\x9f\x47\x39\xb8\x18\xc1\xf2\x4b\x56" +
+	"\xab\x22\xfc\xc8\xdb\xb9\x96\xd9\x83\x75\xa1\x3e\x00\xa3\x51\xc3" +
+	"\x18\x71\xee\x4b\xed\x16\x4b\x2d\x6d\xda\xec\xea\x6d\x37\x3a\x98" +
+	"\xf5\x75\x1b\x8f\x5b\x5a\xd9\xd1\x0c\x33\xda\x28\x2c\x5e\x26\x96" +
+	"\xe8\x77\xd9\xbe\xef\x67\xd8\x9a\xb1\x48\x35\xe7\xb3\xee\x5b\xbd" +
+	"\x62\xa9\xac\x17\xf7\xf8\x0a\x28\x24\x97\x5c\x62\x32\x6e\xb6\x9a" +
+	"\xaa\x69\xf0\x60\xd4\x66\x5f\x8c\x4e\x54\xc9\x65\x35\x38\xcb\x3e" +
+	"\xd5\x13\x4e\x35\xa9\x3b\x8f\xd6\x78\x1e\x97\x85\x6e\xf6\x6d\x7a" +
+	"\x2b\x4d\x92\xfd\xf4\xab\xe5\x56\xd5\x4c\x6b\x68\xd8\x05\x2b\x21" +
+	"\xb6\xf2\x05\x2e\xeb\x95\x73\x95\x44\x66\x6e\xd4\xe1\x30\xd9\xd2" +
+	"\x59\xcf\x91\xd5\xb6\xe4\x55\xab\xfd\x84\x37\x1f\xa6\x33\x12\x1a" +
+	"\xa6\x9b\xf1\xd9\xf3\xb8\xde\x02\xd1\x38\x07\x86\xcb\x0c\x8e\xa3" +
+	"\x46\x5c\xa9\xbc\xc8\x5a\x5a\x2f\xe4\xc4\x71\xdb\xfb\x0c\xec\xc5" +
+	"\x78\xd8\x96\x16\x1d\xd1\x46\x16\x26\xfd\xa9\xec\x1a\x22\xed\xba" +
+	"\x18\x67\x66\x7a\x3f\x80\x34\x07\x86\x08\xa5\xab\xeb\xcd\xe2\xbe" +
+	"\xa1\xfc\x30\x2d\xca\xe6\x30\x25\x46\x32\x45\x57\x87\x06\x8e\x29" +
+	"\x10\xf3\x1a\xa4\xaf\x65\xbf\x2e\x5d\x45\x3e\xf2\x46\xae\xef\x4f" +
+	"\x2b\x89\x05\x6e\xc4\xc2\x59\x44\x54\x46\x82\x79\x14\x09\xd9\x0f" +
+	"\x5c\xf9\x41\x1a\x41\xf5\x2b\xab\xc7\xdc\x5b\xbb\xc7\x43\xc7\x96" +
+	"\x69\x27\xb3\x31\x3c\x43\x12\x3c\xdf\xf0\x31\xf7\xd6\xf0\xf1\xd0" +
+	"\xb1\xe1\x48\xb8\xfe\x6c\x2b\x70\x20\xf4\xdb\x2b\xb0\x9f\x87\xdf" +
+	"\x5a\x80\x3f\x25\xf9\x9b\x68\x0f\x0a\x80\x4d\x85\x1c\xc9\x7e\x4c" +
+	"\x4d\xa4\x41\x9b\x70\x62\xf2\xee\x92\x03\x81\xb1\x2e\xc6\xc5\x7a" +
+	"\xbc\xbc\x1b\xfe\xca\xc1\x23\xe6\xc0\xbf\x0b\x24\x33\x57\x47\x43" +
+	"\x31\x51\x03\x8e\xd3\x21\x87\xb6\x18\x22\x67\x9f\x45\x57\xbe\x8a" +
+	"\x6c\x77\xa5\xc8\x35\xf3\x7f\x25\xb2\xf3\xac\x7d\x38\x76\x62\x27" +
+	"\x4c\x03\x0b\xdb\x2d\x1a\x1b\xd9\x85\x68\x26\xf6\xc2\xb9\x0a\x5c" +
+	"\x58\x31\x50\x73\xe0\xb9\xda\x42\x27\x02\x7d\x41\x39\x8f\xf5\x9d" +
+	"\xc2\xf1\x01\xd7\xae\x83\x0f\xfb\xc9\x4f\x40\xec\x94\xfd\x01\x50" +
+	"\xa8\xf1\xab\x88\xb5\xad\xa7\x6c\xda\xcc\xd5\x1c\xd8\x0c\x2a\x91" +
+	"\x93\x10\x2a\xee\xb4\xc4\x34\x68\x28\x78\x46\x88\xd1\xe8\xd6\x08" +
+	"\xfd\x51\xee\x2d\x1b\xb1\xb5\x13\xf9\x27\xfa\xd7\xfd\x8e\x8f\x56" +
+	"\x63\x3b\x76\xaa\x0a\xa4\xdf\x3d\xc8\x3f\xb3\xf7\xdb\x09\x7e\x39" +
+	"\x99\xe8\x6f\xb5\x7e\x5d\xed\x01\x74\x1f\x65\x67\x81\xfd\xa3\xc0" +
+	"\x21\xde\x3f\x8e\x9e\x31\xc1\x7f\xc5\x83\x93\x02\x87\xd3\xfc\xe7" +
+	"\x1e\x1c\x4d\xb8\x63\x3c\xfd\xcf\x21\xe5\x73\x94\xfc\x39\x35\x3a" +
+	"\xda\x0e\x7f\x3b\x4e\x4f\xa3\xe4\xe8\x28\xba\xab\xc6\x0d\x63\x49" +
+	"\x48\x3a\x86\xce\xf6\x22\xe1\x73\x8f\x03\xc1\xfb\x4f\xe4\x3e\xed" +
+	"\x52\x98\xb5\xc0\x2f\x41\x62\x57\xe5\x8f\x60\xd8\xb5\xfe\x1c\x3d" +
+	"\x3d\x53\x3b\xf4\xfc\xbc\x29\x02\x5d\xab\x7f\x4d\x98\x9c\xe1\x4f" +
+	"\xf7\x16\x7f\x89\x1c\xae\x01\xf6\x1b\x52\x28\x49\xff\x7d\x0a\x45" +
+	"\x59\x8c\x6c\x3a\xe6\xb0\x53\x2e\xd2\xdb\x53\xd2\x30\x17\x25\x34" +
+	"\x67\x90\x54\x3a\xb1\xc1\x33\x41\x00\xa5\xbd\x5d\x93\x31\x0d\x51" +
+	"\x43\xa2\xba\xe1\x3e\xd9\x5b\x44\x1e\x8a\x71\xa2\xf4\xe1\xf2\xfd" +
+	"\xdd\x06\x86\x02\x99\x3f\x82\xf0\x6f\x98\x3f\x2c\xe6\xae\xc4\xc4" +
+	"\x9e\xb6\x4b\xd2\x6a\xb5\x67\xdf\x92\xde\xdf\xd7\x9c\xab\xd5\xd6" +
+	"\xf4\x89\xfb\xbd\x77\x92\xe4\x58\x6d\x7d\x05\xb4\x5a\x85\x77\x41" +
+	"\xef\xef\xd0\x90\x56\x2b\x32\x1a\xe3\xa8\x24\x55\x11\xa4\xa3\x14" +
+	"\xd9\x9c\x6d\x83\x2b\xbe\xc8\xc3\x05\xd9\xe1\x8b\xc1\x95\xcf\x03" +
+	"\x6d\x01\x8a\xd8\x0e\x9e\xe4\x45\x7a\xa8\x22\x03\x68\x6b\xd3\x77" +
+	"\x4c\x31\x4e\x28\x94\xba\xd3\x19\x90\x02\xa9\x78\x28\x76\x20\xce" +
+	"\x99\x46\x70\x4f\x2b\xdb\x10\x9e\xd4\x22\x9f\xa7\x27\x0c\xee\x50" +
+	"\x78\x1b\xf1\x4f\x84\x5b\xc1\xc6\xc0\x5c\x21\x83\x11\x67\x24\x7b" +
+	"\x22\x86\xb6\x73\x7d\xa0\x8b\xec\x1d\x4c\x6c\x40\x3c\xfd\xa9\xb2" +
+	"\x8e\x4b\xce\x9e\x36\x63\xca\x7b\xba\x18\x0f\x11\xfd\x74\xad\xfb" +
+	"\x10\x5e\x8b\x90\xa0\xc4\x11\x46\x8e\xa0\xef\xe7\xca\x29\x3f\x23" +
+	"\x41\xc3\x41\xd8\x3f\xd6\x54\x59\x0f\xfc\x82\x8b\xf4\x02\xfe\x83" +
+	"\x4b\xcf\x84\x74\x9e\x33\x3b\x35\xf4\x2e\xff\x94\x2e\x21\xbc\xe3" +
+	"\x3f\x4f\x9f\xb0\xd1\x72\x3a\xc2\x8d\xb2\x33\xa3\xa4\x17\xcc\xf4" +
+	"\x8f\x86\x52\x24\x15\xa1\xe2\x15\x53\x7f\x02\x78\x76\x42\x8c\x3c" +
+	"\x43\x6c\xaf\x27\x21\x81\x7b\xd1\xda\xcc\xa6\xf3\xe7\x8e\x69\xe8" +
+	"\xa6\xeb\x40\x73\x0e\xed\x52\xe4\x93\x0d\x0b\xcf\x90\x73\xc3\x7c" +
+	"\xdb\x9f\xb7\xdf\xae\xef\x77\xb2\x2e\x2e\x45\x3e\x59\xa7\x3f\x11" +
+	"\x25\xa1\x0d\x6c\xa4\x0b\x16\x10\xe1\x1a\xa1\xcc\xbf\x99\x36\xb0" +
+	"\xd5\x4d\xba\x82\xb9\x63\xd1\x95\x25\x70\x99\x2c\x67\xeb\x55\xac" +
+	"\x18\x0f\xfe\xa6\xfa\x4f\x00\x00\x00\xff\xff\xdb\x07\xd3\x5c\xad" +
+	"\x1a\x00\x00"
+
+// DetailedTemplate returns the binary data for a given file.
+func DetailedTemplate() []byte {
+	// This bit of black magic ensures we do not get
+	// unneccesary memcpy's and can read directly from
+	// the .rodata section.
+	var empty [0]byte
+	sx := (*reflect.StringHeader)(unsafe.Pointer(&_DetailedTemplate))
+	b := empty[:]
+	bx := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	bx.Data = sx.Data
+	bx.Len = len(_DetailedTemplate)
+	bx.Cap = bx.Len
+
+	gz, err := gzip.NewReader(bytes.NewBuffer(b))
+
+	if err != nil {
+		panic("Decompression failed: " + err.Error())
+	}
+
+	var buf bytes.Buffer
+	io.Copy(&buf, gz)
+	gz.Close()
+
+	return buf.Bytes()
+}

vendor/github.com/JalfResi/justext/htmlRenderer.go

@@ -0,0 +1,89 @@
+package justext
+
+import (
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/levigross/exp-html"
+)
+
+/**
+This should be a separate package!
+And it should be a Writer!
+*/
+
+var selfClosingTags = map[string]bool{
+	"area":     true,
+	"base":     true,
+	"basefont": true,
+	"br":       true,
+	"hr":       true,
+	"input":    true,
+	"img":      true,
+	"link":     true,
+	"meta":     true,
+}
+
+// nodesToString loops over a node tree and generate HTML string
+// Should be moved into html/utils package
+func nodesToString(node *html.Node) string {
+	var response string = ""
+
+	switch node.Type {
+	case html.TextNode:
+		response = html.EscapeString(strings.TrimSpace(node.Data))
+
+	case html.ElementNode, html.DoctypeNode:
+		var att string = ""
+		if len(node.Attr) > 0 {
+			for _, a := range node.Attr {
+				att = fmt.Sprintf("%s %s=\"%s\"", att, a.Key, a.Val)
+			}
+		}
+
+		if _, ok := selfClosingTags[node.Data]; ok {
+			return fmt.Sprintf("<%s%s>", node.Data, att)
+		}
+
+		var content string = ""
+		if len(node.Child) > 0 {
+			for _, n := range node.Child {
+				content = fmt.Sprintf("%s%s", content, nodesToString(n))
+			}
+		}
+		response = fmt.Sprintf("<%s%s>%s</%s>", node.Data, att, content, node.Data)
+
+	case html.DocumentNode:
+		if len(node.Child) > 0 {
+			for _, n := range node.Child {
+				response = nodesToString(n)
+			}
+		}
+
+	case html.CommentNode:
+		// ignore
+
+	default:
+		log.Printf("Unhandled node: %s", nodeTypeToString(node))
+	}
+	return response
+}
+
+func nodeTypeToString(n *html.Node) (t string) {
+	switch n.Type {
+	case html.ErrorNode:
+		t = "Error"
+	case html.TextNode:
+		t = "Text"
+	case html.DocumentNode:
+		t = "Document"
+	case html.ElementNode:
+		t = "Element"
+	case html.CommentNode:
+		t = "Comment"
+	case html.DoctypeNode:
+		t = "Doctype"
+	}
+	return t
+}

vendor/github.com/JalfResi/justext/paragraphObjectModel.go

@@ -0,0 +1,160 @@
+package justext
+
+import (
+	"fmt"
+	"github.com/levigross/exp-html"
+	"io"
+	"regexp"
+	"strings"
+)
+
+var (
+	paragraphTags = map[string]bool{
+		"blockquote": true,
+		"caption":    true,
+		"center":     true,
+		"col":        true,
+		"colgroup":   true,
+		"dd":         true,
+		"div":        true,
+		"dl":         true,
+		"dt":         true,
+		"fieldset":   true,
+		"form":       true,
+		"legend":     true,
+		"optgroup":   true,
+		"option":     true,
+		"p":          true,
+		"pre":        true,
+		"table":      true,
+		"td":         true,
+		"textarea":   true,
+		"tfoot":      true,
+		"th":         true,
+		"thead":      true,
+		"tr":         true,
+		"ul":         true,
+		"li":         true,
+		"h1":         true,
+		"h2":         true,
+		"h3":         true,
+		"h4":         true,
+		"h5":         true,
+		"h6":         true,
+	}
+	matchWhiteSpace *regexp.Regexp = regexp.MustCompile("[\n\r\t]+")
+)
+
+type Paragraph struct {
+	DomPath         string
+	TextNodes       []string
+	WordCount       int
+	LinkedCharCount int
+	TagCount        int
+	Text            string
+	StopwordCount   int
+	StopwordDensity float64
+	LinkDensity     float64
+	Heading         bool
+	CfClass         string
+	Class           string
+}
+
+func paragraphObjectModel(htmlStr string) ([]*Paragraph, error) {
+
+	var dom []string
+	var paragraphs []*Paragraph
+	var paragraph *Paragraph = &Paragraph{WordCount: 0, LinkedCharCount: 0, TagCount: 0}
+	var link bool = false
+	var br bool = false
+	var matchToDoErrors *regexp.Regexp = regexp.MustCompile("^html: TODO: ")
+
+	var startNewParagraph func()
+	startNewParagraph = func() {
+		if len(paragraph.TextNodes) != 0 {
+			paragraph.Text = strings.TrimSpace(matchWhiteSpace.ReplaceAllString(strings.Join(paragraph.TextNodes, " "), " "))
+			paragraphs = append(paragraphs, paragraph)
+		}
+		paragraph = &Paragraph{
+			DomPath:         strings.Join(dom, "."),
+			WordCount:       0,
+			LinkedCharCount: 0,
+			TagCount:        0,
+		}
+	}
+
+	z := html.NewTokenizer(strings.NewReader(htmlStr))
+
+	for {
+		tt := z.Next()
+		switch tt {
+
+		case html.ErrorToken:
+			if z.Err() == io.EOF {
+				return paragraphs, nil
+			}
+			if matchToDoErrors.MatchString(fmt.Sprintf("%s", z.Err())) {
+				return nil, z.Err()
+			}
+			continue
+
+		case html.StartTagToken:
+			tmpName, _ := z.TagName()
+			name := string(tmpName)
+			//log.Println("Matched start tag: ", name)
+			dom = append(dom, name)
+			_, ok := paragraphTags[name]
+			if ok || (name == "br" && br) {
+				if name == "br" {
+					paragraph.TagCount--
+				}
+				startNewParagraph()
+			} else {
+				if name == "br" {
+					br = true
+				} else {
+					br = false
+				}
+				if name == "a" {
+					link = true
+				}
+				paragraph.TagCount++
+			}
+
+		case html.EndTagToken:
+			tmpName, _ := z.TagName()
+			name := string(tmpName)
+			//log.Println("Matched end tag: ", name)
+			dom = dom[0 : len(dom)-1]
+			if _, ok := paragraphTags[name]; ok {
+				startNewParagraph()
+			}
+			if name == "a" {
+				link = false
+			}
+
+		case html.TextToken:
+			text := strings.TrimSpace(string(z.Text()))
+			e := 15
+			if len(text) < e {
+				e = len(text)
+			}
+			//log.Println("Matched text: ", text[:e], "...")
+			if text == "" {
+				continue
+			}
+			text = strings.TrimSpace(matchWhiteSpace.ReplaceAllString(text, " "))
+			paragraph.TextNodes = append(paragraph.TextNodes, text)
+			words := strings.Split(text, " ")
+			paragraph.WordCount += len(words)
+			if link {
+				paragraph.LinkedCharCount += len(text)
+			}
+			br = false
+
+		}
+	}
+	startNewParagraph()
+
+	return paragraphs, nil
+}

vendor/github.com/JalfResi/justext/preprocess.go

@@ -0,0 +1,111 @@
+package justext
+
+import (
+	"github.com/levigross/exp-html"
+	"regexp"
+	"strings"
+)
+
+func preprocess(htmlStr, encoding, defaultEncoding, encErrors string) (*html.Node, error) {
+
+	root, err := html.Parse(strings.NewReader(htmlStr))
+	if err != nil {
+		return nil, err
+	}
+
+	addKwTags(root)
+	removeElements(root, []string{"head", "script", "style"})
+
+	return root, nil
+}
+
+type nodeIterator func(n *html.Node)
+
+func nodeIter(n *html.Node, f nodeIterator) {
+	f(n)
+	for _, c := range n.Child {
+		nodeIter(c, f)
+	}
+}
+
+func addKwTags(root *html.Node) *html.Node {
+	var blankText *regexp.Regexp = regexp.MustCompile("^[\n\r\t ]*$")
+	var nodesWithText []*html.Node
+
+	var markTextAndTail nodeIterator
+	markTextAndTail = func(node *html.Node) {
+		if node.Type != html.CommentNode || node.Type != html.DoctypeNode {
+			if node.Type == html.TextNode {
+				nodesWithText = append(nodesWithText, node)
+			}
+		}
+	}
+	nodeIter(root, markTextAndTail)
+
+	for _, node := range nodesWithText {
+		if blankText.MatchString(node.Data) {
+			node.Data = ""
+		} else {
+			kw := &html.Node{
+				Parent: nil,
+				Type:   html.ElementNode,
+				Data:   "kw",
+			}
+			node2 := CopyNode(node, true)
+			kw.Child = append(kw.Child, node2)
+			insertNode(node, kw)
+			node.Parent.Remove(node)
+		}
+	}
+
+	return root
+}
+
+func removeElements(root *html.Node, elementsToRemove []string) {
+	var toBeRemoved []*html.Node
+	var markRemovableNodes = func(node *html.Node) {
+		if node.Type == html.ElementNode {
+			for _, nodeName := range elementsToRemove {
+				if node.Data == nodeName {
+					toBeRemoved = append(toBeRemoved, node)
+				}
+			}
+		}
+	}
+	nodeIter(root, markRemovableNodes)
+
+	for _, node := range toBeRemoved {
+		node.Parent.Remove(node)
+	}
+}
+
+// insertsNode inserts a Node in a Node tree at the position of another node.
+// Should be moved into html/utils package
+func insertNode(originalNode *html.Node, newNode *html.Node) {
+	slice := originalNode.Parent.Child
+	for position, n := range slice {
+		if n == originalNode {
+			originalNode.Parent.Child = append(slice[:position], append([]*html.Node{newNode}, slice[position:]...)...)
+			return
+		}
+	}
+}
+
+func CopyNode(node *html.Node, deep bool) *html.Node {
+	newNode := &html.Node{
+		Type: node.Type,
+		Data: node.Data,
+	}
+
+	if deep && len(node.Child) > 0 {
+		for _, n := range node.Child {
+			newNode.Child = append(newNode.Child, CopyNode(n, true))
+		}
+	}
+
+	for _, i := range node.Attr {
+		newNode.Attr = append(newNode.Attr, html.Attribute{Key: i.Key, Val: i.Val})
+	}
+
+	return newNode
+}

vendor/github.com/JalfResi/justext/reader.go

@@ -0,0 +1,98 @@
+package justext
+
+import (
+	"errors"
+	"fmt"
+	"github.com/levigross/exp-html"
+	"io"
+	"io/ioutil"
+	"strings"
+)
+
+type Reader struct {
+	LengthLow          int
+	LengthHigh         int
+	Stoplist           map[string]bool
+	StopwordsLow       float64
+	StopwordsHigh      float64
+	MaxLinkDensity     float64
+	MaxHeadingDistance int
+	NoHeadings         bool
+	r                  io.Reader
+}
+
+func NewReader(r io.Reader) *Reader {
+	return &Reader{
+		LengthLow:          70,
+		LengthHigh:         200,
+		StopwordsLow:       0.30,
+		StopwordsHigh:      0.32,
+		MaxLinkDensity:     0.2,
+		MaxHeadingDistance: 200,
+		NoHeadings:         false,
+		r:                  r,
+	}
+}
+
+func (r *Reader) ReadAll() ([]*Paragraph, error) {
+	in, err := ioutil.ReadAll(r.r)
+	if err != nil {
+		return nil, err
+	}
+
+	root, err := preprocess(string(in), "utf-8", "utf-8", "errors")
+	if err != nil {
+		return nil, err
+	}
+	if root == nil {
+		return nil, errors.New("Preprocess has resulted in nil")
+	}
+
+	htmlSource := nodesToString(root)
+	if len(htmlSource) == 0 {
+		return nil, errors.New("MAIN: perprocess has returned an empty string")
+	}
+
+	p, err := paragraphObjectModel(htmlSource)
+	if err != nil {
+		return nil, err
+	}
+	if p == nil {
+		return nil, errors.New("MAIN: P is nil")
+	}
+
+	classifyParagraphs(p, r.Stoplist, r.LengthLow, r.LengthHigh, r.StopwordsLow, r.StopwordsHigh, r.MaxLinkDensity, r.NoHeadings)
+	reviseParagraphClassification(p, r.MaxHeadingDistance)
+
+	return p, nil
+}
+
+func dumpNodes(n *html.Node, tab int, exploreChildNodes bool) string {
+	var childNodes string = ""
+	if exploreChildNodes == true {
+		if len(n.Child) > 0 {
+			for _, c := range n.Child {
+				childNodes = fmt.Sprintf("%s%s\n", childNodes, dumpNodes(c, tab+1, true))
+			}
+		}
+	}
+
+	var t string
+	switch n.Type {
+	case html.ErrorNode:
+		t = "Err"
+	case html.TextNode:
+		t = "T"
+	case html.DocumentNode:
+		t = "D"
+	case html.ElementNode:
+		t = "E"
+	case html.CommentNode:
+		t = "C"
+	case html.DoctypeNode:
+		t = "Dt"
+	}
+
+	tabStr := strings.Repeat(" ", tab)
+	return fmt.Sprintf("%s%s:%s\n%s", tabStr, t, strings.TrimSpace(strings.Replace(n.Data, "\n", "", -1)), childNodes)
+}

vendor/github.com/JalfResi/justext/reviseParagraphClassification.go

@@ -0,0 +1,118 @@
+package justext
+
+// Context-sensitive paragraph classification. Assumes that classify_pragraphs has already been called.
+func reviseParagraphClassification(paragraphs []*Paragraph, maxHeadingDistance int) {
+
+	// Copy classes
+	for _, paragraph := range paragraphs {
+		paragraph.Class = paragraph.CfClass
+	}
+
+	// Good headings
+	var j int = 0
+	var distance int
+	for i, paragraph := range paragraphs {
+		if !(paragraph.Heading && paragraph.Class == "short") {
+			continue
+		}
+
+		j = i + 1
+		distance = 0
+
+		for j < len(paragraphs) && distance <= maxHeadingDistance {
+			if paragraphs[j].Class == "good" {
+				paragraph.Class = "neargood"
+				break
+			}
+			distance += len(paragraphs[j].Text)
+			j += 1
+		}
+	}
+
+	// Classify short
+	var newClasses []string = make([]string, len(paragraphs))
+	for i, paragraph := range paragraphs {
+		if paragraph.Class != "short" {
+			continue
+		}
+
+		var prevNeighbour string = getPrevNeighbour(i, paragraphs, true)
+		var nextNeighbour string = getNextNeighbour(i, paragraphs, true)
+
+		var neighbours map[string]bool = make(map[string]bool)
+		neighbours[prevNeighbour] = true
+		neighbours[nextNeighbour] = true
+
+		if _, ok := neighbours["good"]; ok && len(neighbours) == 1 {
+			newClasses[i] = "good"
+		} else if _, ok := neighbours["bad"]; ok && len(neighbours) == 1 {
+			newClasses[i] = "bad"
+			// neighbours must contain both good and bad
+		} else if (prevNeighbour == "bad" && getPrevNeighbour(i, paragraphs, false) == "neargood") || (nextNeighbour == "bad" && getNextNeighbour(i, paragraphs, false) == "neargood") {
+			newClasses[i] = "good"
+		} else {
+			newClasses[i] = "bad"
+		}
+	}
+
+	for i, c := range newClasses {
+		if c != "" {
+			paragraphs[i].Class = c
+		}
+	}
+
+	// revise neargood
+	for i, paragraph := range paragraphs {
+		if paragraph.Class != "neargood" {
+			continue
+		}
+
+		var prevNeighbour string = getPrevNeighbour(i, paragraphs, true)
+		var nextNeighbour string = getNextNeighbour(i, paragraphs, true)
+
+		if prevNeighbour == "bad" && nextNeighbour == "bad" {
+			paragraph.Class = "bad"
+		} else {
+			paragraph.Class = "good"
+		}
+	}
+
+	// more good headings
+	for i, paragraph := range paragraphs {
+		if !(paragraph.Heading && paragraph.Class == "bad" && paragraph.CfClass != "bad") {
+			continue
+		}
+		j = i + 1
+		distance = 0
+		for j < len(paragraphs) && distance <= maxHeadingDistance {
+			if paragraphs[j].Class == "good" {
+				paragraph.Class = "good"
+				break
+			}
+			distance += len(paragraphs[j].Text)
+			j += 1
+		}
+	}
+}
+
+func getPrevNeighbour(i int, paragraphs []*Paragraph, ignoreNeargood bool) string {
+	return getNeighbour(i, paragraphs, ignoreNeargood, -1, -1)
+}
+
+func getNextNeighbour(i int, paragraphs []*Paragraph, ignoreNeargood bool) string {
+	return getNeighbour(i, paragraphs, ignoreNeargood, 1, len(paragraphs))
+}
+
+func getNeighbour(i int, paragraphs []*Paragraph, ignoreNeargood bool, inc int, boundary int) string {
+	for i+inc != boundary {
+		i += inc
+		var c string = paragraphs[i].Class
+		if c == "good" || c == "bad" {
+			return c
+		}
+		if c == "neargood" && !ignoreNeargood {
+			return c
+		}
+	}
+	return "bad"
+}

vendor/github.com/JalfResi/justext/stoplists.go

@@ -0,0 +1,158 @@
+package justext
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+)
+
+type ResourceFunc func() ([]byte, error)
+
+var stoplists = map[string]ResourceFunc{}
+
+func RegisterStoplist(name string, resourceFunc ResourceFunc) {
+	stoplists[name] = resourceFunc
+}
+
+/*
+{
+	"Afrikaans":               AfrikaansStoplist,
+	"Albanian":                AlbanianStoplist,
+	"Arabic":                  ArabicStoplist,
+	"Aragonese":               AragoneseStoplist,
+	"Armenian":                ArmenianStoplist,
+	"Aromanian":               AromanianStoplist,
+	"Asturian":                AsturianStoplist,
+	"Azerbaijani":             AzerbaijaniStoplist,
+	"Basque":                  BasqueStoplist,
+	"Belarusian":              BelarusianStoplist,
+	"Belarusian_Taraskievica": Belarusian_TaraskievicaStoplist,
+	"Bengali":                 BengaliStoplist,
+	"Bishnupriya_Manipuri":    Bishnupriya_ManipuriStoplist,
+	"Bosnian":                 BosnianStoplist,
+	"Breton":                  BretonStoplist,
+	"Bulgarian":               BulgarianStoplist,
+	"Catalan":                 CatalanStoplist,
+	"Cebuano":                 CebuanoStoplist,
+	"Chuvash":                 ChuvashStoplist,
+	"Croatian":                CroatianStoplist,
+	"Czech":                   CzechStoplist,
+	"Danish":                  DanishStoplist,
+	"Dutch":                   DutchStoplist,
+	"English":                 EnglishStoplist,
+	"Esperanto":               EsperantoStoplist,
+	"Estonian":                EstonianStoplist,
+	"Finnish":                 FinnishStoplist,
+	"French":                  FrenchStoplist,
+	"Galician":                GalicianStoplist,
+	"Georgian":                GeorgianStoplist,
+	"German":                  GermanStoplist,
+	"Greek":                   GreekStoplist,
+	"Gujarati":                GujaratiStoplist,
+	"Haitian":                 HaitianStoplist,
+	"Hebrew":                  HebrewStoplist,
+	"Hindi":                   HindiStoplist,
+	"Hungarian":               HungarianStoplist,
+	"Icelandic":               IcelandicStoplist,
+	"Ido":                     IdoStoplist,
+	"Igbo":                    IgboStoplist,
+	"Indonesian":              IndonesianStoplist,
+	"Irish":                   IrishStoplist,
+	"Italian":                 ItalianStoplist,
+	"Javanese":                JavaneseStoplist,
+	"Kannada":                 KannadaStoplist,
+	"Kazakh":                  KazakhStoplist,
+	"Korean":                  KoreanStoplist,
+	"Kurdish":                 KurdishStoplist,
+	"Kyrgyz":                  KyrgyzStoplist,
+	"Latin":                   LatinStoplist,
+	"Latvian":                 LatvianStoplist,
+	"Lithuanian":              LithuanianStoplist,
+	"Lombard":                 LombardStoplist,
+	"Low_Saxon":               Low_SaxonStoplist,
+	"Luxembourgish":           LuxembourgishStoplist,
+	"Macedonian":              MacedonianStoplist,
+	"Malay":                   MalayStoplist,
+	"Malayalam":               MalayalamStoplist,
+	"Maltese":                 MalteseStoplist,
+	"Marathi":                 MarathiStoplist,
+	"Neapolitan":              NeapolitanStoplist,
+	"Nepali":                  NepaliStoplist,
+	"Newar":                   NewarStoplist,
+	"Norwegian_Bokmal":        Norwegian_BokmalStoplist,
+	"Norwegian_Nynorsk":       Norwegian_NynorskStoplist,
+	"Occitan":                 OccitanStoplist,
+	"Persian":                 PersianStoplist,
+	"Piedmontese":             PiedmonteseStoplist,
+	"Polish":                  PolishStoplist,
+	"Portuguese":              PortugueseStoplist,
+	"Quechua":                 QuechuaStoplist,
+	"Romanian":                RomanianStoplist,
+	"Russian":                 RussianStoplist,
+	"Samogitian":              SamogitianStoplist,
+	"Serbian":                 SerbianStoplist,
+	"Serbo_Croatian":          Serbo_CroatianStoplist,
+	"Sicilian":                SicilianStoplist,
+	"Simple_English":          Simple_EnglishStoplist,
+	"Slovak":                  SlovakStoplist,
+	"Slovenian":               SlovenianStoplist,
+	"Spanish":                 SpanishStoplist,
+	"Sundanese":               SundaneseStoplist,
+	"Swahili":                 SwahiliStoplist,
+	"Swedish":                 SwedishStoplist,
+	"Tagalog":                 TagalogStoplist,
+	"Tamil":                   TamilStoplist,
+	"Telugu":                  TeluguStoplist,
+	"Turkish":                 TurkishStoplist,
+	"Turkmen":                 TurkmenStoplist,
+	"Ukrainian":               UkrainianStoplist,
+	"Urdu":                    UrduStoplist,
+	"Uzbek":                   UzbekStoplist,
+	"Vietnamese":              VietnameseStoplist,
+	"Volapuk":                 VolapukStoplist,
+	"Walloon":                 WalloonStoplist,
+	"Waray_Waray":             Waray_WarayStoplist,
+	"Welsh":                   WelshStoplist,
+	"West_Frisian":            West_FrisianStoplist,
+	"Western_Panjabi":         Western_PanjabiStoplist,
+	"Yoruba":                  YorubaStoplist,
+}
+*/
+
+func ReadStoplist(filename string) (map[string]bool, error) {
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	db := bytes.Split(data, []uint8("\n"))
+
+	// Convert to map
+	var list = make(map[string]bool)
+	for _, val := range db {
+		list[string(val)] = true
+	}
+
+	return list, nil
+}
+
+func GetStoplist(language string) (map[string]bool, error) {
+	if _, ok := stoplists[language]; !ok {
+		return nil, errors.New(fmt.Sprintf("Language %s not supported", language))
+	}
+
+	data, err := stoplists[language]()
+	if err != nil {
+		return nil, err
+	}
+	db := bytes.Split(data, []uint8("\n"))
+
+	// Convert to map
+	var list = make(map[string]bool)
+	for _, val := range db {
+		list[string(val)] = true
+	}
+
+	return list, nil
+}

vendor/github.com/JalfResi/justext/writer.go

@@ -0,0 +1,161 @@
+package justext
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"strings"
+	"text/template"
+)
+
+// NOTE:
+// Make a new type:
+//  type JusText []paragraphs
+
+const (
+	MODE_DEFAULT  = 1
+	MODE_DETAILED = 2
+)
+
+type Writer struct {
+	Mode          int
+	NoBoilerplate bool
+	Stoplist      map[string]bool
+	w             io.Writer
+}
+
+func NewWriter(w io.Writer) *Writer {
+	return &Writer{
+		Mode:          MODE_DEFAULT,
+		NoBoilerplate: true,
+		w:             w,
+	}
+}
+
+func (w *Writer) WriteAll(paragraphs []*Paragraph) error {
+	switch w.Mode {
+	case MODE_DEFAULT:
+		return w.outputDefault(paragraphs)
+		break
+
+	case MODE_DETAILED:
+		return w.outputDetailed(paragraphs)
+		break
+
+	default:
+		return errors.New("Unrecognised mode")
+	}
+
+	return nil
+}
+
+func IsGood(args ...interface{}) (result bool) {
+	result = true
+	for _, val := range args {
+		if val != "good" {
+			result = false
+			return
+		}
+	}
+	return
+}
+
+func (w *Writer) outputDefault(paragraphs []*Paragraph) error {
+	templateData := DefaultTemplate()
+	t := template.New("default")
+	t.Funcs(template.FuncMap{"TrimSpace": strings.TrimSpace})
+	t.Funcs(template.FuncMap{"IsGood": IsGood})
+
+	templ, err := t.Parse(string(templateData))
+	if err != nil {
+		return err
+	}
+
+	var data = struct {
+		Paragraphs    []*Paragraph
+		NoBoilerplate bool
+	}{paragraphs, w.NoBoilerplate}
+
+	return templ.Execute(w.w, data)
+}
+
+func (w *Writer) outputDetailed(paragraphs []*Paragraph) error {
+	templateData := DetailedTemplate()
+	var markStopwords func(args ...interface{}) string
+	markStopwords = func(args ...interface{}) string {
+
+		var output string = ""
+		words := strings.Split(args[0].(string), " ")
+		for _, word := range words {
+			if _, ok := w.Stoplist[strings.TrimSpace(word)]; ok {
+				output = fmt.Sprintf("%s<span class=\"stopword\">%s</span> ", output, word)
+			} else {
+				output = fmt.Sprintf("%s%s ", output, word)
+			}
+		}
+
+		return output
+	}
+
+	t := template.New("detailed")
+	t.Funcs(template.FuncMap{"TrimSpace": strings.TrimSpace})
+	t.Funcs(template.FuncMap{"MarkStopwords": markStopwords})
+
+	templ, err := t.Parse(string(templateData))
+	if err != nil {
+		return err
+	}
+
+	var data = struct {
+		Paragraphs []*Paragraph
+	}{paragraphs}
+
+	return templ.Execute(w.w, data)
+}
+
+func (w *Writer) OutputDebug(paragraphs []*Paragraph) {
+	for _, paragraph := range paragraphs {
+		log.Println(paragraph.DomPath)
+		log.Println("\tfinal class: ", paragraph.Class)
+		log.Println("\tcontext-free class: ", paragraph.CfClass)
+		log.Println("\theading: ", paragraph.Heading)
+		log.Println("\tlength (in characters): ", len(paragraph.Text))
+		log.Println("\tnumber of characters with links: ", paragraph.LinkedCharCount)
+		log.Println("\tlink density: ", paragraph.LinkDensity)
+		log.Println("\tnumber of words: ", paragraph.WordCount)
+		log.Println("\tnumber of stop words: ", paragraph.StopwordCount)
+		log.Println("\tstop word density: ", paragraph.StopwordDensity)
+	}
+}
+
+// TO-DO:
+// Need an output feature that returns a de-duped space separated text file of all the
+// words in the output document sans-boilerplate. Also needs option to exclude stoplist
+// words from that output too.
+
+// TO-DO:
+// Need an output feature that returns the content of a stop list (or do we just make
+// the function getStoplist public? Might be a lot easier...)
+
+/*
+func (w *Writer) outputKrdwrd(paragraphs []*Paragraph) (output string) {
+	for _, paragraph := range paragraphs {
+		var cls int
+		if paragraph.Class == "good" || paragraph.Class == "neargood" {
+			if paragraph.Heading {
+				cls = 2
+			} else {
+				cls = 3
+			}
+		} else {
+			cls = 1
+		}
+		for _, textNode := range paragraph.TextNodes {
+			output = fmt.Sprintf("%s%i\t%s", output, cls, strings.TrimSpace(textNode))
+		}
+	}
+
+	return output
+}
+*/

vendor/github.com/PuerkitoBio/goquery/.gitattributes

@@ -0,0 +1 @@
+testdata/* linguist-vendored

vendor/github.com/PuerkitoBio/goquery/.gitignore

@@ -0,0 +1,16 @@
+# editor temporary files
+*.sublime-*
+.DS_Store
+*.swp
+#*.*#
+tags
+
+# direnv config
+.env*
+
+# test binaries
+*.test
+
+# coverage and profilte outputs
+*.out
+

vendor/github.com/PuerkitoBio/goquery/.travis.yml

@@ -0,0 +1,17 @@
+language: go
+
+go:
+    - 1.2.x
+    - 1.3.x
+    - 1.4.x
+    - 1.5.x
+    - 1.6.x
+    - 1.7.x
+    - 1.8.x
+    - 1.9.x
+    - 1.10.x
+    - 1.11.x
+    - 1.12.x
+    - 1.13.x
+    - tip
+

vendor/github.com/PuerkitoBio/goquery/LICENSE

@@ -0,0 +1,12 @@
+Copyright (c) 2012-2016, Martin Angers & Contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/PuerkitoBio/goquery/README.md

@@ -0,0 +1,183 @@
+# goquery - a little like that j-thing, only in Go
+[![build status](https://secure.travis-ci.org/PuerkitoBio/goquery.svg?branch=master)](http://travis-ci.org/PuerkitoBio/goquery) [![GoDoc](https://godoc.org/github.com/PuerkitoBio/goquery?status.png)](http://godoc.org/github.com/PuerkitoBio/goquery) [![Sourcegraph Badge](https://sourcegraph.com/github.com/PuerkitoBio/goquery/-/badge.svg)](https://sourcegraph.com/github.com/PuerkitoBio/goquery?badge)
+
+goquery brings a syntax and a set of features similar to [jQuery][] to the [Go language][go]. It is based on Go's [net/html package][html] and the CSS Selector library [cascadia][]. Since the net/html parser returns nodes, and not a full-featured DOM tree, jQuery's stateful manipulation functions (like height(), css(), detach()) have been left off.
+
+Also, because the net/html parser requires UTF-8 encoding, so does goquery: it is the caller's responsibility to ensure that the source document provides UTF-8 encoded HTML. See the [wiki][] for various options to do this.
+
+Syntax-wise, it is as close as possible to jQuery, with the same function names when possible, and that warm and fuzzy chainable interface. jQuery being the ultra-popular library that it is, I felt that writing a similar HTML-manipulating library was better to follow its API than to start anew (in the same spirit as Go's `fmt` package), even though some of its methods are less than intuitive (looking at you, [index()][index]...).
+
+## Table of Contents
+
+* [Installation](#installation)
+* [Changelog](#changelog)
+* [API](#api)
+* [Examples](#examples)
+* [Related Projects](#related-projects)
+* [Support](#support)
+* [License](#license)
+
+## Installation
+
+Please note that because of the net/html dependency, goquery requires Go1.1+.
+
+    $ go get github.com/PuerkitoBio/goquery
+
+(optional) To run unit tests:
+
+    $ cd $GOPATH/src/github.com/PuerkitoBio/goquery
+    $ go test
+
+(optional) To run benchmarks (warning: it runs for a few minutes):
+
+    $ cd $GOPATH/src/github.com/PuerkitoBio/goquery
+    $ go test -bench=".*"
+
+## Changelog
+
+**Note that goquery's API is now stable, and will not break.**
+
+*    **2020-02-04 (v1.5.1)** : Update module dependencies.
+*    **2018-11-15 (v1.5.0)** : Go module support (thanks @Zaba505).
+*    **2018-06-07 (v1.4.1)** : Add `NewDocumentFromReader` examples.
+*    **2018-03-24 (v1.4.0)** : Deprecate `NewDocument(url)` and `NewDocumentFromResponse(response)`.
+*    **2018-01-28 (v1.3.0)** : Add `ToEnd` constant to `Slice` until the end of the selection (thanks to @davidjwilkins for raising the issue).
+*    **2018-01-11 (v1.2.0)** : Add `AddBack*` and deprecate `AndSelf` (thanks to @davidjwilkins).
+*    **2017-02-12 (v1.1.0)** : Add `SetHtml` and `SetText` (thanks to @glebtv).
+*    **2016-12-29 (v1.0.2)** : Optimize allocations for `Selection.Text` (thanks to @radovskyb).
+*    **2016-08-28 (v1.0.1)** : Optimize performance for large documents.
+*    **2016-07-27 (v1.0.0)** : Tag version 1.0.0.
+*    **2016-06-15** : Invalid selector strings internally compile to a `Matcher` implementation that never matches any node (instead of a panic). So for example, `doc.Find("~")` returns an empty `*Selection` object.
+*    **2016-02-02** : Add `NodeName` utility function similar to the DOM's `nodeName` property. It returns the tag name of the first element in a selection, and other relevant values of non-element nodes (see godoc for details). Add `OuterHtml` utility function similar to the DOM's `outerHTML` property (named `OuterHtml` in small caps for consistency with the existing `Html` method on the `Selection`).
+*    **2015-04-20** : Add `AttrOr` helper method to return the attribute's value or a default value if absent. Thanks to [piotrkowalczuk][piotr].
+*    **2015-02-04** : Add more manipulation functions - Prepend* - thanks again to [Andrew Stone][thatguystone].
+*    **2014-11-28** : Add more manipulation functions - ReplaceWith*, Wrap* and Unwrap - thanks again to [Andrew Stone][thatguystone].
+*    **2014-11-07** : Add manipulation functions (thanks to [Andrew Stone][thatguystone]) and `*Matcher` functions, that receive compiled cascadia selectors instead of selector strings, thus avoiding potential panics thrown by goquery via `cascadia.MustCompile` calls. This results in better performance (selectors can be compiled once and reused) and more idiomatic error handling (you can handle cascadia's compilation errors, instead of recovering from panics, which had been bugging me for a long time). Note that the actual type expected is a `Matcher` interface, that `cascadia.Selector` implements. Other matcher implementations could be used.
+*    **2014-11-06** : Change import paths of net/html to golang.org/x/net/html (see https://groups.google.com/forum/#!topic/golang-nuts/eD8dh3T9yyA). Make sure to update your code to use the new import path too when you call goquery with `html.Node`s.
+*    **v0.3.2** : Add `NewDocumentFromReader()` (thanks jweir) which allows creating a goquery document from an io.Reader.
+*    **v0.3.1** : Add `NewDocumentFromResponse()` (thanks assassingj) which allows creating a goquery document from an http response.
+*    **v0.3.0** : Add `EachWithBreak()` which allows to break out of an `Each()` loop by returning false. This function was added instead of changing the existing `Each()` to avoid breaking compatibility.
+*    **v0.2.1** : Make go-getable, now that [go.net/html is Go1.0-compatible][gonet] (thanks to @matrixik for pointing this out).
+*    **v0.2.0** : Add support for negative indices in Slice(). **BREAKING CHANGE** `Document.Root` is removed, `Document` is now a `Selection` itself (a selection of one, the root element, just like `Document.Root` was before). Add jQuery's Closest() method.
+*    **v0.1.1** : Add benchmarks to use as baseline for refactorings, refactor Next...() and Prev...() methods to use the new html package's linked list features (Next/PrevSibling, FirstChild). Good performance boost (40+% in some cases).
+*    **v0.1.0** : Initial release.
+
+## API
+
+goquery exposes two structs, `Document` and `Selection`, and the `Matcher` interface. Unlike jQuery, which is loaded as part of a DOM document, and thus acts on its containing document, goquery doesn't know which HTML document to act upon. So it needs to be told, and that's what the `Document` type is for. It holds the root document node as the initial Selection value to manipulate.
+
+jQuery often has many variants for the same function (no argument, a selector string argument, a jQuery object argument, a DOM element argument, ...). Instead of exposing the same features in goquery as a single method with variadic empty interface arguments, statically-typed signatures are used following this naming convention:
+
+*    When the jQuery equivalent can be called with no argument, it has the same name as jQuery for the no argument signature (e.g.: `Prev()`), and the version with a selector string argument is called `XxxFiltered()` (e.g.: `PrevFiltered()`)
+*    When the jQuery equivalent **requires** one argument, the same name as jQuery is used for the selector string version (e.g.: `Is()`)
+*    The signatures accepting a jQuery object as argument are defined in goquery as `XxxSelection()` and take a `*Selection` object as argument (e.g.: `FilterSelection()`)
+*    The signatures accepting a DOM element as argument in jQuery are defined in goquery as `XxxNodes()` and take a variadic argument of type `*html.Node` (e.g.: `FilterNodes()`)
+*    The signatures accepting a function as argument in jQuery are defined in goquery as `XxxFunction()` and take a function as argument (e.g.: `FilterFunction()`)
+*    The goquery methods that can be called with a selector string have a corresponding version that take a `Matcher` interface and are defined as `XxxMatcher()` (e.g.: `IsMatcher()`)
+
+Utility functions that are not in jQuery but are useful in Go are implemented as functions (that take a `*Selection` as parameter), to avoid a potential naming clash on the `*Selection`'s methods (reserved for jQuery-equivalent behaviour).
+
+The complete [godoc reference documentation can be found here][doc].
+
+Please note that Cascadia's selectors do not necessarily match all supported selectors of jQuery (Sizzle). See the [cascadia project][cascadia] for details. Invalid selector strings compile to a `Matcher` that fails to match any node. Behaviour of the various functions that take a selector string as argument follows from that fact, e.g. (where `~` is an invalid selector string):
+
+* `Find("~")` returns an empty selection because the selector string doesn't match anything.
+* `Add("~")` returns a new selection that holds the same nodes as the original selection, because it didn't add any node (selector string didn't match anything).
+* `ParentsFiltered("~")` returns an empty selection because the selector string doesn't match anything.
+* `ParentsUntil("~")` returns all parents of the selection because the selector string didn't match any element to stop before the top element.
+
+## Examples
+
+See some tips and tricks in the [wiki][].
+
+Adapted from example_test.go:
+
+```Go
+package main
+
+import (
+  "fmt"
+  "log"
+  "net/http"
+
+  "github.com/PuerkitoBio/goquery"
+)
+
+func ExampleScrape() {
+  // Request the HTML page.
+  res, err := http.Get("http://metalsucks.net")
+  if err != nil {
+    log.Fatal(err)
+  }
+  defer res.Body.Close()
+  if res.StatusCode != 200 {
+    log.Fatalf("status code error: %d %s", res.StatusCode, res.Status)
+  }
+
+  // Load the HTML document
+  doc, err := goquery.NewDocumentFromReader(res.Body)
+  if err != nil {
+    log.Fatal(err)
+  }
+
+  // Find the review items
+  doc.Find(".sidebar-reviews article .content-block").Each(func(i int, s *goquery.Selection) {
+    // For each item found, get the band and title
+    band := s.Find("a").Text()
+    title := s.Find("i").Text()
+    fmt.Printf("Review %d: %s - %s\n", i, band, title)
+  })
+}
+
+func main() {
+  ExampleScrape()
+}
+```
+
+## Related Projects
+
+- [Goq][goq], an HTML deserialization and scraping library based on goquery and struct tags.
+- [andybalholm/cascadia][cascadia], the CSS selector library used by goquery.
+- [suntong/cascadia][cascadiacli], a command-line interface to the cascadia CSS selector library, useful to test selectors.
+- [gocolly/colly](https://github.com/gocolly/colly), a lightning fast and elegant Scraping Framework
+- [gnulnx/goperf](https://github.com/gnulnx/goperf), a website performance test tool that also fetches static assets.
+- [MontFerret/ferret](https://github.com/MontFerret/ferret), declarative web scraping.
+- [tacusci/berrycms](https://github.com/tacusci/berrycms), a modern simple to use CMS with easy to write plugins
+- [Dataflow kit](https://github.com/slotix/dataflowkit), Web Scraping framework for Gophers.  
+- [Geziyor](https://github.com/geziyor/geziyor), a fast web crawling & scraping framework for Go. Supports JS rendering.
+
+## Support
+
+There are a number of ways you can support the project:
+
+* Use it, star it, build something with it, spread the word!
+  - If you do build something open-source or otherwise publicly-visible, let me know so I can add it to the [Related Projects](#related-projects) section!
+* Raise issues to improve the project (note: doc typos and clarifications are issues too!)
+  - Please search existing issues before opening a new one - it may have already been adressed.
+* Pull requests: please discuss new code in an issue first, unless the fix is really trivial.
+  - Make sure new code is tested.
+  - Be mindful of existing code - PRs that break existing code have a high probability of being declined, unless it fixes a serious issue.
+
+If you desperately want to send money my way, I have a BuyMeACoffee.com page:
+
+<a href="https://www.buymeacoffee.com/mna" target="_blank"><img src="https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png" alt="Buy Me A Coffee" style="height: 41px !important;width: 174px !important;box-shadow: 0px 3px 2px 0px rgba(190, 190, 190, 0.5) !important;-webkit-box-shadow: 0px 3px 2px 0px rgba(190, 190, 190, 0.5) !important;" ></a>
+
+## License
+
+The [BSD 3-Clause license][bsd], the same as the [Go language][golic]. Cascadia's license is [here][caslic].
+
+[jquery]: http://jquery.com/
+[go]: http://golang.org/
+[cascadia]: https://github.com/andybalholm/cascadia
+[cascadiacli]: https://github.com/suntong/cascadia
+[bsd]: http://opensource.org/licenses/BSD-3-Clause
+[golic]: http://golang.org/LICENSE
+[caslic]: https://github.com/andybalholm/cascadia/blob/master/LICENSE
+[doc]: http://godoc.org/github.com/PuerkitoBio/goquery
+[index]: http://api.jquery.com/index/
+[gonet]: https://github.com/golang/net/
+[html]: http://godoc.org/golang.org/x/net/html
+[wiki]: https://github.com/PuerkitoBio/goquery/wiki/Tips-and-tricks
+[thatguystone]: https://github.com/thatguystone
+[piotr]: https://github.com/piotrkowalczuk
+[goq]: https://github.com/andrewstuart/goq

vendor/github.com/PuerkitoBio/goquery/array.go

@@ -0,0 +1,124 @@
+package goquery
+
+import (
+	"golang.org/x/net/html"
+)
+
+const (
+	maxUint = ^uint(0)
+	maxInt  = int(maxUint >> 1)
+
+	// ToEnd is a special index value that can be used as end index in a call
+	// to Slice so that all elements are selected until the end of the Selection.
+	// It is equivalent to passing (*Selection).Length().
+	ToEnd = maxInt
+)
+
+// First reduces the set of matched elements to the first in the set.
+// It returns a new Selection object, and an empty Selection object if the
+// the selection is empty.
+func (s *Selection) First() *Selection {
+	return s.Eq(0)
+}
+
+// Last reduces the set of matched elements to the last in the set.
+// It returns a new Selection object, and an empty Selection object if
+// the selection is empty.
+func (s *Selection) Last() *Selection {
+	return s.Eq(-1)
+}
+
+// Eq reduces the set of matched elements to the one at the specified index.
+// If a negative index is given, it counts backwards starting at the end of the
+// set. It returns a new Selection object, and an empty Selection object if the
+// index is invalid.
+func (s *Selection) Eq(index int) *Selection {
+	if index < 0 {
+		index += len(s.Nodes)
+	}
+
+	if index >= len(s.Nodes) || index < 0 {
+		return newEmptySelection(s.document)
+	}
+
+	return s.Slice(index, index+1)
+}
+
+// Slice reduces the set of matched elements to a subset specified by a range
+// of indices. The start index is 0-based and indicates the index of the first
+// element to select. The end index is 0-based and indicates the index at which
+// the elements stop being selected (the end index is not selected).
+//
+// The indices may be negative, in which case they represent an offset from the
+// end of the selection.
+//
+// The special value ToEnd may be specified as end index, in which case all elements
+// until the end are selected. This works both for a positive and negative start
+// index.
+func (s *Selection) Slice(start, end int) *Selection {
+	if start < 0 {
+		start += len(s.Nodes)
+	}
+	if end == ToEnd {
+		end = len(s.Nodes)
+	} else if end < 0 {
+		end += len(s.Nodes)
+	}
+	return pushStack(s, s.Nodes[start:end])
+}
+
+// Get retrieves the underlying node at the specified index.
+// Get without parameter is not implemented, since the node array is available
+// on the Selection object.
+func (s *Selection) Get(index int) *html.Node {
+	if index < 0 {
+		index += len(s.Nodes) // Negative index gets from the end
+	}
+	return s.Nodes[index]
+}
+
+// Index returns the position of the first element within the Selection object
+// relative to its sibling elements.
+func (s *Selection) Index() int {
+	if len(s.Nodes) > 0 {
+		return newSingleSelection(s.Nodes[0], s.document).PrevAll().Length()
+	}
+	return -1
+}
+
+// IndexSelector returns the position of the first element within the
+// Selection object relative to the elements matched by the selector, or -1 if
+// not found.
+func (s *Selection) IndexSelector(selector string) int {
+	if len(s.Nodes) > 0 {
+		sel := s.document.Find(selector)
+		return indexInSlice(sel.Nodes, s.Nodes[0])
+	}
+	return -1
+}
+
+// IndexMatcher returns the position of the first element within the
+// Selection object relative to the elements matched by the matcher, or -1 if
+// not found.
+func (s *Selection) IndexMatcher(m Matcher) int {
+	if len(s.Nodes) > 0 {
+		sel := s.document.FindMatcher(m)
+		return indexInSlice(sel.Nodes, s.Nodes[0])
+	}
+	return -1
+}
+
+// IndexOfNode returns the position of the specified node within the Selection
+// object, or -1 if not found.
+func (s *Selection) IndexOfNode(node *html.Node) int {
+	return indexInSlice(s.Nodes, node)
+}
+
+// IndexOfSelection returns the position of the first node in the specified
+// Selection object within this Selection object, or -1 if not found.
+func (s *Selection) IndexOfSelection(sel *Selection) int {
+	if sel != nil && len(sel.Nodes) > 0 {
+		return indexInSlice(s.Nodes, sel.Nodes[0])
+	}
+	return -1
+}

vendor/github.com/PuerkitoBio/goquery/doc.go

@@ -0,0 +1,123 @@
+// Copyright (c) 2012-2016, Martin Angers & Contributors
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation and/or
+// other materials provided with the distribution.
+// * Neither the name of the author nor the names of its contributors may be used to
+// endorse or promote products derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS
+// OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
+// WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+/*
+Package goquery implements features similar to jQuery, including the chainable
+syntax, to manipulate and query an HTML document.
+
+It brings a syntax and a set of features similar to jQuery to the Go language.
+It is based on Go's net/html package and the CSS Selector library cascadia.
+Since the net/html parser returns nodes, and not a full-featured DOM
+tree, jQuery's stateful manipulation functions (like height(), css(), detach())
+have been left off.
+
+Also, because the net/html parser requires UTF-8 encoding, so does goquery: it is
+the caller's responsibility to ensure that the source document provides UTF-8 encoded HTML.
+See the repository's wiki for various options on how to do this.
+
+Syntax-wise, it is as close as possible to jQuery, with the same method names when
+possible, and that warm and fuzzy chainable interface. jQuery being the
+ultra-popular library that it is, writing a similar HTML-manipulating
+library was better to follow its API than to start anew (in the same spirit as
+Go's fmt package), even though some of its methods are less than intuitive (looking
+at you, index()...).
+
+It is hosted on GitHub, along with additional documentation in the README.md
+file: https://github.com/puerkitobio/goquery
+
+Please note that because of the net/html dependency, goquery requires Go1.1+.
+
+The various methods are split into files based on the category of behavior.
+The three dots (...) indicate that various "overloads" are available.
+
+* array.go : array-like positional manipulation of the selection.
+    - Eq()
+    - First()
+    - Get()
+    - Index...()
+    - Last()
+    - Slice()
+
+* expand.go : methods that expand or augment the selection's set.
+    - Add...()
+    - AndSelf()
+    - Union(), which is an alias for AddSelection()
+
+* filter.go : filtering methods, that reduce the selection's set.
+    - End()
+    - Filter...()
+    - Has...()
+    - Intersection(), which is an alias of FilterSelection()
+    - Not...()
+
+* iteration.go : methods to loop over the selection's nodes.
+    - Each()
+    - EachWithBreak()
+    - Map()
+
+* manipulation.go : methods for modifying the document
+    - After...()
+    - Append...()
+    - Before...()
+    - Clone()
+    - Empty()
+    - Prepend...()
+    - Remove...()
+    - ReplaceWith...()
+    - Unwrap()
+    - Wrap...()
+    - WrapAll...()
+    - WrapInner...()
+
+* property.go : methods that inspect and get the node's properties values.
+    - Attr*(), RemoveAttr(), SetAttr()
+    - AddClass(), HasClass(), RemoveClass(), ToggleClass()
+    - Html()
+    - Length()
+    - Size(), which is an alias for Length()
+    - Text()
+
+* query.go : methods that query, or reflect, a node's identity.
+    - Contains()
+    - Is...()
+
+* traversal.go : methods to traverse the HTML document tree.
+    - Children...()
+    - Contents()
+    - Find...()
+    - Next...()
+    - Parent[s]...()
+    - Prev...()
+    - Siblings...()
+
+* type.go : definition of the types exposed by goquery.
+    - Document
+    - Selection
+    - Matcher
+
+* utilities.go : definition of helper functions (and not methods on a *Selection)
+that are not part of jQuery, but are useful to goquery.
+    - NodeName
+    - OuterHtml
+*/
+package goquery

vendor/github.com/PuerkitoBio/goquery/expand.go

@@ -0,0 +1,70 @@
+package goquery
+
+import "golang.org/x/net/html"
+
+// Add adds the selector string's matching nodes to those in the current
+// selection and returns a new Selection object.
+// The selector string is run in the context of the document of the current
+// Selection object.
+func (s *Selection) Add(selector string) *Selection {
+	return s.AddNodes(findWithMatcher([]*html.Node{s.document.rootNode}, compileMatcher(selector))...)
+}
+
+// AddMatcher adds the matcher's matching nodes to those in the current
+// selection and returns a new Selection object.
+// The matcher is run in the context of the document of the current
+// Selection object.
+func (s *Selection) AddMatcher(m Matcher) *Selection {
+	return s.AddNodes(findWithMatcher([]*html.Node{s.document.rootNode}, m)...)
+}
+
+// AddSelection adds the specified Selection object's nodes to those in the
+// current selection and returns a new Selection object.
+func (s *Selection) AddSelection(sel *Selection) *Selection {
+	if sel == nil {
+		return s.AddNodes()
+	}
+	return s.AddNodes(sel.Nodes...)
+}
+
+// Union is an alias for AddSelection.
+func (s *Selection) Union(sel *Selection) *Selection {
+	return s.AddSelection(sel)
+}
+
+// AddNodes adds the specified nodes to those in the
+// current selection and returns a new Selection object.
+func (s *Selection) AddNodes(nodes ...*html.Node) *Selection {
+	return pushStack(s, appendWithoutDuplicates(s.Nodes, nodes, nil))
+}
+
+// AndSelf adds the previous set of elements on the stack to the current set.
+// It returns a new Selection object containing the current Selection combined
+// with the previous one.
+// Deprecated: This function has been deprecated and is now an alias for AddBack().
+func (s *Selection) AndSelf() *Selection {
+	return s.AddBack()
+}
+
+// AddBack adds the previous set of elements on the stack to the current set.
+// It returns a new Selection object containing the current Selection combined
+// with the previous one.
+func (s *Selection) AddBack() *Selection {
+	return s.AddSelection(s.prevSel)
+}
+
+// AddBackFiltered reduces the previous set of elements on the stack to those that
+// match the selector string, and adds them to the current set.
+// It returns a new Selection object containing the current Selection combined
+// with the filtered previous one
+func (s *Selection) AddBackFiltered(selector string) *Selection {
+	return s.AddSelection(s.prevSel.Filter(selector))
+}
+
+// AddBackMatcher reduces the previous set of elements on the stack to those that match
+// the mateher, and adds them to the curernt set.
+// It returns a new Selection object containing the current Selection combined
+// with the filtered previous one
+func (s *Selection) AddBackMatcher(m Matcher) *Selection {
+	return s.AddSelection(s.prevSel.FilterMatcher(m))
+}

vendor/github.com/PuerkitoBio/goquery/filter.go

@@ -0,0 +1,163 @@
+package goquery
+
+import "golang.org/x/net/html"
+
+// Filter reduces the set of matched elements to those that match the selector string.
+// It returns a new Selection object for this subset of matching elements.
+func (s *Selection) Filter(selector string) *Selection {
+	return s.FilterMatcher(compileMatcher(selector))
+}
+
+// FilterMatcher reduces the set of matched elements to those that match
+// the given matcher. It returns a new Selection object for this subset
+// of matching elements.
+func (s *Selection) FilterMatcher(m Matcher) *Selection {
+	return pushStack(s, winnow(s, m, true))
+}
+
+// Not removes elements from the Selection that match the selector string.
+// It returns a new Selection object with the matching elements removed.
+func (s *Selection) Not(selector string) *Selection {
+	return s.NotMatcher(compileMatcher(selector))
+}
+
+// NotMatcher removes elements from the Selection that match the given matcher.
+// It returns a new Selection object with the matching elements removed.
+func (s *Selection) NotMatcher(m Matcher) *Selection {
+	return pushStack(s, winnow(s, m, false))
+}
+
+// FilterFunction reduces the set of matched elements to those that pass the function's test.
+// It returns a new Selection object for this subset of elements.
+func (s *Selection) FilterFunction(f func(int, *Selection) bool) *Selection {
+	return pushStack(s, winnowFunction(s, f, true))
+}
+
+// NotFunction removes elements from the Selection that pass the function's test.
+// It returns a new Selection object with the matching elements removed.
+func (s *Selection) NotFunction(f func(int, *Selection) bool) *Selection {
+	return pushStack(s, winnowFunction(s, f, false))
+}
+
+// FilterNodes reduces the set of matched elements to those that match the specified nodes.
+// It returns a new Selection object for this subset of elements.
+func (s *Selection) FilterNodes(nodes ...*html.Node) *Selection {
+	return pushStack(s, winnowNodes(s, nodes, true))
+}
+
+// NotNodes removes elements from the Selection that match the specified nodes.
+// It returns a new Selection object with the matching elements removed.
+func (s *Selection) NotNodes(nodes ...*html.Node) *Selection {
+	return pushStack(s, winnowNodes(s, nodes, false))
+}
+
+// FilterSelection reduces the set of matched elements to those that match a
+// node in the specified Selection object.
+// It returns a new Selection object for this subset of elements.
+func (s *Selection) FilterSelection(sel *Selection) *Selection {
+	if sel == nil {
+		return pushStack(s, winnowNodes(s, nil, true))
+	}
+	return pushStack(s, winnowNodes(s, sel.Nodes, true))
+}
+
+// NotSelection removes elements from the Selection that match a node in the specified
+// Selection object. It returns a new Selection object with the matching elements removed.
+func (s *Selection) NotSelection(sel *Selection) *Selection {
+	if sel == nil {
+		return pushStack(s, winnowNodes(s, nil, false))
+	}
+	return pushStack(s, winnowNodes(s, sel.Nodes, false))
+}
+
+// Intersection is an alias for FilterSelection.
+func (s *Selection) Intersection(sel *Selection) *Selection {
+	return s.FilterSelection(sel)
+}
+
+// Has reduces the set of matched elements to those that have a descendant
+// that matches the selector.
+// It returns a new Selection object with the matching elements.
+func (s *Selection) Has(selector string) *Selection {
+	return s.HasSelection(s.document.Find(selector))
+}
+
+// HasMatcher reduces the set of matched elements to those that have a descendant
+// that matches the matcher.
+// It returns a new Selection object with the matching elements.
+func (s *Selection) HasMatcher(m Matcher) *Selection {
+	return s.HasSelection(s.document.FindMatcher(m))
+}
+
+// HasNodes reduces the set of matched elements to those that have a
+// descendant that matches one of the nodes.
+// It returns a new Selection object with the matching elements.
+func (s *Selection) HasNodes(nodes ...*html.Node) *Selection {
+	return s.FilterFunction(func(_ int, sel *Selection) bool {
+		// Add all nodes that contain one of the specified nodes
+		for _, n := range nodes {
+			if sel.Contains(n) {
+				return true
+			}
+		}
+		return false
+	})
+}
+
+// HasSelection reduces the set of matched elements to those that have a
+// descendant that matches one of the nodes of the specified Selection object.
+// It returns a new Selection object with the matching elements.
+func (s *Selection) HasSelection(sel *Selection) *Selection {
+	if sel == nil {
+		return s.HasNodes()
+	}
+	return s.HasNodes(sel.Nodes...)
+}
+
+// End ends the most recent filtering operation in the current chain and
+// returns the set of matched elements to its previous state.
+func (s *Selection) End() *Selection {
+	if s.prevSel != nil {
+		return s.prevSel
+	}
+	return newEmptySelection(s.document)
+}
+
+// Filter based on the matcher, and the indicator to keep (Filter) or
+// to get rid of (Not) the matching elements.
+func winnow(sel *Selection, m Matcher, keep bool) []*html.Node {
+	// Optimize if keep is requested
+	if keep {
+		return m.Filter(sel.Nodes)
+	}
+	// Use grep
+	return grep(sel, func(i int, s *Selection) bool {
+		return !m.Match(s.Get(0))
+	})
+}
+
+// Filter based on an array of nodes, and the indicator to keep (Filter) or
+// to get rid of (Not) the matching elements.
+func winnowNodes(sel *Selection, nodes []*html.Node, keep bool) []*html.Node {
+	if len(nodes)+len(sel.Nodes) < minNodesForSet {
+		return grep(sel, func(i int, s *Selection) bool {
+			return isInSlice(nodes, s.Get(0)) == keep
+		})
+	}
+
+	set := make(map[*html.Node]bool)
+	for _, n := range nodes {
+		set[n] = true
+	}
+	return grep(sel, func(i int, s *Selection) bool {
+		return set[s.Get(0)] == keep
+	})
+}
+
+// Filter based on a function test, and the indicator to keep (Filter) or
+// to get rid of (Not) the matching elements.
+func winnowFunction(sel *Selection, f func(int, *Selection) bool, keep bool) []*html.Node {
+	return grep(sel, func(i int, s *Selection) bool {
+		return f(i, s) == keep
+	})
+}

vendor/github.com/PuerkitoBio/goquery/go.mod

@@ -0,0 +1,8 @@
+module github.com/PuerkitoBio/goquery
+
+require (
+	github.com/andybalholm/cascadia v1.1.0
+	golang.org/x/net v0.0.0-20200202094626-16171245cfb2
+)
+
+go 1.13

vendor/github.com/PuerkitoBio/goquery/go.sum

@@ -0,0 +1,8 @@
+github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

vendor/github.com/PuerkitoBio/goquery/iteration.go

@@ -0,0 +1,39 @@
+package goquery
+
+// Each iterates over a Selection object, executing a function for each
+// matched element. It returns the current Selection object. The function
+// f is called for each element in the selection with the index of the
+// element in that selection starting at 0, and a *Selection that contains
+// only that element.
+func (s *Selection) Each(f func(int, *Selection)) *Selection {
+	for i, n := range s.Nodes {
+		f(i, newSingleSelection(n, s.document))
+	}
+	return s
+}
+
+// EachWithBreak iterates over a Selection object, executing a function for each
+// matched element. It is identical to Each except that it is possible to break
+// out of the loop by returning false in the callback function. It returns the
+// current Selection object.
+func (s *Selection) EachWithBreak(f func(int, *Selection) bool) *Selection {
+	for i, n := range s.Nodes {
+		if !f(i, newSingleSelection(n, s.document)) {
+			return s
+		}
+	}
+	return s
+}
+
+// Map passes each element in the current matched set through a function,
+// producing a slice of string holding the returned values. The function
+// f is called for each element in the selection with the index of the
+// element in that selection starting at 0, and a *Selection that contains
+// only that element.
+func (s *Selection) Map(f func(int, *Selection) string) (result []string) {
+	for i, n := range s.Nodes {
+		result = append(result, f(i, newSingleSelection(n, s.document)))
+	}
+
+	return result
+}

vendor/github.com/PuerkitoBio/goquery/manipulation.go

@@ -0,0 +1,574 @@
+package goquery
+
+import (
+	"strings"
+
+	"golang.org/x/net/html"
+)
+
+// After applies the selector from the root document and inserts the matched elements
+// after the elements in the set of matched elements.
+//
+// If one of the matched elements in the selection is not currently in the
+// document, it's impossible to insert nodes after it, so it will be ignored.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) After(selector string) *Selection {
+	return s.AfterMatcher(compileMatcher(selector))
+}
+
+// AfterMatcher applies the matcher from the root document and inserts the matched elements
+// after the elements in the set of matched elements.
+//
+// If one of the matched elements in the selection is not currently in the
+// document, it's impossible to insert nodes after it, so it will be ignored.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) AfterMatcher(m Matcher) *Selection {
+	return s.AfterNodes(m.MatchAll(s.document.rootNode)...)
+}
+
+// AfterSelection inserts the elements in the selection after each element in the set of matched
+// elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) AfterSelection(sel *Selection) *Selection {
+	return s.AfterNodes(sel.Nodes...)
+}
+
+// AfterHtml parses the html and inserts it after the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) AfterHtml(html string) *Selection {
+	return s.AfterNodes(parseHtml(html)...)
+}
+
+// AfterNodes inserts the nodes after each element in the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) AfterNodes(ns ...*html.Node) *Selection {
+	return s.manipulateNodes(ns, true, func(sn *html.Node, n *html.Node) {
+		if sn.Parent != nil {
+			sn.Parent.InsertBefore(n, sn.NextSibling)
+		}
+	})
+}
+
+// Append appends the elements specified by the selector to the end of each element
+// in the set of matched elements, following those rules:
+//
+// 1) The selector is applied to the root document.
+//
+// 2) Elements that are part of the document will be moved to the new location.
+//
+// 3) If there are multiple locations to append to, cloned nodes will be
+// appended to all target locations except the last one, which will be moved
+// as noted in (2).
+func (s *Selection) Append(selector string) *Selection {
+	return s.AppendMatcher(compileMatcher(selector))
+}
+
+// AppendMatcher appends the elements specified by the matcher to the end of each element
+// in the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) AppendMatcher(m Matcher) *Selection {
+	return s.AppendNodes(m.MatchAll(s.document.rootNode)...)
+}
+
+// AppendSelection appends the elements in the selection to the end of each element
+// in the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) AppendSelection(sel *Selection) *Selection {
+	return s.AppendNodes(sel.Nodes...)
+}
+
+// AppendHtml parses the html and appends it to the set of matched elements.
+func (s *Selection) AppendHtml(html string) *Selection {
+	return s.AppendNodes(parseHtml(html)...)
+}
+
+// AppendNodes appends the specified nodes to each node in the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) AppendNodes(ns ...*html.Node) *Selection {
+	return s.manipulateNodes(ns, false, func(sn *html.Node, n *html.Node) {
+		sn.AppendChild(n)
+	})
+}
+
+// Before inserts the matched elements before each element in the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) Before(selector string) *Selection {
+	return s.BeforeMatcher(compileMatcher(selector))
+}
+
+// BeforeMatcher inserts the matched elements before each element in the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) BeforeMatcher(m Matcher) *Selection {
+	return s.BeforeNodes(m.MatchAll(s.document.rootNode)...)
+}
+
+// BeforeSelection inserts the elements in the selection before each element in the set of matched
+// elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) BeforeSelection(sel *Selection) *Selection {
+	return s.BeforeNodes(sel.Nodes...)
+}
+
+// BeforeHtml parses the html and inserts it before the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) BeforeHtml(html string) *Selection {
+	return s.BeforeNodes(parseHtml(html)...)
+}
+
+// BeforeNodes inserts the nodes before each element in the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) BeforeNodes(ns ...*html.Node) *Selection {
+	return s.manipulateNodes(ns, false, func(sn *html.Node, n *html.Node) {
+		if sn.Parent != nil {
+			sn.Parent.InsertBefore(n, sn)
+		}
+	})
+}
+
+// Clone creates a deep copy of the set of matched nodes. The new nodes will not be
+// attached to the document.
+func (s *Selection) Clone() *Selection {
+	ns := newEmptySelection(s.document)
+	ns.Nodes = cloneNodes(s.Nodes)
+	return ns
+}
+
+// Empty removes all children nodes from the set of matched elements.
+// It returns the children nodes in a new Selection.
+func (s *Selection) Empty() *Selection {
+	var nodes []*html.Node
+
+	for _, n := range s.Nodes {
+		for c := n.FirstChild; c != nil; c = n.FirstChild {
+			n.RemoveChild(c)
+			nodes = append(nodes, c)
+		}
+	}
+
+	return pushStack(s, nodes)
+}
+
+// Prepend prepends the elements specified by the selector to each element in
+// the set of matched elements, following the same rules as Append.
+func (s *Selection) Prepend(selector string) *Selection {
+	return s.PrependMatcher(compileMatcher(selector))
+}
+
+// PrependMatcher prepends the elements specified by the matcher to each
+// element in the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) PrependMatcher(m Matcher) *Selection {
+	return s.PrependNodes(m.MatchAll(s.document.rootNode)...)
+}
+
+// PrependSelection prepends the elements in the selection to each element in
+// the set of matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) PrependSelection(sel *Selection) *Selection {
+	return s.PrependNodes(sel.Nodes...)
+}
+
+// PrependHtml parses the html and prepends it to the set of matched elements.
+func (s *Selection) PrependHtml(html string) *Selection {
+	return s.PrependNodes(parseHtml(html)...)
+}
+
+// PrependNodes prepends the specified nodes to each node in the set of
+// matched elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) PrependNodes(ns ...*html.Node) *Selection {
+	return s.manipulateNodes(ns, true, func(sn *html.Node, n *html.Node) {
+		// sn.FirstChild may be nil, in which case this functions like
+		// sn.AppendChild()
+		sn.InsertBefore(n, sn.FirstChild)
+	})
+}
+
+// Remove removes the set of matched elements from the document.
+// It returns the same selection, now consisting of nodes not in the document.
+func (s *Selection) Remove() *Selection {
+	for _, n := range s.Nodes {
+		if n.Parent != nil {
+			n.Parent.RemoveChild(n)
+		}
+	}
+
+	return s
+}
+
+// RemoveFiltered removes the set of matched elements by selector.
+// It returns the Selection of removed nodes.
+func (s *Selection) RemoveFiltered(selector string) *Selection {
+	return s.RemoveMatcher(compileMatcher(selector))
+}
+
+// RemoveMatcher removes the set of matched elements.
+// It returns the Selection of removed nodes.
+func (s *Selection) RemoveMatcher(m Matcher) *Selection {
+	return s.FilterMatcher(m).Remove()
+}
+
+// ReplaceWith replaces each element in the set of matched elements with the
+// nodes matched by the given selector.
+// It returns the removed elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) ReplaceWith(selector string) *Selection {
+	return s.ReplaceWithMatcher(compileMatcher(selector))
+}
+
+// ReplaceWithMatcher replaces each element in the set of matched elements with
+// the nodes matched by the given Matcher.
+// It returns the removed elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) ReplaceWithMatcher(m Matcher) *Selection {
+	return s.ReplaceWithNodes(m.MatchAll(s.document.rootNode)...)
+}
+
+// ReplaceWithSelection replaces each element in the set of matched elements with
+// the nodes from the given Selection.
+// It returns the removed elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) ReplaceWithSelection(sel *Selection) *Selection {
+	return s.ReplaceWithNodes(sel.Nodes...)
+}
+
+// ReplaceWithHtml replaces each element in the set of matched elements with
+// the parsed HTML.
+// It returns the removed elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) ReplaceWithHtml(html string) *Selection {
+	return s.ReplaceWithNodes(parseHtml(html)...)
+}
+
+// ReplaceWithNodes replaces each element in the set of matched elements with
+// the given nodes.
+// It returns the removed elements.
+//
+// This follows the same rules as Selection.Append.
+func (s *Selection) ReplaceWithNodes(ns ...*html.Node) *Selection {
+	s.AfterNodes(ns...)
+	return s.Remove()
+}
+
+// SetHtml sets the html content of each element in the selection to
+// specified html string.
+func (s *Selection) SetHtml(html string) *Selection {
+	return setHtmlNodes(s, parseHtml(html)...)
+}
+
+// SetText sets the content of each element in the selection to specified content.
+// The provided text string is escaped.
+func (s *Selection) SetText(text string) *Selection {
+	return s.SetHtml(html.EscapeString(text))
+}
+
+// Unwrap removes the parents of the set of matched elements, leaving the matched
+// elements (and their siblings, if any) in their place.
+// It returns the original selection.
+func (s *Selection) Unwrap() *Selection {
+	s.Parent().Each(func(i int, ss *Selection) {
+		// For some reason, jquery allows unwrap to remove the <head> element, so
+		// allowing it here too. Same for <html>. Why it allows those elements to
+		// be unwrapped while not allowing body is a mystery to me.
+		if ss.Nodes[0].Data != "body" {
+			ss.ReplaceWithSelection(ss.Contents())
+		}
+	})
+
+	return s
+}
+
+// Wrap wraps each element in the set of matched elements inside the first
+// element matched by the given selector. The matched child is cloned before
+// being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) Wrap(selector string) *Selection {
+	return s.WrapMatcher(compileMatcher(selector))
+}
+
+// WrapMatcher wraps each element in the set of matched elements inside the
+// first element matched by the given matcher. The matched child is cloned
+// before being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapMatcher(m Matcher) *Selection {
+	return s.wrapNodes(m.MatchAll(s.document.rootNode)...)
+}
+
+// WrapSelection wraps each element in the set of matched elements inside the
+// first element in the given Selection. The element is cloned before being
+// inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapSelection(sel *Selection) *Selection {
+	return s.wrapNodes(sel.Nodes...)
+}
+
+// WrapHtml wraps each element in the set of matched elements inside the inner-
+// most child of the given HTML.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapHtml(html string) *Selection {
+	return s.wrapNodes(parseHtml(html)...)
+}
+
+// WrapNode wraps each element in the set of matched elements inside the inner-
+// most child of the given node. The given node is copied before being inserted
+// into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapNode(n *html.Node) *Selection {
+	return s.wrapNodes(n)
+}
+
+func (s *Selection) wrapNodes(ns ...*html.Node) *Selection {
+	s.Each(func(i int, ss *Selection) {
+		ss.wrapAllNodes(ns...)
+	})
+
+	return s
+}
+
+// WrapAll wraps a single HTML structure, matched by the given selector, around
+// all elements in the set of matched elements. The matched child is cloned
+// before being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapAll(selector string) *Selection {
+	return s.WrapAllMatcher(compileMatcher(selector))
+}
+
+// WrapAllMatcher wraps a single HTML structure, matched by the given Matcher,
+// around all elements in the set of matched elements. The matched child is
+// cloned before being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapAllMatcher(m Matcher) *Selection {
+	return s.wrapAllNodes(m.MatchAll(s.document.rootNode)...)
+}
+
+// WrapAllSelection wraps a single HTML structure, the first node of the given
+// Selection, around all elements in the set of matched elements. The matched
+// child is cloned before being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapAllSelection(sel *Selection) *Selection {
+	return s.wrapAllNodes(sel.Nodes...)
+}
+
+// WrapAllHtml wraps the given HTML structure around all elements in the set of
+// matched elements. The matched child is cloned before being inserted into the
+// document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapAllHtml(html string) *Selection {
+	return s.wrapAllNodes(parseHtml(html)...)
+}
+
+func (s *Selection) wrapAllNodes(ns ...*html.Node) *Selection {
+	if len(ns) > 0 {
+		return s.WrapAllNode(ns[0])
+	}
+	return s
+}
+
+// WrapAllNode wraps the given node around the first element in the Selection,
+// making all other nodes in the Selection children of the given node. The node
+// is cloned before being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapAllNode(n *html.Node) *Selection {
+	if s.Size() == 0 {
+		return s
+	}
+
+	wrap := cloneNode(n)
+
+	first := s.Nodes[0]
+	if first.Parent != nil {
+		first.Parent.InsertBefore(wrap, first)
+		first.Parent.RemoveChild(first)
+	}
+
+	for c := getFirstChildEl(wrap); c != nil; c = getFirstChildEl(wrap) {
+		wrap = c
+	}
+
+	newSingleSelection(wrap, s.document).AppendSelection(s)
+
+	return s
+}
+
+// WrapInner wraps an HTML structure, matched by the given selector, around the
+// content of element in the set of matched elements. The matched child is
+// cloned before being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapInner(selector string) *Selection {
+	return s.WrapInnerMatcher(compileMatcher(selector))
+}
+
+// WrapInnerMatcher wraps an HTML structure, matched by the given selector,
+// around the content of element in the set of matched elements. The matched
+// child is cloned before being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapInnerMatcher(m Matcher) *Selection {
+	return s.wrapInnerNodes(m.MatchAll(s.document.rootNode)...)
+}
+
+// WrapInnerSelection wraps an HTML structure, matched by the given selector,
+// around the content of element in the set of matched elements. The matched
+// child is cloned before being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapInnerSelection(sel *Selection) *Selection {
+	return s.wrapInnerNodes(sel.Nodes...)
+}
+
+// WrapInnerHtml wraps an HTML structure, matched by the given selector, around
+// the content of element in the set of matched elements. The matched child is
+// cloned before being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapInnerHtml(html string) *Selection {
+	return s.wrapInnerNodes(parseHtml(html)...)
+}
+
+// WrapInnerNode wraps an HTML structure, matched by the given selector, around
+// the content of element in the set of matched elements. The matched child is
+// cloned before being inserted into the document.
+//
+// It returns the original set of elements.
+func (s *Selection) WrapInnerNode(n *html.Node) *Selection {
+	return s.wrapInnerNodes(n)
+}
+
+func (s *Selection) wrapInnerNodes(ns ...*html.Node) *Selection {
+	if len(ns) == 0 {
+		return s
+	}
+
+	s.Each(func(i int, s *Selection) {
+		contents := s.Contents()
+
+		if contents.Size() > 0 {
+			contents.wrapAllNodes(ns...)
+		} else {
+			s.AppendNodes(cloneNode(ns[0]))
+		}
+	})
+
+	return s
+}
+
+func parseHtml(h string) []*html.Node {
+	// Errors are only returned when the io.Reader returns any error besides
+	// EOF, but strings.Reader never will
+	nodes, err := html.ParseFragment(strings.NewReader(h), &html.Node{Type: html.ElementNode})
+	if err != nil {
+		panic("goquery: failed to parse HTML: " + err.Error())
+	}
+	return nodes
+}
+
+func setHtmlNodes(s *Selection, ns ...*html.Node) *Selection {
+	for _, n := range s.Nodes {
+		for c := n.FirstChild; c != nil; c = n.FirstChild {
+			n.RemoveChild(c)
+		}
+		for _, c := range ns {
+			n.AppendChild(cloneNode(c))
+		}
+	}
+	return s
+}
+
+// Get the first child that is an ElementNode
+func getFirstChildEl(n *html.Node) *html.Node {
+	c := n.FirstChild
+	for c != nil && c.Type != html.ElementNode {
+		c = c.NextSibling
+	}
+	return c
+}
+
+// Deep copy a slice of nodes.
+func cloneNodes(ns []*html.Node) []*html.Node {
+	cns := make([]*html.Node, 0, len(ns))
+
+	for _, n := range ns {
+		cns = append(cns, cloneNode(n))
+	}
+
+	return cns
+}
+
+// Deep copy a node. The new node has clones of all the original node's
+// children but none of its parents or siblings.
+func cloneNode(n *html.Node) *html.Node {
+	nn := &html.Node{
+		Type:     n.Type,
+		DataAtom: n.DataAtom,
+		Data:     n.Data,
+		Attr:     make([]html.Attribute, len(n.Attr)),
+	}
+
+	copy(nn.Attr, n.Attr)
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		nn.AppendChild(cloneNode(c))
+	}
+
+	return nn
+}
+
+func (s *Selection) manipulateNodes(ns []*html.Node, reverse bool,
+	f func(sn *html.Node, n *html.Node)) *Selection {
+
+	lasti := s.Size() - 1
+
+	// net.Html doesn't provide document fragments for insertion, so to get
+	// things in the correct order with After() and Prepend(), the callback
+	// needs to be called on the reverse of the nodes.
+	if reverse {
+		for i, j := 0, len(ns)-1; i < j; i, j = i+1, j-1 {
+			ns[i], ns[j] = ns[j], ns[i]
+		}
+	}
+
+	for i, sn := range s.Nodes {
+		for _, n := range ns {
+			if i != lasti {
+				f(sn, cloneNode(n))
+			} else {
+				if n.Parent != nil {
+					n.Parent.RemoveChild(n)
+				}
+				f(sn, n)
+			}
+		}
+	}
+
+	return s
+}

vendor/github.com/PuerkitoBio/goquery/property.go

@@ -0,0 +1,275 @@
+package goquery
+
+import (
+	"bytes"
+	"regexp"
+	"strings"
+
+	"golang.org/x/net/html"
+)
+
+var rxClassTrim = regexp.MustCompile("[\t\r\n]")
+
+// Attr gets the specified attribute's value for the first element in the
+// Selection. To get the value for each element individually, use a looping
+// construct such as Each or Map method.
+func (s *Selection) Attr(attrName string) (val string, exists bool) {
+	if len(s.Nodes) == 0 {
+		return
+	}
+	return getAttributeValue(attrName, s.Nodes[0])
+}
+
+// AttrOr works like Attr but returns default value if attribute is not present.
+func (s *Selection) AttrOr(attrName, defaultValue string) string {
+	if len(s.Nodes) == 0 {
+		return defaultValue
+	}
+
+	val, exists := getAttributeValue(attrName, s.Nodes[0])
+	if !exists {
+		return defaultValue
+	}
+
+	return val
+}
+
+// RemoveAttr removes the named attribute from each element in the set of matched elements.
+func (s *Selection) RemoveAttr(attrName string) *Selection {
+	for _, n := range s.Nodes {
+		removeAttr(n, attrName)
+	}
+
+	return s
+}
+
+// SetAttr sets the given attribute on each element in the set of matched elements.
+func (s *Selection) SetAttr(attrName, val string) *Selection {
+	for _, n := range s.Nodes {
+		attr := getAttributePtr(attrName, n)
+		if attr == nil {
+			n.Attr = append(n.Attr, html.Attribute{Key: attrName, Val: val})
+		} else {
+			attr.Val = val
+		}
+	}
+
+	return s
+}
+
+// Text gets the combined text contents of each element in the set of matched
+// elements, including their descendants.
+func (s *Selection) Text() string {
+	var buf bytes.Buffer
+
+	// Slightly optimized vs calling Each: no single selection object created
+	var f func(*html.Node)
+	f = func(n *html.Node) {
+		if n.Type == html.TextNode {
+			// Keep newlines and spaces, like jQuery
+			buf.WriteString(n.Data)
+		}
+		if n.FirstChild != nil {
+			for c := n.FirstChild; c != nil; c = c.NextSibling {
+				f(c)
+			}
+		}
+	}
+	for _, n := range s.Nodes {
+		f(n)
+	}
+
+	return buf.String()
+}
+
+// Size is an alias for Length.
+func (s *Selection) Size() int {
+	return s.Length()
+}
+
+// Length returns the number of elements in the Selection object.
+func (s *Selection) Length() int {
+	return len(s.Nodes)
+}
+
+// Html gets the HTML contents of the first element in the set of matched
+// elements. It includes text and comment nodes.
+func (s *Selection) Html() (ret string, e error) {
+	// Since there is no .innerHtml, the HTML content must be re-created from
+	// the nodes using html.Render.
+	var buf bytes.Buffer
+
+	if len(s.Nodes) > 0 {
+		for c := s.Nodes[0].FirstChild; c != nil; c = c.NextSibling {
+			e = html.Render(&buf, c)
+			if e != nil {
+				return
+			}
+		}
+		ret = buf.String()
+	}
+
+	return
+}
+
+// AddClass adds the given class(es) to each element in the set of matched elements.
+// Multiple class names can be specified, separated by a space or via multiple arguments.
+func (s *Selection) AddClass(class ...string) *Selection {
+	classStr := strings.TrimSpace(strings.Join(class, " "))
+
+	if classStr == "" {
+		return s
+	}
+
+	tcls := getClassesSlice(classStr)
+	for _, n := range s.Nodes {
+		curClasses, attr := getClassesAndAttr(n, true)
+		for _, newClass := range tcls {
+			if !strings.Contains(curClasses, " "+newClass+" ") {
+				curClasses += newClass + " "
+			}
+		}
+
+		setClasses(n, attr, curClasses)
+	}
+
+	return s
+}
+
+// HasClass determines whether any of the matched elements are assigned the
+// given class.
+func (s *Selection) HasClass(class string) bool {
+	class = " " + class + " "
+	for _, n := range s.Nodes {
+		classes, _ := getClassesAndAttr(n, false)
+		if strings.Contains(classes, class) {
+			return true
+		}
+	}
+	return false
+}
+
+// RemoveClass removes the given class(es) from each element in the set of matched elements.
+// Multiple class names can be specified, separated by a space or via multiple arguments.
+// If no class name is provided, all classes are removed.
+func (s *Selection) RemoveClass(class ...string) *Selection {
+	var rclasses []string
+
+	classStr := strings.TrimSpace(strings.Join(class, " "))
+	remove := classStr == ""
+
+	if !remove {
+		rclasses = getClassesSlice(classStr)
+	}
+
+	for _, n := range s.Nodes {
+		if remove {
+			removeAttr(n, "class")
+		} else {
+			classes, attr := getClassesAndAttr(n, true)
+			for _, rcl := range rclasses {
+				classes = strings.Replace(classes, " "+rcl+" ", " ", -1)
+			}
+
+			setClasses(n, attr, classes)
+		}
+	}
+
+	return s
+}
+
+// ToggleClass adds or removes the given class(es) for each element in the set of matched elements.
+// Multiple class names can be specified, separated by a space or via multiple arguments.
+func (s *Selection) ToggleClass(class ...string) *Selection {
+	classStr := strings.TrimSpace(strings.Join(class, " "))
+
+	if classStr == "" {
+		return s
+	}
+
+	tcls := getClassesSlice(classStr)
+
+	for _, n := range s.Nodes {
+		classes, attr := getClassesAndAttr(n, true)
+		for _, tcl := range tcls {
+			if strings.Contains(classes, " "+tcl+" ") {
+				classes = strings.Replace(classes, " "+tcl+" ", " ", -1)
+			} else {
+				classes += tcl + " "
+			}
+		}
+
+		setClasses(n, attr, classes)
+	}
+
+	return s
+}
+
+func getAttributePtr(attrName string, n *html.Node) *html.Attribute {
+	if n == nil {
+		return nil
+	}
+
+	for i, a := range n.Attr {
+		if a.Key == attrName {
+			return &n.Attr[i]
+		}
+	}
+	return nil
+}
+
+// Private function to get the specified attribute's value from a node.
+func getAttributeValue(attrName string, n *html.Node) (val string, exists bool) {
+	if a := getAttributePtr(attrName, n); a != nil {
+		val = a.Val
+		exists = true
+	}
+	return
+}
+
+// Get and normalize the "class" attribute from the node.
+func getClassesAndAttr(n *html.Node, create bool) (classes string, attr *html.Attribute) {
+	// Applies only to element nodes
+	if n.Type == html.ElementNode {
+		attr = getAttributePtr("class", n)
+		if attr == nil && create {
+			n.Attr = append(n.Attr, html.Attribute{
+				Key: "class",
+				Val: "",
+			})
+			attr = &n.Attr[len(n.Attr)-1]
+		}
+	}
+
+	if attr == nil {
+		classes = " "
+	} else {
+		classes = rxClassTrim.ReplaceAllString(" "+attr.Val+" ", " ")
+	}
+
+	return
+}
+
+func getClassesSlice(classes string) []string {
+	return strings.Split(rxClassTrim.ReplaceAllString(" "+classes+" ", " "), " ")
+}
+
+func removeAttr(n *html.Node, attrName string) {
+	for i, a := range n.Attr {
+		if a.Key == attrName {
+			n.Attr[i], n.Attr[len(n.Attr)-1], n.Attr =
+				n.Attr[len(n.Attr)-1], html.Attribute{}, n.Attr[:len(n.Attr)-1]
+			return
+		}
+	}
+}
+
+func setClasses(n *html.Node, attr *html.Attribute, classes string) {
+	classes = strings.TrimSpace(classes)
+	if classes == "" {
+		removeAttr(n, "class")
+		return
+	}
+
+	attr.Val = classes
+}

vendor/github.com/PuerkitoBio/goquery/query.go

@@ -0,0 +1,49 @@
+package goquery
+
+import "golang.org/x/net/html"
+
+// Is checks the current matched set of elements against a selector and
+// returns true if at least one of these elements matches.
+func (s *Selection) Is(selector string) bool {
+	return s.IsMatcher(compileMatcher(selector))
+}
+
+// IsMatcher checks the current matched set of elements against a matcher and
+// returns true if at least one of these elements matches.
+func (s *Selection) IsMatcher(m Matcher) bool {
+	if len(s.Nodes) > 0 {
+		if len(s.Nodes) == 1 {
+			return m.Match(s.Nodes[0])
+		}
+		return len(m.Filter(s.Nodes)) > 0
+	}
+
+	return false
+}
+
+// IsFunction checks the current matched set of elements against a predicate and
+// returns true if at least one of these elements matches.
+func (s *Selection) IsFunction(f func(int, *Selection) bool) bool {
+	return s.FilterFunction(f).Length() > 0
+}
+
+// IsSelection checks the current matched set of elements against a Selection object
+// and returns true if at least one of these elements matches.
+func (s *Selection) IsSelection(sel *Selection) bool {
+	return s.FilterSelection(sel).Length() > 0
+}
+
+// IsNodes checks the current matched set of elements against the specified nodes
+// and returns true if at least one of these elements matches.
+func (s *Selection) IsNodes(nodes ...*html.Node) bool {
+	return s.FilterNodes(nodes...).Length() > 0
+}
+
+// Contains returns true if the specified Node is within,
+// at any depth, one of the nodes in the Selection object.
+// It is NOT inclusive, to behave like jQuery's implementation, and
+// unlike Javascript's .contains, so if the contained
+// node is itself in the selection, it returns false.
+func (s *Selection) Contains(n *html.Node) bool {
+	return sliceContains(s.Nodes, n)
+}

vendor/github.com/PuerkitoBio/goquery/traversal.go

@@ -0,0 +1,698 @@
+package goquery
+
+import "golang.org/x/net/html"
+
+type siblingType int
+
+// Sibling type, used internally when iterating over children at the same
+// level (siblings) to specify which nodes are requested.
+const (
+	siblingPrevUntil siblingType = iota - 3
+	siblingPrevAll
+	siblingPrev
+	siblingAll
+	siblingNext
+	siblingNextAll
+	siblingNextUntil
+	siblingAllIncludingNonElements
+)
+
+// Find gets the descendants of each element in the current set of matched
+// elements, filtered by a selector. It returns a new Selection object
+// containing these matched elements.
+func (s *Selection) Find(selector string) *Selection {
+	return pushStack(s, findWithMatcher(s.Nodes, compileMatcher(selector)))
+}
+
+// FindMatcher gets the descendants of each element in the current set of matched
+// elements, filtered by the matcher. It returns a new Selection object
+// containing these matched elements.
+func (s *Selection) FindMatcher(m Matcher) *Selection {
+	return pushStack(s, findWithMatcher(s.Nodes, m))
+}
+
+// FindSelection gets the descendants of each element in the current
+// Selection, filtered by a Selection. It returns a new Selection object
+// containing these matched elements.
+func (s *Selection) FindSelection(sel *Selection) *Selection {
+	if sel == nil {
+		return pushStack(s, nil)
+	}
+	return s.FindNodes(sel.Nodes...)
+}
+
+// FindNodes gets the descendants of each element in the current
+// Selection, filtered by some nodes. It returns a new Selection object
+// containing these matched elements.
+func (s *Selection) FindNodes(nodes ...*html.Node) *Selection {
+	return pushStack(s, mapNodes(nodes, func(i int, n *html.Node) []*html.Node {
+		if sliceContains(s.Nodes, n) {
+			return []*html.Node{n}
+		}
+		return nil
+	}))
+}
+
+// Contents gets the children of each element in the Selection,
+// including text and comment nodes. It returns a new Selection object
+// containing these elements.
+func (s *Selection) Contents() *Selection {
+	return pushStack(s, getChildrenNodes(s.Nodes, siblingAllIncludingNonElements))
+}
+
+// ContentsFiltered gets the children of each element in the Selection,
+// filtered by the specified selector. It returns a new Selection
+// object containing these elements. Since selectors only act on Element nodes,
+// this function is an alias to ChildrenFiltered unless the selector is empty,
+// in which case it is an alias to Contents.
+func (s *Selection) ContentsFiltered(selector string) *Selection {
+	if selector != "" {
+		return s.ChildrenFiltered(selector)
+	}
+	return s.Contents()
+}
+
+// ContentsMatcher gets the children of each element in the Selection,
+// filtered by the specified matcher. It returns a new Selection
+// object containing these elements. Since matchers only act on Element nodes,
+// this function is an alias to ChildrenMatcher.
+func (s *Selection) ContentsMatcher(m Matcher) *Selection {
+	return s.ChildrenMatcher(m)
+}
+
+// Children gets the child elements of each element in the Selection.
+// It returns a new Selection object containing these elements.
+func (s *Selection) Children() *Selection {
+	return pushStack(s, getChildrenNodes(s.Nodes, siblingAll))
+}
+
+// ChildrenFiltered gets the child elements of each element in the Selection,
+// filtered by the specified selector. It returns a new
+// Selection object containing these elements.
+func (s *Selection) ChildrenFiltered(selector string) *Selection {
+	return filterAndPush(s, getChildrenNodes(s.Nodes, siblingAll), compileMatcher(selector))
+}
+
+// ChildrenMatcher gets the child elements of each element in the Selection,
+// filtered by the specified matcher. It returns a new
+// Selection object containing these elements.
+func (s *Selection) ChildrenMatcher(m Matcher) *Selection {
+	return filterAndPush(s, getChildrenNodes(s.Nodes, siblingAll), m)
+}
+
+// Parent gets the parent of each element in the Selection. It returns a
+// new Selection object containing the matched elements.
+func (s *Selection) Parent() *Selection {
+	return pushStack(s, getParentNodes(s.Nodes))
+}
+
+// ParentFiltered gets the parent of each element in the Selection filtered by a
+// selector. It returns a new Selection object containing the matched elements.
+func (s *Selection) ParentFiltered(selector string) *Selection {
+	return filterAndPush(s, getParentNodes(s.Nodes), compileMatcher(selector))
+}
+
+// ParentMatcher gets the parent of each element in the Selection filtered by a
+// matcher. It returns a new Selection object containing the matched elements.
+func (s *Selection) ParentMatcher(m Matcher) *Selection {
+	return filterAndPush(s, getParentNodes(s.Nodes), m)
+}
+
+// Closest gets the first element that matches the selector by testing the
+// element itself and traversing up through its ancestors in the DOM tree.
+func (s *Selection) Closest(selector string) *Selection {
+	cs := compileMatcher(selector)
+	return s.ClosestMatcher(cs)
+}
+
+// ClosestMatcher gets the first element that matches the matcher by testing the
+// element itself and traversing up through its ancestors in the DOM tree.
+func (s *Selection) ClosestMatcher(m Matcher) *Selection {
+	return pushStack(s, mapNodes(s.Nodes, func(i int, n *html.Node) []*html.Node {
+		// For each node in the selection, test the node itself, then each parent
+		// until a match is found.
+		for ; n != nil; n = n.Parent {
+			if m.Match(n) {
+				return []*html.Node{n}
+			}
+		}
+		return nil
+	}))
+}
+
+// ClosestNodes gets the first element that matches one of the nodes by testing the
+// element itself and traversing up through its ancestors in the DOM tree.
+func (s *Selection) ClosestNodes(nodes ...*html.Node) *Selection {
+	set := make(map[*html.Node]bool)
+	for _, n := range nodes {
+		set[n] = true
+	}
+	return pushStack(s, mapNodes(s.Nodes, func(i int, n *html.Node) []*html.Node {
+		// For each node in the selection, test the node itself, then each parent
+		// until a match is found.
+		for ; n != nil; n = n.Parent {
+			if set[n] {
+				return []*html.Node{n}
+			}
+		}
+		return nil
+	}))
+}
+
+// ClosestSelection gets the first element that matches one of the nodes in the
+// Selection by testing the element itself and traversing up through its ancestors
+// in the DOM tree.
+func (s *Selection) ClosestSelection(sel *Selection) *Selection {
+	if sel == nil {
+		return pushStack(s, nil)
+	}
+	return s.ClosestNodes(sel.Nodes...)
+}
+
+// Parents gets the ancestors of each element in the current Selection. It
+// returns a new Selection object with the matched elements.
+func (s *Selection) Parents() *Selection {
+	return pushStack(s, getParentsNodes(s.Nodes, nil, nil))
+}
+
+// ParentsFiltered gets the ancestors of each element in the current
+// Selection. It returns a new Selection object with the matched elements.
+func (s *Selection) ParentsFiltered(selector string) *Selection {
+	return filterAndPush(s, getParentsNodes(s.Nodes, nil, nil), compileMatcher(selector))
+}
+
+// ParentsMatcher gets the ancestors of each element in the current
+// Selection. It returns a new Selection object with the matched elements.
+func (s *Selection) ParentsMatcher(m Matcher) *Selection {
+	return filterAndPush(s, getParentsNodes(s.Nodes, nil, nil), m)
+}
+
+// ParentsUntil gets the ancestors of each element in the Selection, up to but
+// not including the element matched by the selector. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) ParentsUntil(selector string) *Selection {
+	return pushStack(s, getParentsNodes(s.Nodes, compileMatcher(selector), nil))
+}
+
+// ParentsUntilMatcher gets the ancestors of each element in the Selection, up to but
+// not including the element matched by the matcher. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) ParentsUntilMatcher(m Matcher) *Selection {
+	return pushStack(s, getParentsNodes(s.Nodes, m, nil))
+}
+
+// ParentsUntilSelection gets the ancestors of each element in the Selection,
+// up to but not including the elements in the specified Selection. It returns a
+// new Selection object containing the matched elements.
+func (s *Selection) ParentsUntilSelection(sel *Selection) *Selection {
+	if sel == nil {
+		return s.Parents()
+	}
+	return s.ParentsUntilNodes(sel.Nodes...)
+}
+
+// ParentsUntilNodes gets the ancestors of each element in the Selection,
+// up to but not including the specified nodes. It returns a
+// new Selection object containing the matched elements.
+func (s *Selection) ParentsUntilNodes(nodes ...*html.Node) *Selection {
+	return pushStack(s, getParentsNodes(s.Nodes, nil, nodes))
+}
+
+// ParentsFilteredUntil is like ParentsUntil, with the option to filter the
+// results based on a selector string. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) ParentsFilteredUntil(filterSelector, untilSelector string) *Selection {
+	return filterAndPush(s, getParentsNodes(s.Nodes, compileMatcher(untilSelector), nil), compileMatcher(filterSelector))
+}
+
+// ParentsFilteredUntilMatcher is like ParentsUntilMatcher, with the option to filter the
+// results based on a matcher. It returns a new Selection object containing the matched elements.
+func (s *Selection) ParentsFilteredUntilMatcher(filter, until Matcher) *Selection {
+	return filterAndPush(s, getParentsNodes(s.Nodes, until, nil), filter)
+}
+
+// ParentsFilteredUntilSelection is like ParentsUntilSelection, with the
+// option to filter the results based on a selector string. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) ParentsFilteredUntilSelection(filterSelector string, sel *Selection) *Selection {
+	return s.ParentsMatcherUntilSelection(compileMatcher(filterSelector), sel)
+}
+
+// ParentsMatcherUntilSelection is like ParentsUntilSelection, with the
+// option to filter the results based on a matcher. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) ParentsMatcherUntilSelection(filter Matcher, sel *Selection) *Selection {
+	if sel == nil {
+		return s.ParentsMatcher(filter)
+	}
+	return s.ParentsMatcherUntilNodes(filter, sel.Nodes...)
+}
+
+// ParentsFilteredUntilNodes is like ParentsUntilNodes, with the
+// option to filter the results based on a selector string. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) ParentsFilteredUntilNodes(filterSelector string, nodes ...*html.Node) *Selection {
+	return filterAndPush(s, getParentsNodes(s.Nodes, nil, nodes), compileMatcher(filterSelector))
+}
+
+// ParentsMatcherUntilNodes is like ParentsUntilNodes, with the
+// option to filter the results based on a matcher. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) ParentsMatcherUntilNodes(filter Matcher, nodes ...*html.Node) *Selection {
+	return filterAndPush(s, getParentsNodes(s.Nodes, nil, nodes), filter)
+}
+
+// Siblings gets the siblings of each element in the Selection. It returns
+// a new Selection object containing the matched elements.
+func (s *Selection) Siblings() *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingAll, nil, nil))
+}
+
+// SiblingsFiltered gets the siblings of each element in the Selection
+// filtered by a selector. It returns a new Selection object containing the
+// matched elements.
+func (s *Selection) SiblingsFiltered(selector string) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingAll, nil, nil), compileMatcher(selector))
+}
+
+// SiblingsMatcher gets the siblings of each element in the Selection
+// filtered by a matcher. It returns a new Selection object containing the
+// matched elements.
+func (s *Selection) SiblingsMatcher(m Matcher) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingAll, nil, nil), m)
+}
+
+// Next gets the immediately following sibling of each element in the
+// Selection. It returns a new Selection object containing the matched elements.
+func (s *Selection) Next() *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingNext, nil, nil))
+}
+
+// NextFiltered gets the immediately following sibling of each element in the
+// Selection filtered by a selector. It returns a new Selection object
+// containing the matched elements.
+func (s *Selection) NextFiltered(selector string) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingNext, nil, nil), compileMatcher(selector))
+}
+
+// NextMatcher gets the immediately following sibling of each element in the
+// Selection filtered by a matcher. It returns a new Selection object
+// containing the matched elements.
+func (s *Selection) NextMatcher(m Matcher) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingNext, nil, nil), m)
+}
+
+// NextAll gets all the following siblings of each element in the
+// Selection. It returns a new Selection object containing the matched elements.
+func (s *Selection) NextAll() *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingNextAll, nil, nil))
+}
+
+// NextAllFiltered gets all the following siblings of each element in the
+// Selection filtered by a selector. It returns a new Selection object
+// containing the matched elements.
+func (s *Selection) NextAllFiltered(selector string) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingNextAll, nil, nil), compileMatcher(selector))
+}
+
+// NextAllMatcher gets all the following siblings of each element in the
+// Selection filtered by a matcher. It returns a new Selection object
+// containing the matched elements.
+func (s *Selection) NextAllMatcher(m Matcher) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingNextAll, nil, nil), m)
+}
+
+// Prev gets the immediately preceding sibling of each element in the
+// Selection. It returns a new Selection object containing the matched elements.
+func (s *Selection) Prev() *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingPrev, nil, nil))
+}
+
+// PrevFiltered gets the immediately preceding sibling of each element in the
+// Selection filtered by a selector. It returns a new Selection object
+// containing the matched elements.
+func (s *Selection) PrevFiltered(selector string) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingPrev, nil, nil), compileMatcher(selector))
+}
+
+// PrevMatcher gets the immediately preceding sibling of each element in the
+// Selection filtered by a matcher. It returns a new Selection object
+// containing the matched elements.
+func (s *Selection) PrevMatcher(m Matcher) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingPrev, nil, nil), m)
+}
+
+// PrevAll gets all the preceding siblings of each element in the
+// Selection. It returns a new Selection object containing the matched elements.
+func (s *Selection) PrevAll() *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingPrevAll, nil, nil))
+}
+
+// PrevAllFiltered gets all the preceding siblings of each element in the
+// Selection filtered by a selector. It returns a new Selection object
+// containing the matched elements.
+func (s *Selection) PrevAllFiltered(selector string) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingPrevAll, nil, nil), compileMatcher(selector))
+}
+
+// PrevAllMatcher gets all the preceding siblings of each element in the
+// Selection filtered by a matcher. It returns a new Selection object
+// containing the matched elements.
+func (s *Selection) PrevAllMatcher(m Matcher) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingPrevAll, nil, nil), m)
+}
+
+// NextUntil gets all following siblings of each element up to but not
+// including the element matched by the selector. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) NextUntil(selector string) *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingNextUntil,
+		compileMatcher(selector), nil))
+}
+
+// NextUntilMatcher gets all following siblings of each element up to but not
+// including the element matched by the matcher. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) NextUntilMatcher(m Matcher) *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingNextUntil,
+		m, nil))
+}
+
+// NextUntilSelection gets all following siblings of each element up to but not
+// including the element matched by the Selection. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) NextUntilSelection(sel *Selection) *Selection {
+	if sel == nil {
+		return s.NextAll()
+	}
+	return s.NextUntilNodes(sel.Nodes...)
+}
+
+// NextUntilNodes gets all following siblings of each element up to but not
+// including the element matched by the nodes. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) NextUntilNodes(nodes ...*html.Node) *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingNextUntil,
+		nil, nodes))
+}
+
+// PrevUntil gets all preceding siblings of each element up to but not
+// including the element matched by the selector. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) PrevUntil(selector string) *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingPrevUntil,
+		compileMatcher(selector), nil))
+}
+
+// PrevUntilMatcher gets all preceding siblings of each element up to but not
+// including the element matched by the matcher. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) PrevUntilMatcher(m Matcher) *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingPrevUntil,
+		m, nil))
+}
+
+// PrevUntilSelection gets all preceding siblings of each element up to but not
+// including the element matched by the Selection. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) PrevUntilSelection(sel *Selection) *Selection {
+	if sel == nil {
+		return s.PrevAll()
+	}
+	return s.PrevUntilNodes(sel.Nodes...)
+}
+
+// PrevUntilNodes gets all preceding siblings of each element up to but not
+// including the element matched by the nodes. It returns a new Selection
+// object containing the matched elements.
+func (s *Selection) PrevUntilNodes(nodes ...*html.Node) *Selection {
+	return pushStack(s, getSiblingNodes(s.Nodes, siblingPrevUntil,
+		nil, nodes))
+}
+
+// NextFilteredUntil is like NextUntil, with the option to filter
+// the results based on a selector string.
+// It returns a new Selection object containing the matched elements.
+func (s *Selection) NextFilteredUntil(filterSelector, untilSelector string) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingNextUntil,
+		compileMatcher(untilSelector), nil), compileMatcher(filterSelector))
+}
+
+// NextFilteredUntilMatcher is like NextUntilMatcher, with the option to filter
+// the results based on a matcher.
+// It returns a new Selection object containing the matched elements.
+func (s *Selection) NextFilteredUntilMatcher(filter, until Matcher) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingNextUntil,
+		until, nil), filter)
+}
+
+// NextFilteredUntilSelection is like NextUntilSelection, with the
+// option to filter the results based on a selector string. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) NextFilteredUntilSelection(filterSelector string, sel *Selection) *Selection {
+	return s.NextMatcherUntilSelection(compileMatcher(filterSelector), sel)
+}
+
+// NextMatcherUntilSelection is like NextUntilSelection, with the
+// option to filter the results based on a matcher. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) NextMatcherUntilSelection(filter Matcher, sel *Selection) *Selection {
+	if sel == nil {
+		return s.NextMatcher(filter)
+	}
+	return s.NextMatcherUntilNodes(filter, sel.Nodes...)
+}
+
+// NextFilteredUntilNodes is like NextUntilNodes, with the
+// option to filter the results based on a selector string. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) NextFilteredUntilNodes(filterSelector string, nodes ...*html.Node) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingNextUntil,
+		nil, nodes), compileMatcher(filterSelector))
+}
+
+// NextMatcherUntilNodes is like NextUntilNodes, with the
+// option to filter the results based on a matcher. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) NextMatcherUntilNodes(filter Matcher, nodes ...*html.Node) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingNextUntil,
+		nil, nodes), filter)
+}
+
+// PrevFilteredUntil is like PrevUntil, with the option to filter
+// the results based on a selector string.
+// It returns a new Selection object containing the matched elements.
+func (s *Selection) PrevFilteredUntil(filterSelector, untilSelector string) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingPrevUntil,
+		compileMatcher(untilSelector), nil), compileMatcher(filterSelector))
+}
+
+// PrevFilteredUntilMatcher is like PrevUntilMatcher, with the option to filter
+// the results based on a matcher.
+// It returns a new Selection object containing the matched elements.
+func (s *Selection) PrevFilteredUntilMatcher(filter, until Matcher) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingPrevUntil,
+		until, nil), filter)
+}
+
+// PrevFilteredUntilSelection is like PrevUntilSelection, with the
+// option to filter the results based on a selector string. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) PrevFilteredUntilSelection(filterSelector string, sel *Selection) *Selection {
+	return s.PrevMatcherUntilSelection(compileMatcher(filterSelector), sel)
+}
+
+// PrevMatcherUntilSelection is like PrevUntilSelection, with the
+// option to filter the results based on a matcher. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) PrevMatcherUntilSelection(filter Matcher, sel *Selection) *Selection {
+	if sel == nil {
+		return s.PrevMatcher(filter)
+	}
+	return s.PrevMatcherUntilNodes(filter, sel.Nodes...)
+}
+
+// PrevFilteredUntilNodes is like PrevUntilNodes, with the
+// option to filter the results based on a selector string. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) PrevFilteredUntilNodes(filterSelector string, nodes ...*html.Node) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingPrevUntil,
+		nil, nodes), compileMatcher(filterSelector))
+}
+
+// PrevMatcherUntilNodes is like PrevUntilNodes, with the
+// option to filter the results based on a matcher. It returns a new
+// Selection object containing the matched elements.
+func (s *Selection) PrevMatcherUntilNodes(filter Matcher, nodes ...*html.Node) *Selection {
+	return filterAndPush(s, getSiblingNodes(s.Nodes, siblingPrevUntil,
+		nil, nodes), filter)
+}
+
+// Filter and push filters the nodes based on a matcher, and pushes the results
+// on the stack, with the srcSel as previous selection.
+func filterAndPush(srcSel *Selection, nodes []*html.Node, m Matcher) *Selection {
+	// Create a temporary Selection with the specified nodes to filter using winnow
+	sel := &Selection{nodes, srcSel.document, nil}
+	// Filter based on matcher and push on stack
+	return pushStack(srcSel, winnow(sel, m, true))
+}
+
+// Internal implementation of Find that return raw nodes.
+func findWithMatcher(nodes []*html.Node, m Matcher) []*html.Node {
+	// Map nodes to find the matches within the children of each node
+	return mapNodes(nodes, func(i int, n *html.Node) (result []*html.Node) {
+		// Go down one level, becausejQuery's Find selects only within descendants
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			if c.Type == html.ElementNode {
+				result = append(result, m.MatchAll(c)...)
+			}
+		}
+		return
+	})
+}
+
+// Internal implementation to get all parent nodes, stopping at the specified
+// node (or nil if no stop).
+func getParentsNodes(nodes []*html.Node, stopm Matcher, stopNodes []*html.Node) []*html.Node {
+	return mapNodes(nodes, func(i int, n *html.Node) (result []*html.Node) {
+		for p := n.Parent; p != nil; p = p.Parent {
+			sel := newSingleSelection(p, nil)
+			if stopm != nil {
+				if sel.IsMatcher(stopm) {
+					break
+				}
+			} else if len(stopNodes) > 0 {
+				if sel.IsNodes(stopNodes...) {
+					break
+				}
+			}
+			if p.Type == html.ElementNode {
+				result = append(result, p)
+			}
+		}
+		return
+	})
+}
+
+// Internal implementation of sibling nodes that return a raw slice of matches.
+func getSiblingNodes(nodes []*html.Node, st siblingType, untilm Matcher, untilNodes []*html.Node) []*html.Node {
+	var f func(*html.Node) bool
+
+	// If the requested siblings are ...Until, create the test function to
+	// determine if the until condition is reached (returns true if it is)
+	if st == siblingNextUntil || st == siblingPrevUntil {
+		f = func(n *html.Node) bool {
+			if untilm != nil {
+				// Matcher-based condition
+				sel := newSingleSelection(n, nil)
+				return sel.IsMatcher(untilm)
+			} else if len(untilNodes) > 0 {
+				// Nodes-based condition
+				sel := newSingleSelection(n, nil)
+				return sel.IsNodes(untilNodes...)
+			}
+			return false
+		}
+	}
+
+	return mapNodes(nodes, func(i int, n *html.Node) []*html.Node {
+		return getChildrenWithSiblingType(n.Parent, st, n, f)
+	})
+}
+
+// Gets the children nodes of each node in the specified slice of nodes,
+// based on the sibling type request.
+func getChildrenNodes(nodes []*html.Node, st siblingType) []*html.Node {
+	return mapNodes(nodes, func(i int, n *html.Node) []*html.Node {
+		return getChildrenWithSiblingType(n, st, nil, nil)
+	})
+}
+
+// Gets the children of the specified parent, based on the requested sibling
+// type, skipping a specified node if required.
+func getChildrenWithSiblingType(parent *html.Node, st siblingType, skipNode *html.Node,
+	untilFunc func(*html.Node) bool) (result []*html.Node) {
+
+	// Create the iterator function
+	var iter = func(cur *html.Node) (ret *html.Node) {
+		// Based on the sibling type requested, iterate the right way
+		for {
+			switch st {
+			case siblingAll, siblingAllIncludingNonElements:
+				if cur == nil {
+					// First iteration, start with first child of parent
+					// Skip node if required
+					if ret = parent.FirstChild; ret == skipNode && skipNode != nil {
+						ret = skipNode.NextSibling
+					}
+				} else {
+					// Skip node if required
+					if ret = cur.NextSibling; ret == skipNode && skipNode != nil {
+						ret = skipNode.NextSibling
+					}
+				}
+			case siblingPrev, siblingPrevAll, siblingPrevUntil:
+				if cur == nil {
+					// Start with previous sibling of the skip node
+					ret = skipNode.PrevSibling
+				} else {
+					ret = cur.PrevSibling
+				}
+			case siblingNext, siblingNextAll, siblingNextUntil:
+				if cur == nil {
+					// Start with next sibling of the skip node
+					ret = skipNode.NextSibling
+				} else {
+					ret = cur.NextSibling
+				}
+			default:
+				panic("Invalid sibling type.")
+			}
+			if ret == nil || ret.Type == html.ElementNode || st == siblingAllIncludingNonElements {
+				return
+			}
+			// Not a valid node, try again from this one
+			cur = ret
+		}
+	}
+
+	for c := iter(nil); c != nil; c = iter(c) {
+		// If this is an ...Until case, test before append (returns true
+		// if the until condition is reached)
+		if st == siblingNextUntil || st == siblingPrevUntil {
+			if untilFunc(c) {
+				return
+			}
+		}
+		result = append(result, c)
+		if st == siblingNext || st == siblingPrev {
+			// Only one node was requested (immediate next or previous), so exit
+			return
+		}
+	}
+	return
+}
+
+// Internal implementation of parent nodes that return a raw slice of Nodes.
+func getParentNodes(nodes []*html.Node) []*html.Node {
+	return mapNodes(nodes, func(i int, n *html.Node) []*html.Node {
+		if n.Parent != nil && n.Parent.Type == html.ElementNode {
+			return []*html.Node{n.Parent}
+		}
+		return nil
+	})
+}
+
+// Internal map function used by many traversing methods. Takes the source nodes
+// to iterate on and the mapping function that returns an array of nodes.
+// Returns an array of nodes mapped by calling the callback function once for
+// each node in the source nodes.
+func mapNodes(nodes []*html.Node, f func(int, *html.Node) []*html.Node) (result []*html.Node) {
+	set := make(map[*html.Node]bool)
+	for i, n := range nodes {
+		if vals := f(i, n); len(vals) > 0 {
+			result = appendWithoutDuplicates(result, vals, set)
+		}
+	}
+	return result
+}

vendor/github.com/PuerkitoBio/goquery/type.go

@@ -0,0 +1,141 @@
+package goquery
+
+import (
+	"errors"
+	"io"
+	"net/http"
+	"net/url"
+
+	"github.com/andybalholm/cascadia"
+
+	"golang.org/x/net/html"
+)
+
+// Document represents an HTML document to be manipulated. Unlike jQuery, which
+// is loaded as part of a DOM document, and thus acts upon its containing
+// document, GoQuery doesn't know which HTML document to act upon. So it needs
+// to be told, and that's what the Document class is for. It holds the root
+// document node to manipulate, and can make selections on this document.
+type Document struct {
+	*Selection
+	Url      *url.URL
+	rootNode *html.Node
+}
+
+// NewDocumentFromNode is a Document constructor that takes a root html Node
+// as argument.
+func NewDocumentFromNode(root *html.Node) *Document {
+	return newDocument(root, nil)
+}
+
+// NewDocument is a Document constructor that takes a string URL as argument.
+// It loads the specified document, parses it, and stores the root Document
+// node, ready to be manipulated.
+//
+// Deprecated: Use the net/http standard library package to make the request
+// and validate the response before calling goquery.NewDocumentFromReader
+// with the response's body.
+func NewDocument(url string) (*Document, error) {
+	// Load the URL
+	res, e := http.Get(url)
+	if e != nil {
+		return nil, e
+	}
+	return NewDocumentFromResponse(res)
+}
+
+// NewDocumentFromReader returns a Document from an io.Reader.
+// It returns an error as second value if the reader's data cannot be parsed
+// as html. It does not check if the reader is also an io.Closer, the
+// provided reader is never closed by this call. It is the responsibility
+// of the caller to close it if required.
+func NewDocumentFromReader(r io.Reader) (*Document, error) {
+	root, e := html.Parse(r)
+	if e != nil {
+		return nil, e
+	}
+	return newDocument(root, nil), nil
+}
+
+// NewDocumentFromResponse is another Document constructor that takes an http response as argument.
+// It loads the specified response's document, parses it, and stores the root Document
+// node, ready to be manipulated. The response's body is closed on return.
+//
+// Deprecated: Use goquery.NewDocumentFromReader with the response's body.
+func NewDocumentFromResponse(res *http.Response) (*Document, error) {
+	if res == nil {
+		return nil, errors.New("Response is nil")
+	}
+	defer res.Body.Close()
+	if res.Request == nil {
+		return nil, errors.New("Response.Request is nil")
+	}
+
+	// Parse the HTML into nodes
+	root, e := html.Parse(res.Body)
+	if e != nil {
+		return nil, e
+	}
+
+	// Create and fill the document
+	return newDocument(root, res.Request.URL), nil
+}
+
+// CloneDocument creates a deep-clone of a document.
+func CloneDocument(doc *Document) *Document {
+	return newDocument(cloneNode(doc.rootNode), doc.Url)
+}
+
+// Private constructor, make sure all fields are correctly filled.
+func newDocument(root *html.Node, url *url.URL) *Document {
+	// Create and fill the document
+	d := &Document{nil, url, root}
+	d.Selection = newSingleSelection(root, d)
+	return d
+}
+
+// Selection represents a collection of nodes matching some criteria. The
+// initial Selection can be created by using Document.Find, and then
+// manipulated using the jQuery-like chainable syntax and methods.
+type Selection struct {
+	Nodes    []*html.Node
+	document *Document
+	prevSel  *Selection
+}
+
+// Helper constructor to create an empty selection
+func newEmptySelection(doc *Document) *Selection {
+	return &Selection{nil, doc, nil}
+}
+
+// Helper constructor to create a selection of only one node
+func newSingleSelection(node *html.Node, doc *Document) *Selection {
+	return &Selection{[]*html.Node{node}, doc, nil}
+}
+
+// Matcher is an interface that defines the methods to match
+// HTML nodes against a compiled selector string. Cascadia's
+// Selector implements this interface.
+type Matcher interface {
+	Match(*html.Node) bool
+	MatchAll(*html.Node) []*html.Node
+	Filter([]*html.Node) []*html.Node
+}
+
+// compileMatcher compiles the selector string s and returns
+// the corresponding Matcher. If s is an invalid selector string,
+// it returns a Matcher that fails all matches.
+func compileMatcher(s string) Matcher {
+	cs, err := cascadia.Compile(s)
+	if err != nil {
+		return invalidMatcher{}
+	}
+	return cs
+}
+
+// invalidMatcher is a Matcher that always fails to match.
+type invalidMatcher struct{}
+
+func (invalidMatcher) Match(n *html.Node) bool             { return false }
+func (invalidMatcher) MatchAll(n *html.Node) []*html.Node  { return nil }
+func (invalidMatcher) Filter(ns []*html.Node) []*html.Node { return nil }

vendor/github.com/PuerkitoBio/goquery/utilities.go

@@ -0,0 +1,161 @@
+package goquery
+
+import (
+	"bytes"
+
+	"golang.org/x/net/html"
+)
+
+// used to determine if a set (map[*html.Node]bool) should be used
+// instead of iterating over a slice. The set uses more memory and
+// is slower than slice iteration for small N.
+const minNodesForSet = 1000
+
+var nodeNames = []string{
+	html.ErrorNode:    "#error",
+	html.TextNode:     "#text",
+	html.DocumentNode: "#document",
+	html.CommentNode:  "#comment",
+}
+
+// NodeName returns the node name of the first element in the selection.
+// It tries to behave in a similar way as the DOM's nodeName property
+// (https://developer.mozilla.org/en-US/docs/Web/API/Node/nodeName).
+//
+// Go's net/html package defines the following node types, listed with
+// the corresponding returned value from this function:
+//
+//     ErrorNode : #error
+//     TextNode : #text
+//     DocumentNode : #document
+//     ElementNode : the element's tag name
+//     CommentNode : #comment
+//     DoctypeNode : the name of the document type
+//
+func NodeName(s *Selection) string {
+	if s.Length() == 0 {
+		return ""
+	}
+	switch n := s.Get(0); n.Type {
+	case html.ElementNode, html.DoctypeNode:
+		return n.Data
+	default:
+		if n.Type >= 0 && int(n.Type) < len(nodeNames) {
+			return nodeNames[n.Type]
+		}
+		return ""
+	}
+}
+
+// OuterHtml returns the outer HTML rendering of the first item in
+// the selection - that is, the HTML including the first element's
+// tag and attributes.
+//
+// Unlike InnerHtml, this is a function and not a method on the Selection,
+// because this is not a jQuery method (in javascript-land, this is
+// a property provided by the DOM).
+func OuterHtml(s *Selection) (string, error) {
+	var buf bytes.Buffer
+
+	if s.Length() == 0 {
+		return "", nil
+	}
+	n := s.Get(0)
+	if err := html.Render(&buf, n); err != nil {
+		return "", err
+	}
+	return buf.String(), nil
+}
+
+// Loop through all container nodes to search for the target node.
+func sliceContains(container []*html.Node, contained *html.Node) bool {
+	for _, n := range container {
+		if nodeContains(n, contained) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Checks if the contained node is within the container node.
+func nodeContains(container *html.Node, contained *html.Node) bool {
+	// Check if the parent of the contained node is the container node, traversing
+	// upward until the top is reached, or the container is found.
+	for contained = contained.Parent; contained != nil; contained = contained.Parent {
+		if container == contained {
+			return true
+		}
+	}
+	return false
+}
+
+// Checks if the target node is in the slice of nodes.
+func isInSlice(slice []*html.Node, node *html.Node) bool {
+	return indexInSlice(slice, node) > -1
+}
+
+// Returns the index of the target node in the slice, or -1.
+func indexInSlice(slice []*html.Node, node *html.Node) int {
+	if node != nil {
+		for i, n := range slice {
+			if n == node {
+				return i
+			}
+		}
+	}
+	return -1
+}
+
+// Appends the new nodes to the target slice, making sure no duplicate is added.
+// There is no check to the original state of the target slice, so it may still
+// contain duplicates. The target slice is returned because append() may create
+// a new underlying array. If targetSet is nil, a local set is created with the
+// target if len(target) + len(nodes) is greater than minNodesForSet.
+func appendWithoutDuplicates(target []*html.Node, nodes []*html.Node, targetSet map[*html.Node]bool) []*html.Node {
+	// if there are not that many nodes, don't use the map, faster to just use nested loops
+	// (unless a non-nil targetSet is passed, in which case the caller knows better).
+	if targetSet == nil && len(target)+len(nodes) < minNodesForSet {
+		for _, n := range nodes {
+			if !isInSlice(target, n) {
+				target = append(target, n)
+			}
+		}
+		return target
+	}
+
+	// if a targetSet is passed, then assume it is reliable, otherwise create one
+	// and initialize it with the current target contents.
+	if targetSet == nil {
+		targetSet = make(map[*html.Node]bool, len(target))
+		for _, n := range target {
+			targetSet[n] = true
+		}
+	}
+	for _, n := range nodes {
+		if !targetSet[n] {
+			target = append(target, n)
+			targetSet[n] = true
+		}
+	}
+
+	return target
+}
+
+// Loop through a selection, returning only those nodes that pass the predicate
+// function.
+func grep(sel *Selection, predicate func(i int, s *Selection) bool) (result []*html.Node) {
+	for i, n := range sel.Nodes {
+		if predicate(i, newSingleSelection(n, sel.document)) {
+			result = append(result, n)
+		}
+	}
+	return result
+}
+
+// Creates a new Selection object based on the specified nodes, and keeps the
+// source Selection object on the stack (linked list).
+func pushStack(fromSel *Selection, nodes []*html.Node) *Selection {
+	result := &Selection{nodes, fromSel.document, fromSel}
+	return result
+}

vendor/github.com/advancedlogic/GoOse/.gitignore

@@ -0,0 +1,26 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+*.prof
+
+target

vendor/github.com/advancedlogic/GoOse/.travis.yml

@@ -0,0 +1,22 @@
+language: go
+
+sudo: false
+
+go:
+  - 1.5
+  - tip
+
+before_install:
+  - go get github.com/mattn/goveralls
+
+before_script:
+  - make deps
+
+script:
+  - make qa
+
+after_failure:
+  - cat ./target/test/report.xml
+
+after_script:
+  - $HOME/gopath/bin/goveralls -covermode=count -coverprofile=target/report/coverage.out -service=travis-ci

vendor/github.com/advancedlogic/GoOse/Gopkg.lock

@@ -0,0 +1,129 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  digest = "1:a62f6ed230a8cd138a9efbe718e7d0b0294f139266f5f55cd942769a9aac8de2"
+  name = "github.com/PuerkitoBio/goquery"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "dc2ec5c7ca4d9aae063b79b9f581dd3ea6afd2b2"
+  version = "v1.4.1"
+
+[[projects]]
+  digest = "1:66b3310cf22cdc96c35ef84ede4f7b9b370971c4025f394c89a2638729653b11"
+  name = "github.com/andybalholm/cascadia"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "901648c87902174f774fac311d7f176f8647bdaa"
+  version = "v1.0.0"
+
+[[projects]]
+  branch = "master"
+  digest = "1:90d593cd34221cf33017a2c944998eb29f5182a15cfbe0cf6e2a0f62cb8bbf29"
+  name = "github.com/araddon/dateparse"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "cfd92a431d0efe36a1b81ca25d15b98aae4dbdb6"
+
+[[projects]]
+  digest = "1:5f1d44e13c151d777a80b0d8ca850347530773e73ccce5995a520cdc6947c205"
+  name = "github.com/fatih/set"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "2c768e3c5489976167bfc42b5c7c92ca783f4389"
+  version = "v0.2.1"
+
+[[projects]]
+  branch = "master"
+  digest = "1:131e93e8b8f318c52b1666e5a8f3af7d236af62f5a477a11973e46af59bca7da"
+  name = "github.com/gigawattio/window"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "0f5467e35573c4ffebddd992935e1f7ceeac0696"
+
+[[projects]]
+  branch = "master"
+  digest = "1:8bacc2118e6cc6a207eb144656537ff7ee0397f8bc8c1f7639519b0de2bb949a"
+  name = "github.com/jaytaylor/html2text"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "57d518f124b0cf46ea2021f25a01396b3522e6fb"
+
+[[projects]]
+  digest = "1:cdb899c199f907ac9fb50495ec71212c95cb5b0e0a8ee0800da0238036091033"
+  name = "github.com/mattn/go-runewidth"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "ce7b0b5c7b45a81508558cd1dba6bb1e4ddb51bb"
+  version = "v0.0.3"
+
+[[projects]]
+  branch = "master"
+  digest = "1:4daa045e1e1f3e23f4b07db6880cdf9f259dab65312dfe244a878e6070faaf77"
+  name = "github.com/olekukonko/tablewriter"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "d4647c9c7a84d847478d890b816b7d8b62b0b279"
+
+[[projects]]
+  branch = "master"
+  digest = "1:362b269942e8508aed33025593090ad4cdbd9077de4b1cd505e6124728e9e856"
+  name = "github.com/ssor/bom"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "6386211fdfcf24c0bfbdaceafd02849ed9a8a509"
+
+[[projects]]
+  branch = "master"
+  digest = "1:1d3f083a0b29a9db5ac57dc4d486dbb97cd58696bae3da441a375916b7e8c53b"
+  name = "golang.org/x/net"
+  packages = [
+    "html",
+    "html/atom",
+    "html/charset",
+  ]
+  pruneopts = "UT"
+  revision = "4bcd98cce591d8c7061bf313d7a3cbad05b58549"
+
+[[projects]]
+  digest = "1:aa4d6967a3237f8367b6bf91503964a77183ecf696f1273e8ad3551bb4412b5f"
+  name = "golang.org/x/text"
+  packages = [
+    "encoding",
+    "encoding/charmap",
+    "encoding/htmlindex",
+    "encoding/internal",
+    "encoding/internal/identifier",
+    "encoding/japanese",
+    "encoding/korean",
+    "encoding/simplifiedchinese",
+    "encoding/traditionalchinese",
+    "encoding/unicode",
+    "internal/gen",
+    "internal/tag",
+    "internal/utf8internal",
+    "language",
+    "runes",
+    "transform",
+    "unicode/cldr",
+  ]
+  pruneopts = "UT"
+  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
+  version = "v0.3.0"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  input-imports = [
+    "github.com/PuerkitoBio/goquery",
+    "github.com/araddon/dateparse",
+    "github.com/fatih/set",
+    "github.com/gigawattio/window",
+    "github.com/jaytaylor/html2text",
+    "golang.org/x/net/html",
+    "golang.org/x/net/html/atom",
+    "golang.org/x/net/html/charset",
+    "golang.org/x/text/transform",
+  ]
+  solver-name = "gps-cdcl"
+  solver-version = 1

vendor/github.com/advancedlogic/GoOse/Gopkg.toml

@@ -0,0 +1,58 @@
+# Gopkg.toml example
+#
+# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#   name = "github.com/x/y"
+#   version = "2.4.0"
+#
+# [prune]
+#   non-go = false
+#   go-tests = true
+#   unused-packages = true
+
+
+[[constraint]]
+  name = "github.com/PuerkitoBio/goquery"
+  version = "1.4.1"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/araddon/dateparse"
+
+[[constraint]]
+  name = "github.com/fatih/set"
+  version = "0.2.1"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/gigawattio/window"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/jaytaylor/html2text"
+
+[[constraint]]
+  branch = "master"
+  name = "golang.org/x/net"
+
+[[constraint]]
+  name = "golang.org/x/text"
+  version = "0.3.0"
+
+[prune]
+  go-tests = true
+  unused-packages = true

vendor/github.com/advancedlogic/GoOse/LICENSE

@@ -0,0 +1,202 @@
+Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

vendor/github.com/advancedlogic/GoOse/Makefile

@@ -0,0 +1,203 @@
+# MAKEFILE
+#
+# @author      Nicola Asuni <info@tecnick.com>
+# @link        https://github.com/advancedlogic/GoOse
+#
+# This file is intended to be executed in a Linux-compatible system.
+# It also assumes that the project has been cloned in the right path under GOPATH:
+# $GOPATH/src/github.com/advancedlogic/GoOse
+#
+# ------------------------------------------------------------------------------
+
+# List special make targets that are not associated with files
+.PHONY: help all test format fmtcheck vet lint coverage cyclo ineffassign misspell structcheck varcheck errcheck gosimple astscan qa deps clean nuke
+
+# Use bash as shell (Note: Ubuntu now uses dash which doesn't support PIPESTATUS).
+SHELL=/bin/bash
+
+# CVS path (path to the parent dir containing the project)
+CVSPATH=github.com/advancedlogic
+
+# Project owner
+OWNER=advancedlogic
+
+# Project vendor
+VENDOR=advancedlogic
+
+# Project name
+PROJECT=GoOse
+
+# Project version
+VERSION=$(shell cat VERSION)
+
+# Name of RPM or DEB package
+PKGNAME=${VENDOR}-${PROJECT}
+
+# Current directory
+CURRENTDIR=$(shell pwd)
+
+# GO lang path
+ifneq ($(GOPATH),)
+	ifeq ($(findstring $(GOPATH),$(CURRENTDIR)),)
+		# the defined GOPATH is not valid
+		GOPATH=
+	endif
+endif
+ifeq ($(GOPATH),)
+	# extract the GOPATH
+	GOPATH=$(firstword $(subst /src/, ,$(CURRENTDIR)))
+endif
+
+# --- MAKE TARGETS ---
+
+# Display general help about this command
+help:
+	@echo ""
+	@echo "$(PROJECT) Makefile."
+	@echo "GOPATH=$(GOPATH)"
+	@echo "The following commands are available:"
+	@echo ""
+	@echo "    make qa          : Run all the tests"
+	@echo "    make test        : Run the unit tests"
+	@echo ""
+	@echo "    make format      : Format the source code"
+	@echo "    make fmtcheck    : Check if the source code has been formatted"
+	@echo "    make vet         : Check for suspicious constructs"
+	@echo "    make lint        : Check for style errors"
+	@echo "    make coverage    : Generate the coverage report"
+	@echo "    make cyclo       : Generate the cyclomatic complexity report"
+	@echo "    make ineffassign : Detect ineffectual assignments"
+	@echo "    make misspell    : Detect commonly misspelled words in source files"
+	@echo "    make structcheck : Find unused struct fields"
+	@echo "    make varcheck    : Find unused global variables and constants"
+	@echo "    make errcheck    : Check that error return values are used"
+	@echo "    make gosimple    : Suggest code simplifications"
+	@echo "    make astscan     : GO AST scanner"
+	@echo ""
+	@echo "    make docs        : Generate source code documentation"
+	@echo ""
+	@echo "    make deps        : Get the dependencies"
+	@echo "    make clean       : Remove any build artifact"
+	@echo "    make nuke        : Deletes any intermediate file"
+	@echo ""
+
+
+# Alias for help target
+all: help
+
+# Run the unit tests
+test:
+	@mkdir -p target/test
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) \
+	go test \
+	-covermode=atomic \
+	-bench=. \
+	-race \
+	-cpuprofile=target/report/cpu.out \
+	-memprofile=target/report/mem.out \
+	-mutexprofile=target/report/mutex.out \
+	-coverprofile=target/report/coverage.out \
+	-v ./... | \
+	tee >(PATH=$(GOPATH)/bin:$(PATH) go-junit-report > target/test/report.xml); \
+	test $${PIPESTATUS[0]} -eq 0
+
+# Format the source code
+format:
+	@find . -type f -name "*.go" -exec gofmt -s -w {} \;
+
+# Check if the source code has been formatted
+fmtcheck:
+	@mkdir -p target
+	@find . -type f -name "*.go" -exec gofmt -s -d {} \; | tee target/format.diff
+	@test ! -s target/format.diff || { echo "ERROR: the source code has not been formatted - please use 'make format' or 'gofmt'"; exit 1; }
+
+# Check for syntax errors
+vet:
+	GOPATH=$(GOPATH) go vet .
+
+# Check for style errors
+lint:
+	GOPATH=$(GOPATH) PATH=$(GOPATH)/bin:$(PATH) golint .
+
+# Generate the coverage report
+coverage:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) \
+	go tool cover -html=target/report/coverage.out -o target/report/coverage.html
+
+# Report cyclomatic complexity
+cyclo:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) gocyclo -avg ./ | tee target/report/cyclo.txt ; test $${PIPESTATUS[0]} -eq 0
+
+# Detect ineffectual assignments
+ineffassign:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) ineffassign ./ | tee target/report/ineffassign.txt ; test $${PIPESTATUS[0]} -eq 0
+
+# Detect commonly misspelled words in source files
+misspell:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) misspell -error ./*.go | tee target/report/misspell.txt
+
+# Find unused struct fields
+structcheck:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) structcheck -a ./ | tee target/report/structcheck.txt
+
+# Find unused global variables and constants
+varcheck:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) varcheck -e ./ | tee target/report/varcheck.txt
+
+# Check that error return values are used
+errcheck:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) errcheck ./ | tee target/report/errcheck.txt
+
+# Suggest code simplifications
+gosimple:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) gosimple ./ | tee target/report/gosimple.txt
+
+# AST scanner
+astscan:
+	@mkdir -p target/report
+	GOPATH=$(GOPATH) gas .//*.go | tee target/report/astscan.txt
+
+# Generate source docs
+docs:
+	@mkdir -p target/docs
+	nohup sh -c 'GOPATH=$(GOPATH) godoc -http=127.0.0.1:6060' > target/godoc_server.log 2>&1 &
+	wget --directory-prefix=target/docs/ --execute robots=off --retry-connrefused --recursive --no-parent --adjust-extension --page-requisites --convert-links http://127.0.0.1:6060/pkg/github.com/${VENDOR}/${PROJECT}/ ; kill -9 `lsof -ti :6060`
+	@echo '<html><head><meta http-equiv="refresh" content="0;./127.0.0.1:6060/pkg/'${CVSPATH}'/'${PROJECT}'/index.html"/></head><a href="./127.0.0.1:6060/pkg/'${CVSPATH}'/'${PROJECT}'/index.html">'${PKGNAME}' Documentation ...</a></html>' > target/docs/index.html
+
+# Alias to run all quality-assurance checks
+qa: fmtcheck test vet lint coverage cyclo ineffassign misspell structcheck varcheck errcheck gosimple astscan
+
+# --- INSTALL ---
+
+# Get the dependencies
+deps:
+	GOPATH=$(GOPATH) go get ./...
+	GOPATH=$(GOPATH) go get github.com/golang/lint/golint
+	GOPATH=$(GOPATH) go get github.com/jstemmer/go-junit-report
+	GOPATH=$(GOPATH) go get github.com/axw/gocov/gocov
+	GOPATH=$(GOPATH) go get github.com/fzipp/gocyclo
+	GOPATH=$(GOPATH) go get github.com/gordonklaus/ineffassign
+	GOPATH=$(GOPATH) go get github.com/client9/misspell/cmd/misspell
+	GOPATH=$(GOPATH) go get github.com/opennota/check/cmd/structcheck
+	GOPATH=$(GOPATH) go get github.com/opennota/check/cmd/varcheck
+	GOPATH=$(GOPATH) go get github.com/kisielk/errcheck
+	GOPATH=$(GOPATH) go get honnef.co/go/tools/cmd/gosimple
+	GOPATH=$(GOPATH) go get github.com/GoASTScanner/gas
+
+# Remove any build artifact
+clean:
+	GOPATH=$(GOPATH) go clean ./...
+
+# Deletes any intermediate file
+nuke:
+	rm -rf ./target
+	GOPATH=$(GOPATH) go clean -i ./...

vendor/github.com/advancedlogic/GoOse/README.md

@@ -0,0 +1,86 @@
+# GoOse
+
+*HTML Content / Article Extractor in Golang*
+
+[![Build Status](https://secure.travis-ci.org/advancedlogic/GoOse.png?branch=master)](https://travis-ci.org/advancedlogic/GoOse?branch=master)
+[![Coverage Status](https://coveralls.io/repos/advancedlogic/GoOse/badge.svg?branch=master&service=github)](https://coveralls.io/github/advancedlogic/GoOse?branch=master)
+[![Go Report Card](https://goreportcard.com/badge/github.com/advancedlogic/GoOse)](https://goreportcard.com/report/github.com/advancedlogic/GoOse)
+[![GoDoc](https://godoc.org/github.com/advancedlogic/GoOse?status.svg)](http://godoc.org/github.com/advancedlogic/GoOse)
+
+
+## Description
+
+This is a golang port of "Goose" originaly licensed to Gravity.com
+under one or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.
+
+Golang port was written by Antonio Linari
+
+Gravity.com licenses this file
+to you under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+## INSTALL
+
+```bash
+go get github.com/advancedlogic/GoOse
+```
+
+## HOW TO USE IT
+
+```Go
+package main
+
+import (
+	"github.com/advancedlogic/GoOse"
+)
+
+func main() {
+	g := goose.New()
+	article, _ := g.ExtractFromURL("http://edition.cnn.com/2012/07/08/opinion/banzi-ted-open-source/index.html")
+	println("title", article.Title)
+	println("description", article.MetaDescription)
+	println("keywords", article.MetaKeywords)
+	println("content", article.CleanedText)
+	println("url", article.FinalURL)
+	println("top image", article.TopImage)
+}
+```
+
+## Development - Getting started
+
+This application is written in GO language, please refere to the guides in https://golang.org for getting started.
+
+This project include a Makefile that allows you to test and build the project with simple commands.
+To see all available options:
+```bash
+make help
+```
+
+Before committing the code, please check if it passes all tests using
+```bash
+make deps
+make qa
+```
+
+## TODO
+- [ ] better organize code
+- [ ] improve "xpath" like queries
+- [ ] add other image extractions techniques (imagemagick)
+
+## THANKS TO
+```
+@Martin Angers for goquery
+@Fatih Arslan for set
+GoLang team for the amazing language and net/html
+```

vendor/github.com/advancedlogic/GoOse/VERSION

@@ -0,0 +1 @@
+1.1.0

vendor/github.com/advancedlogic/GoOse/article.go

@@ -0,0 +1,40 @@
+package goose
+
+import (
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/fatih/set"
+)
+
+// Article is a collection of properties extracted from the HTML body
+type Article struct {
+	Title           string             `json:"title,omitempty"`
+	TitleUnmodified string             `json:"titleunmodified,omitempty"`
+	CleanedText     string             `json:"content,omitempty"`
+	MetaDescription string             `json:"description,omitempty"`
+	MetaLang        string             `json:"lang,omitempty"`
+	MetaFavicon     string             `json:"favicon,omitempty"`
+	MetaKeywords    string             `json:"keywords,omitempty"`
+	CanonicalLink   string             `json:"canonicalurl,omitempty"`
+	Domain          string             `json:"domain,omitempty"`
+	TopNode         *goquery.Selection `json:"-"`
+	TopImage        string             `json:"image,omitempty"`
+	Tags            *set.Set           `json:"tags,omitempty"`
+	Movies          *set.Set           `json:"movies,omitempty"`
+	FinalURL        string             `json:"url,omitempty"`
+	LinkHash        string             `json:"linkhash,omitempty"`
+	RawHTML         string             `json:"rawhtml,omitempty"`
+	Doc             *goquery.Document  `json:"-"`
+	Links           []string           `json:"links,omitempty"`
+	PublishDate     *time.Time         `json:"publishdate,omitempty"`
+	AdditionalData  map[string]string  `json:"additionaldata,omitempty"`
+	Delta           int64              `json:"delta,omitempty"`
+}
+
+// ToString is a simple method to just show the title
+// TODO: add more fields and pretty print
+func (article *Article) ToString() string {
+	out := article.Title
+	return out
+}

vendor/github.com/advancedlogic/GoOse/charset.go

@@ -0,0 +1,92 @@
+package goose
+
+import (
+	"log"
+	"strings"
+	"unicode/utf8"
+
+	"golang.org/x/net/html/charset"
+	"golang.org/x/text/transform"
+)
+
+// NormaliseCharset Overrides/fixes charset names to something we can parse.
+// Fixes common mispellings and uses a canonical name for equivalent encodings.
+// @see https://encoding.spec.whatwg.org#names-and-labels
+func NormaliseCharset(characterSet string) string {
+	characterSet = strings.ToUpper(characterSet)
+	switch characterSet {
+	case "UTF8", "UT-8", "UTR-8", "UFT-8", "UTF8-WITHOUT-BOM", "UTF8_GENERAL_CI":
+		return "UTF-8"
+	// override Japanese
+	// CP943: IBM OS/2 Japanese, superset of Cp932 and Shift-JIS
+	case "CP943", "CP943C", "SIFT_JIS", "SHIFT-JIS":
+		return "SHIFT_JIS"
+	// override Korean
+	case "EUC-KR", "MS949", "KSC5601", "WINDOWS-949", "KS_C_5601-1987", "KSC_5601":
+		return "UHC"
+	// override Thai
+	//case "TIS-620", "WINDOWS-874":
+	//	return "ISO-8859-11"
+	// override latin-2
+	case "LATIN2_HUNGARIAN_CI", "LATIN2":
+		return "LATIN-2"
+	// override cyrillic
+	case "WIN1251", "WIN-1251", "WINDOWS-1251":
+		return "CP1251"
+	// override Hebrew
+	case "WINDOWS-1255":
+		return "ISO-8859-8"
+	// override Turkish
+	//case "WINDOWS-1254":
+	//	return "ISO-8859-9"
+	// override the parsing of ISO-8859-1 to behave as Windows-1252 (CP1252):
+	// in ISO-8859-1, everything from 128-255 in the ASCII table are ctrl characters,
+	// whilst in CP1252 they're symbols
+	// override Baltic
+	case "WINDOWS-1257":
+		return "ISO-8859-13"
+	case "ANSI", "LATIN-1", "ISO", "RFC", "MACINTOSH", "8859-1", "8859-15", "ISO8859-1", "ISO8859-15", "ISO-8559-1", "ISO-8859-1", "ISO-8859-15":
+		return "CP1252"
+	}
+	return characterSet
+}
+
+// UTF8encode converts a string from the source character set to UTF-8, skipping invalid byte sequences
+// @see http://stackoverflow.com/questions/32512500/ignore-illegal-bytes-when-decoding-text-with-go
+func UTF8encode(raw string, sourceCharset string) string {
+	enc, name := charset.Lookup(sourceCharset)
+	if nil == enc {
+		log.Println("Cannot convert from", sourceCharset, ":", name)
+		return raw
+	}
+
+	dst := make([]byte, len(raw))
+	d := enc.NewDecoder()
+
+	var (
+		in  int
+		out int
+	)
+	for in < len(raw) {
+		// Do the transformation
+		ndst, nsrc, err := d.Transform(dst[out:], []byte(raw[in:]), true)
+		in += nsrc
+		out += ndst
+		if err == nil {
+			// Completed transformation
+			break
+		}
+		if err == transform.ErrShortDst {
+			// Our output buffer is too small, so we need to grow it
+			t := make([]byte, (cap(dst)+1)*2)
+			copy(t, dst)
+			dst = t
+			continue
+		}
+		// We're here because of at least one illegal character. Skip over the current rune
+		// and try again.
+		_, width := utf8.DecodeRuneInString(raw[in:])
+		in += width
+	}
+	return string(dst)
+}

vendor/github.com/advancedlogic/GoOse/cleaner.go

@@ -0,0 +1,568 @@
+package goose
+
+import (
+	"container/list"
+	"github.com/PuerkitoBio/goquery"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+	"log"
+	"regexp"
+	"strings"
+)
+
+var whitelistedTextAtomTypes = []atom.Atom{atom.Span, atom.Em, atom.I, atom.Strong, atom.B, atom.P, atom.H1, atom.H2, atom.H3, atom.H4}
+var whitelistedExtAtomTypes = []atom.Atom{atom.A, atom.Span, atom.Em, atom.I, atom.Strong, atom.B, atom.P, atom.H1, atom.H2, atom.H3, atom.H4}
+
+// Cleaner removes menus, ads, sidebars, etc. and leaves the main content
+type Cleaner struct {
+	config Configuration
+}
+
+// NewCleaner returns a new instance of a Cleaner
+func NewCleaner(config Configuration) Cleaner {
+	return Cleaner{
+		config: config,
+	}
+}
+
+// replaceTagWithContents removes the tag, replacing it with its text contents
+// e.g. "<em>some text</em>" becomes "some text"
+func replaceTagWithContents(tagSelection *goquery.Selection, collapsibleAtomTypes []atom.Atom) {
+	if tagSelection.Length() == 0 {
+		return
+	}
+	node := tagSelection.Get(0)
+	node.Data = tagSelection.Text()
+	node.Type = html.TextNode
+	if node.FirstChild == nil {
+		node.Attr = []html.Attribute{}
+		node.DataAtom = 0
+		node.FirstChild = nil
+		node.LastChild = nil
+	} else {
+		// If all children are text only, the parent already contains the text, so drop them
+		collapseTextNodes(node, collapsibleAtomTypes)
+	}
+}
+
+func isAtomTypeWhitelisted(t atom.Atom, whitelist []atom.Atom) bool {
+	for _, allowed := range whitelist {
+		if t == allowed {
+			return true
+		}
+	}
+	return false
+}
+
+func collapseTextNodes(node *html.Node, collapsibleAtomTypes []atom.Atom) {
+	if node.FirstChild == nil {
+		return
+	}
+
+	if !isAtomTypeWhitelisted(node.DataAtom, collapsibleAtomTypes) {
+		return
+	}
+
+	if node.FirstChild.DataAtom == 0 && node.FirstChild == node.LastChild {
+		// this tag only contains a single textual node, already contained in the parent
+		node.Attr = []html.Attribute{}
+		node.Type = html.TextNode
+		node.DataAtom = 0
+		node.FirstChild = nil
+		node.LastChild = nil
+		return
+	}
+
+	// If all children are text only, the parent already contains the text, so drop them
+	allTextNodes := true
+	for c := node.FirstChild; c != nil; c = c.NextSibling {
+		// attempt collapsing recursively
+		collapseTextNodes(c, collapsibleAtomTypes)
+		if c.DataAtom != 0 {
+			// not collapsed
+			allTextNodes = false
+			break
+		}
+	}
+	if allTextNodes {
+		// text already contained in the parent node => drop children
+		node.Attr = []html.Attribute{}
+		node.Type = html.TextNode
+		node.DataAtom = 0
+		node.FirstChild = nil
+		node.LastChild = nil
+	}
+}
+
+var divToPElementsPattern = regexp.MustCompile("<(a|blockquote|dl|div|img|ol|p|pre|table|ul)")
+var tabsRegEx = regexp.MustCompile(`\t|^\s+$]`)
+var removeVisibilityStyleRegEx = regexp.MustCompile("visibility:[ ]*hidden|display:[ ]*none")
+var removeNodesRegEx = regexp.MustCompile("" +
+	"[Cc]omentario|" +
+	"[Ff]ooter|" +
+	"^fn$|" +
+	"^inset$|" +
+	"^print$|" +
+	"^scroll$|" +
+	"^side$|" +
+	"^side_|" +
+	"^widget$|" +
+	"^ab[0-9]$|" +
+	"[_-]ads$|" +
+	"^ad[s]?[ _-]|" +
+	"[_-]ad[s]?[_-]|" +
+	"^ADX_CLIENTSIDE$|" +
+	"ajoutVideo|" +
+	"^alerts|" +
+	"^Anchor$|" +
+	"articleheadings|" +
+	"_articles|" +
+	"^article-gallery-embedded$|" +
+	"author|" +
+	"author-dropdown|" +
+	"^banner|" +
+	"^bar$|" +
+	"blog-pager|" +
+	"brass\\-rail|" +
+	"breadcrumbs|" +
+	"button|" +
+	"byline|" +
+	"cabecalho|" +
+	"^caption$|" +
+	"carousel|" +
+	"^click|" +
+	"cnnStryHghLght|" +
+	"cnn_html_slideshow|" +
+	"cnn_strycaptiontxt|" +
+	"cnn_strylftcntnt|" +
+	"cnn_stryspcvbx|" +
+	"combx|" +
+	"comment|" +
+	"commercial|" +
+	"communitypromo|" +
+	"^comscore$|" +
+	"contact|" +
+	"contentTools2|" +
+	"controls|" +
+	"cookie|" +
+	"CoversMainContent|" +
+	"^css-|" +
+	"^critical-alerts$|" +
+	"^date$|" +
+	"detail_new_|" +
+	"downloadLink|" +
+	"^DYSRC$|" +
+	"^early-body|" +
+	"ec_blogs|" +
+	"^[^entry-]more.*$|" +
+	"error|" +
+	"[^-]facebook|" +
+	"facebook-broadcasting|" +
+	"^fb-root$|" +
+	"^feed[_-]|" +
+	"figcaption|" +
+	"footnote|" +
+	"foot|" +
+	"footer|" +
+	"^ga-track$|" +
+	" google |" +
+	"^gstl_|" +
+	"^GS-UH$|" +
+	"^guide$|" +
+	"header|" +
+	"hidden|" +
+	"img_popup_single|" +
+	"inline-share-tools|" +
+	"inread|" +
+	"^interstitial-ad-modal$|" +
+	"^Inv[0-9]$|" +
+	"js_replies|" +
+	"[Kk]ona[Ff]ilter|" +
+	"^kxhead$|" +
+	"leading|" +
+	"^lede[_-]container$|" +
+	"legende?|" +
+	"^lightningjs-|" +
+	"links|" +
+	"^login-modal$|" +
+	"^lui-mini-profile-body$|" +
+	"^marginalia|" +
+	"^marketing[_-]|" +
+	"^masthead|" +
+	"mediaarticlerelated|" +
+	"^media[_-]viewer$|" +
+	"menu|" +
+	"menucontainer|" +
+	"meta$|" +
+	"^moat$|" +
+	"moreNews|" +
+	"^Moses$|" +
+	"^nav[_-]|" +
+	"navbar|" +
+	"[Nn]avigation|" +
+	"newsUnder|" +
+	"^oauth|" +
+	"^overlay[_-]wrapper|" +
+	"pagetools|" +
+	"[_-]paid[_-]|" +
+	"panelss2|" +
+	"panesCity|" +
+	"player|" +
+	"PopularQuestions|" +
+	"popup|" +
+	"post[_-]attributes|" +
+	"post[_-]title|" +
+	"preview|" +
+	"[_-]print[_-]|" +
+	"products\\-events|" +
+	"^prop[0-9]$|" +
+	"^pulse-loaders|" +
+	"^rail$|" +
+	"recommend|" +
+	"^registration-modal$|" +
+	"relacionado|" +
+	"related|" +
+	"remote|" +
+	"retweet|" +
+	"^ribbon$|" +
+	"rightBlock|" +
+	"rss|" +
+	"runaroundLeft|" +
+	"search[_-]|" +
+	"share[_-]|" +
+	"shoutbox|" +
+	"sidebar|" +
+	"^simplereach$|" +
+	"^site[_-]index$|" +
+	"site[_-]box|" +
+	"site[_-]nav|" +
+	"skyscraper|" +
+	"social[Nn]etworking|" +
+	"social_|" +
+	"social\\-share|" +
+	"social\\-count|" +
+	"socialtools|" +
+	"source|" +
+	"^speed-bump-wrapper$|" +
+	"[_-]spinner$|" +
+	"^Splash$|" +
+	"sponsor|" +
+	"^spr-|" +
+	"storytopbar\\-bucket|" +
+	"^stream-sidebar|" +
+	"sub_nav|" +
+	"subscribe|" +
+	"subscription|" +
+	"^suggestions$|" +
+	"tabsCity|" +
+	"tag_|" +
+	"tags|" +
+	"teaser|" +
+	"the_answers|" +
+	"timestamp|" +
+	"tools|" +
+	"tooltip|" +
+	"^Top[0-9]?$|" +
+	"^TopAd[0-9]?$|" +
+	"[_-]track[_-]|" +
+	"tracking|" +
+	"[^-]twitter|" +
+	"-uix-button|" +
+	"updateBrowser|" +
+	"^username-modal$|" +
+	"^user-|" +
+	"utility-bar|" +
+	"^vestpocket$|" +
+	"vcard|" +
+	"^watch-action-panels$|" +
+	"^watch-discussion$|" +
+	"welcome_form|" +
+	"^whats[_-]next$|" +
+	"wp-caption-text")
+
+// Clean removes HTML elements around the main content and prepares the document for parsing
+func (c *Cleaner) Clean(docToClean *goquery.Document) *goquery.Document {
+	if c.config.debug {
+		log.Println("Starting cleaning phase with Cleaner")
+	}
+	docToClean = c.cleanBr(docToClean)
+	docToClean = c.cleanArticleTags(docToClean)
+	docToClean = c.cleanEMTags(docToClean)
+	docToClean = c.dropCaps(docToClean)
+	docToClean = c.removeScriptsStyle(docToClean)
+	docToClean = c.cleanBadTags(docToClean, removeNodesRegEx, &[]string{"id", "class", "name"})
+	docToClean = c.cleanBadTags(docToClean, removeVisibilityStyleRegEx, &[]string{"style"})
+	docToClean = c.removeTags(docToClean, &[]string{"nav", "footer", "aside", "cite"})
+	docToClean = c.cleanParaSpans(docToClean)
+
+
+	docToClean = c.convertDivsToParagraphs(docToClean, "div")
+
+	docToClean = c.convertDivsToParagraphs(docToClean, "span")
+	docToClean = c.convertDivsToParagraphs(docToClean, "article")
+	docToClean = c.convertDivsToParagraphs(docToClean, "pre")
+
+	return docToClean
+}
+
+func (c *Cleaner) cleanArticleTags(doc *goquery.Document) *goquery.Document {
+	tags := [3]string{"id", "name", "class"}
+	articles := doc.Find("article")
+	articles.Each(func(i int, s *goquery.Selection) {
+		for _, tag := range tags {
+			c.config.parser.delAttr(s, tag)
+		}
+	})
+	return doc
+}
+
+// replace <br /> with \n\n
+func (c *Cleaner) cleanBr(doc *goquery.Document) *goquery.Document {
+	linebreaks := doc.Find("br")
+	linebreaks.Each(func(i int, br *goquery.Selection) {
+		node := br.Get(0)
+		node.Data = "\n\n"
+		node.Type = html.TextNode
+		node.Attr = []html.Attribute{}
+		node.DataAtom = 0
+		node.FirstChild = nil
+		node.LastChild = nil
+	})
+	return doc
+}
+
+func (c *Cleaner) cleanEMTags(doc *goquery.Document) *goquery.Document {
+	ems := doc.Find("em")
+	ems.Each(func(i int, s *goquery.Selection) {
+		images := s.Find("img")
+		if images.Length() == 0 {
+			c.config.parser.dropTag(s)
+		}
+	})
+	if c.config.debug {
+		log.Printf("Cleaning %d EM tags\n", ems.Size())
+	}
+	return doc
+}
+
+func (c *Cleaner) removeTags(doc *goquery.Document, tags *[]string) *goquery.Document {
+	for _, tag := range *tags {
+		node := doc.Find(tag)
+		node.Each(func(i int, s *goquery.Selection) {
+			c.config.parser.removeNode(s)
+		})
+	}
+	return doc
+}
+
+func (c *Cleaner) cleanDivs(doc *goquery.Document) *goquery.Document {
+	frames := make(map[string]int)
+	framesNodes := make(map[string]*list.List)
+	divs := doc.Find("div")
+	divs.Each(func(i int, s *goquery.Selection) {
+		children := s.Children()
+		if children.Size() == 0 {
+			text := strings.Trim(s.Text(), " \t")
+			text = strings.ToLower(text)
+			frames[text]++
+			if framesNodes[text] == nil {
+				framesNodes[text] = list.New()
+			}
+			framesNodes[text].PushBack(s)
+		}
+	})
+	for text, freq := range frames {
+		if freq > 1 {
+			selections := framesNodes[text]
+			for s := selections.Front(); s != nil; s = s.Next() {
+				selection := s.Value.(*goquery.Selection)
+				c.config.parser.removeNode(selection)
+			}
+		}
+	}
+	return doc
+}
+
+func (c *Cleaner) dropCaps(doc *goquery.Document) *goquery.Document {
+	items := doc.Find("span")
+	count := 0 // remove
+	items.Each(func(i int, s *goquery.Selection) {
+		attribute, exists := s.Attr("class")
+		if exists && (strings.Contains(attribute, "dropcap") || strings.Contains(attribute, "drop_cap")) {
+			c.config.parser.dropTag(s)
+			count++
+		}
+	})
+	if c.config.debug && count > 0 {
+		log.Printf("Cleaned %d dropcap tags\n", count)
+	}
+	return doc
+}
+
+func (c *Cleaner) removeScriptsStyle(doc *goquery.Document) *goquery.Document {
+	if c.config.debug {
+		log.Println("Starting to remove script tags")
+	}
+	count := 0 // number of removed nodes
+	scripts := doc.Find("script,noscript,style")
+	scripts.Each(func(i int, s *goquery.Selection) {
+		c.config.parser.removeNode(s)
+		count++
+	})
+	if c.config.debug && count > 0 {
+		log.Printf("Removed %d script and style tags\n", scripts.Size())
+	}
+	return doc
+}
+
+func (c *Cleaner) cleanBadTags(doc *goquery.Document, pattern *regexp.Regexp, selectors *[]string) *goquery.Document {
+	body := doc.Find("html")
+	children := body.Children()
+	children.Each(func(i int, s *goquery.Selection) {
+		for _, selector := range *selectors {
+			naughtyList := s.Find("*[" + selector + "]")
+			count := 0
+			naughtyList.Each(func(j int, node *goquery.Selection) {
+				attribute, _ := node.Attr(selector)
+				if pattern.MatchString(attribute) {
+					if c.config.debug {
+						log.Printf("Cleaning: Removing node with %s: %s\n", selector, c.config.parser.name(selector, node))
+					}
+					c.config.parser.removeNode(node)
+					count++
+				}
+			})
+			if c.config.debug && count > 0 {
+				log.Printf("%d naughty %s elements found", count, selector)
+			}
+		}
+	})
+	return doc
+}
+
+// Replace <p><span>...</span></p>   with  <p>...</p>
+func (c *Cleaner) cleanParaSpans(doc *goquery.Document) *goquery.Document {
+	spans := doc.Find("span")
+	spans.Each(func(i int, s *goquery.Selection) {
+		parent := s.Parent()
+		if parent != nil && parent.Length() > 0 && parent.Get(0).DataAtom == atom.P {
+			replaceTagWithContents(s, whitelistedTextAtomTypes)
+		}
+	})
+	return doc
+}
+
+func (c *Cleaner) getFlushedBuffer(fragment string) []*html.Node {
+	var output []*html.Node
+	reader := strings.NewReader(fragment)
+	document, _ := html.Parse(reader)
+	body := document.FirstChild.LastChild
+	for c := body.FirstChild; c != nil; c = c.NextSibling {
+		output = append(output, c)
+		c.Parent = nil
+		c.PrevSibling = nil
+	}
+
+	for _, o := range output {
+		o.NextSibling = nil
+	}
+	return output
+}
+
+func (c *Cleaner) replaceWithPara(div *goquery.Selection) {
+	if div.Size() > 0 {
+		node := div.Get(0)
+		node.Data = atom.P.String()
+		node.DataAtom = atom.P
+		node.Attr = []html.Attribute{}
+	}
+}
+
+func (c *Cleaner) tabsAndNewLinesReplacements(text string) string {
+	text = strings.Replace(text, "\n", "\n\n", -1)
+	text = tabsRegEx.ReplaceAllString(text, "")
+	return text
+}
+
+func (c *Cleaner) convertDivsToParagraphs(doc *goquery.Document, domType string) *goquery.Document {
+	if c.config.debug {
+		log.Println("Starting to replace bad divs...")
+	}
+	badDivs := 0
+	convertedTextNodes := 0
+	divs := doc.Find(domType)
+
+	divs.Each(func(i int, div *goquery.Selection) {
+		divHTML, _ := div.Html()
+		if divToPElementsPattern.Match([]byte(divHTML)) {
+			c.replaceWithPara(div)
+			badDivs++
+		} else {
+			var replacementText []string
+			nodesToRemove := list.New()
+			children := div.Contents()
+			if c.config.debug {
+				log.Printf("Found %d children of div\n", children.Size())
+			}
+			children.EachWithBreak(func(i int, kid *goquery.Selection) bool {
+				text := kid.Text()
+				kidNode := kid.Get(0)
+				tag := kidNode.Data
+				if tag == text {
+					tag = "#text"
+				}
+				if tag == "#text" {
+					text = strings.Replace(text, "\n", "", -1)
+					text = tabsRegEx.ReplaceAllString(text, "")
+					if text == "" {
+						return true
+					}
+					if len(text) > 1 {
+						prev := kidNode.PrevSibling
+						if c.config.debug {
+							log.Printf("PARENT CLASS: %s NODENAME: %s\n", c.config.parser.name("class", div), tag)
+							log.Printf("TEXTREPLACE: %s\n", strings.Replace(text, "\n", "", -1))
+						}
+						if prev != nil && prev.DataAtom == atom.A {
+							nodeSelection := kid.HasNodes(prev)
+							html, _ := nodeSelection.Html()
+							replacementText = append(replacementText, html)
+							if c.config.debug {
+								log.Printf("SIBLING NODENAME ADDITION: %s TEXT: %s\n", prev.Data, html)
+							}
+						}
+						replacementText = append(replacementText, text)
+						nodesToRemove.PushBack(kidNode)
+						convertedTextNodes++
+					}
+
+				}
+				return true
+			})
+
+			/*
+			newNode := new(html.Node)
+			newNode.Type = html.ElementNode
+			newNode.Data = strings.Join(replacementText, "")
+			newNode.DataAtom = atom.P
+			*/
+/*
+			replacementText = strings.Replace(replacementText, "=C3=A8", "è")
+			replacementText = strings.Replace(replacementText, "=C3=A9", "é")
+*/
+			div.First().BeforeHtml("<p>" + strings.Join(replacementText, "") + "</p>")
+
+			for s := nodesToRemove.Front(); s != nil; s = s.Next() {
+				node := s.Value.(*html.Node)
+				if node != nil && node.Parent != nil {
+					node.Parent.RemoveChild(node)
+				}
+			}
+		}
+	})
+	if c.config.debug {
+		log.Printf("Found %d total divs with %d bad divs replaced and %d textnodes converted inside divs", divs.Size(), badDivs, convertedTextNodes)
+	}
+	return doc
+
+}

vendor/github.com/advancedlogic/GoOse/configuration.go

@@ -0,0 +1,69 @@
+package goose
+
+import (
+	"time"
+)
+
+const defaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/534.52.7 (KHTML, like Gecko) Version/5.1.2 Safari/534.52.7"
+
+// Configuration is a wrapper for various config options
+type Configuration struct {
+	localStoragePath        string //not used in this version
+	imagesMinBytes          int    //not used in this version
+	targetLanguage          string
+	imageMagickConvertPath  string //not used in this version
+	imageMagickIdentifyPath string //not used in this version
+	browserUserAgent        string
+	debug                   bool
+	extractPublishDate      bool
+	additionalDataExtractor bool
+	enableImageFetching     bool
+	useMetaLanguage         bool
+
+	//path to the stopwords folder
+	stopWordsPath string
+	stopWords     StopWords
+	parser        *Parser
+
+	timeout time.Duration
+}
+
+// GetDefaultConfiguration returns safe default configuration options
+func GetDefaultConfiguration(args ...string) Configuration {
+	if len(args) == 0 {
+		return Configuration{
+			localStoragePath:        "",   //not used in this version
+			imagesMinBytes:          4500, //not used in this version
+			enableImageFetching:     true,
+			useMetaLanguage:         true,
+			targetLanguage:          "en",
+			imageMagickConvertPath:  "/usr/bin/convert",  //not used in this version
+			imageMagickIdentifyPath: "/usr/bin/identify", //not used in this version
+			browserUserAgent:        defaultUserAgent,
+			debug:                   false,
+			extractPublishDate:      true,
+			additionalDataExtractor: false,
+			stopWordsPath:           "resources/stopwords",
+			stopWords:               NewStopwords(), //TODO with path
+			parser:                  NewParser(),
+			timeout:                 time.Duration(5 * time.Second),
+		}
+	}
+	return Configuration{
+		localStoragePath:        "",   //not used in this version
+		imagesMinBytes:          4500, //not used in this version
+		enableImageFetching:     true,
+		useMetaLanguage:         true,
+		targetLanguage:          "en",
+		imageMagickConvertPath:  "/usr/bin/convert",  //not used in this version
+		imageMagickIdentifyPath: "/usr/bin/identify", //not used in this version
+		browserUserAgent:        defaultUserAgent,
+		debug:                   false,
+		extractPublishDate:      true,
+		additionalDataExtractor: false,
+		stopWordsPath:           "resources/stopwords",
+		stopWords:               NewStopwords(), //TODO with path
+		parser:                  NewParser(),
+		timeout:                 time.Duration(5 * time.Second),
+	}
+}

vendor/github.com/advancedlogic/GoOse/coverage.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Run test coverage on each subdirectory and merge the coverage profile.
+echo "mode: count" > target/report/profile.cov
+
+# Standard go tooling behavior is to ignore dirs with leading underscors
+for dir in $(find . -maxdepth 10 -not -path './.git*' -not -path '*/_*' -type d); do
+    if ls $dir/*.go &> /dev/null; then
+        go test -covermode=count -coverprofile=$dir/profile.tmp $dir
+        if [ -f $dir/profile.tmp ]; then
+            cat $dir/profile.tmp | tail -n +2 >> target/report/profile.cov
+            rm $dir/profile.tmp
+        fi
+    fi
+done
+go tool cover -html target/report/profile.cov -o target/report/coverage.html
+

vendor/github.com/advancedlogic/GoOse/crawler.go

@@ -0,0 +1,180 @@
+package goose
+
+import (
+	"errors"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+// Crawler can fetch the target HTML page
+type Crawler struct {
+	config  Configuration
+	Charset string
+}
+
+// NewCrawler returns a crawler object initialised with the URL and the [optional] raw HTML body
+func NewCrawler(config Configuration) Crawler {
+	return Crawler{
+		config:  config,
+		Charset: "",
+	}
+}
+
+func getCharsetFromContentType(cs string) string {
+	cs = strings.ToLower(strings.Replace(cs, " ", "", -1))
+	cs = strings.TrimPrefix(cs, "text/html;charset=")
+	cs = strings.TrimPrefix(cs, "text/xhtml;charset=")
+	cs = strings.TrimPrefix(cs, "application/xhtml+xml;charset=")
+	return NormaliseCharset(cs)
+}
+
+// SetCharset can be used to force a charset (e.g. when read from the HTTP headers)
+// rather than relying on the detection from the HTML meta tags
+func (c *Crawler) SetCharset(cs string) {
+	c.Charset = getCharsetFromContentType(cs)
+}
+
+// GetContentType returns the Content-Type string extracted from the meta tags
+func (c Crawler) GetContentType(document *goquery.Document) string {
+	var attr string
+	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+	document.Find("meta[http-equiv#=(?i)^Content\\-type$]").Each(func(i int, s *goquery.Selection) {
+		attr, _ = s.Attr("content")
+	})
+	return attr
+}
+
+// GetCharset returns a normalised charset string extracted from the meta tags
+func (c Crawler) GetCharset(document *goquery.Document) string {
+	// manually-provided charset (from HTTP headers?) takes priority
+	if "" != c.Charset {
+		return c.Charset
+	}
+
+	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+	ct := c.GetContentType(document)
+	if "" != ct && strings.Contains(strings.ToLower(ct), "charset") {
+		return getCharsetFromContentType(ct)
+	}
+
+	// <meta charset="utf-8">
+	selection := document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
+		_, exists := s.Attr("charset")
+		return !exists
+	})
+
+	if selection != nil {
+		cs, _ := selection.Attr("charset")
+		return NormaliseCharset(cs)
+	}
+
+	return ""
+}
+
+// Preprocess fetches the HTML page if needed, converts it to UTF-8 and applies
+// some text normalisation to guarantee better results when extracting the content
+func (c *Crawler) Preprocess(RawHTML string) (*goquery.Document, error) {
+	var err error
+
+	if RawHTML == "" {
+		return nil, errors.New("cannot process empty HTML content")
+	}
+
+	RawHTML = c.addSpacesBetweenTags(RawHTML)
+
+	reader := strings.NewReader(RawHTML)
+	document, err := goquery.NewDocumentFromReader(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	cs := c.GetCharset(document)
+	//log.Println("-------------------------------------------CHARSET:", cs)
+	if "" != cs && "UTF-8" != cs {
+		// the net/html parser and goquery require UTF-8 data
+		RawHTML = UTF8encode(RawHTML, cs)
+		reader = strings.NewReader(RawHTML)
+		if document, err = goquery.NewDocumentFromReader(reader); err != nil {
+			return nil, err
+		}
+	}
+
+	return document, nil
+}
+
+// Crawl fetches the HTML body and returns an Article
+func (c Crawler) Crawl(RawHTML string, url string) (*Article, error) {
+	article := new(Article)
+
+	document, err := c.Preprocess(RawHTML)
+	if nil != err {
+		return nil, err
+	}
+	if nil == document {
+		return article, nil
+	}
+	extractor := NewExtractor(c.config)
+	startTime := time.Now().UnixNano()
+
+	article.RawHTML, err = document.Html()
+	if nil != err {
+		return nil, err
+	}
+	article.FinalURL = url
+	article.Doc = document
+
+	article.TitleUnmodified = extractor.getTitleUnmodified(document)
+	article.Title = extractor.GetTitleFromUnmodifiedTitle(article.TitleUnmodified)
+	article.MetaLang = extractor.GetMetaLanguage(document)
+	article.MetaFavicon = extractor.GetFavicon(document)
+
+	article.MetaDescription = extractor.GetMetaContentWithSelector(document, "meta[name#=(?i)^description$]")
+	article.MetaKeywords = extractor.GetMetaContentWithSelector(document, "meta[name#=(?i)^keywords$]")
+	article.CanonicalLink = extractor.GetCanonicalLink(document)
+	if "" == article.CanonicalLink {
+		article.CanonicalLink = article.FinalURL
+	}
+	article.Domain = extractor.GetDomain(article.CanonicalLink)
+	article.Tags = extractor.GetTags(document)
+
+	if c.config.extractPublishDate {
+		if timestamp := extractor.GetPublishDate(document); timestamp != nil {
+			article.PublishDate = timestamp
+		}
+	}
+
+	cleaner := NewCleaner(c.config)
+	article.Doc = cleaner.Clean(article.Doc)
+
+	article.TopImage = OpenGraphResolver(document)
+	if article.TopImage == "" {
+		article.TopImage = WebPageResolver(article)
+	}
+
+	article.TopNode = extractor.CalculateBestNode(document)
+	if article.TopNode != nil {
+		article.TopNode = extractor.PostCleanup(article.TopNode)
+
+		article.CleanedText, article.Links = extractor.GetCleanTextAndLinks(article.TopNode, article.MetaLang)
+
+		videoExtractor := NewVideoExtractor()
+		article.Movies = videoExtractor.GetVideos(document)
+	}
+
+	article.Delta = time.Now().UnixNano() - startTime
+
+	return article, nil
+}
+
+// In many cases, like at the end of each <li> element or between </span><span> tags,
+// we need to add spaces, otherwise the text on either side will get joined together into one word.
+// This method also adds newlines after each </p> tag to preserve paragraphs.
+func (c Crawler) addSpacesBetweenTags(text string) string {
+	text = strings.Replace(text, "><", "> <", -1)
+	text = strings.Replace(text, "</blockquote>", "</blockquote>\n", -1)
+	text = strings.Replace(text, "<img ", "\n<img ", -1)
+	text = strings.Replace(text, "</li>", "</li>\n", -1)
+	return strings.Replace(text, "</p>", "</p>\n", -1)
+}

vendor/github.com/advancedlogic/GoOse/crawlershort.go

@@ -0,0 +1,155 @@
+package goose
+
+import (
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/pkg/errors"
+)
+
+// Crawler can fetch the target HTML page
+type CrawlerShort struct {
+	config  Configuration
+	Charset string
+}
+
+// NewCrawler returns a crawler object initialised with the URL and the [optional] raw HTML body
+func NewCrawlerShort(config Configuration) CrawlerShort {
+	return CrawlerShort{
+		config:  config,
+		Charset: "",
+	}
+}
+
+// SetCharset can be used to force a charset (e.g. when read from the HTTP headers)
+// rather than relying on the detection from the HTML meta tags
+func (c *CrawlerShort) SetCharset(cs string) {
+	c.Charset = getCharsetFromContentType(cs)
+}
+
+// GetContentType returns the Content-Type string extracted from the meta tags
+func (c CrawlerShort) GetContentType(document *goquery.Document) string {
+	var attr string
+	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+	document.Find("meta[http-equiv#=(?i)^Content\\-type$]").Each(func(i int, s *goquery.Selection) {
+		attr, _ = s.Attr("content")
+	})
+	return attr
+}
+
+// GetCharset returns a normalised charset string extracted from the meta tags
+func (c CrawlerShort) GetCharset(document *goquery.Document) string {
+	// manually-provided charset (from HTTP headers?) takes priority
+	if "" != c.Charset {
+		return c.Charset
+	}
+
+	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+	ct := c.GetContentType(document)
+	if "" != ct && strings.Contains(strings.ToLower(ct), "charset") {
+		return getCharsetFromContentType(ct)
+	}
+
+	// <meta charset="utf-8">
+	selection := document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
+		_, exists := s.Attr("charset")
+		return !exists
+	})
+
+	if selection != nil {
+		cs, _ := selection.Attr("charset")
+		return NormaliseCharset(cs)
+	}
+
+	return ""
+}
+
+// Preprocess fetches the HTML page if needed, converts it to UTF-8 and applies
+// some text normalisation to guarantee better results when extracting the content
+func (c *CrawlerShort) Preprocess(RawHTML string) (*goquery.Document, error) {
+	var err error
+
+	RawHTML = c.addSpacesBetweenTags(RawHTML)
+
+	reader := strings.NewReader(RawHTML)
+	document, err := goquery.NewDocumentFromReader(reader)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not perform goquery.NewDocumentFromReader(reader)")
+	}
+
+	cs := c.GetCharset(document)
+	//log.Println("-------------------------------------------CHARSET:", cs)
+	if "" != cs && "UTF-8" != cs {
+		// the net/html parser and goquery require UTF-8 data
+		RawHTML = UTF8encode(RawHTML, cs)
+		reader = strings.NewReader(RawHTML)
+		if document, err = goquery.NewDocumentFromReader(reader); err != nil {
+			return nil, errors.Wrap(err, "could not perform goquery.NewDocumentFromReader(reader)")
+		}
+	}
+
+	return document, nil
+}
+
+// Crawl fetches the HTML body and returns an Article
+func (c CrawlerShort) Crawl(RawHTML, url string) (*Article, error) {
+	article := new(Article)
+
+	document, err := c.Preprocess(RawHTML)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not Preprocess RawHTML")
+	}
+	if document == nil {
+		return article, nil
+	}
+
+	extractor := NewExtractor(c.config)
+
+	startTime := time.Now().UnixNano()
+
+	article.RawHTML, err = document.Html()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get html from document")
+	}
+	article.FinalURL = url
+
+	article.Title = extractor.GetTitle(document)
+	article.MetaDescription = extractor.GetMetaContentWithSelector(document, "meta[name#=(?i)^description$]")
+
+	if c.config.extractPublishDate {
+		if timestamp := extractor.GetPublishDate(document); timestamp != nil {
+			article.PublishDate = timestamp
+		}
+	}
+
+	cleaner := NewCleaner(c.config)
+	article.Doc = cleaner.Clean(article.Doc)
+
+	article.TopImage = OpenGraphResolver(document)
+	if article.TopImage == "" {
+		article.TopImage = WebPageResolver(article)
+	}
+
+	article.TopNode = extractor.CalculateBestNode(document)
+	if article.TopNode != nil {
+		article.TopNode = extractor.PostCleanup(article.TopNode)
+
+		article.CleanedText, article.Links = extractor.GetCleanTextAndLinks(article.TopNode, article.MetaLang)
+
+	}
+	article.Delta = time.Now().UnixNano() - startTime
+
+	return article, nil
+}
+
+// In many cases, like at the end of each <li> element or between </span><span> tags,
+// we need to add spaces, otherwise the text on either side will get joined together into one word.
+// This method also adds newlines after each </p> tag to preserve paragraphs.
+func (c CrawlerShort) addSpacesBetweenTags(text string) string {
+	text = strings.Replace(text, "><", "> <", -1)
+	text = strings.Replace(text, "</blockquote>", "</blockquote>\n", -1)
+	text = strings.Replace(text, "<img ", "\n<img ", -1)
+	text = strings.Replace(text, "</li>", "</li>\n", -1)
+	return strings.Replace(text, "</p>", "</p>\n", -1)
+}

vendor/github.com/advancedlogic/GoOse/doc.go

@@ -0,0 +1,23 @@
+/*
+This is a golang port of "Goose" originally licensed to Gravity.com
+under one or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.
+
+Golang port was written by Antonio Linari
+
+Gravity.com licenses this file
+to you under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package goose

vendor/github.com/advancedlogic/GoOse/extractor.go

@@ -0,0 +1,754 @@
+package goose
+
+import (
+	"container/list"
+	"log"
+	"math"
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/araddon/dateparse"
+	"github.com/fatih/set"
+	"github.com/gigawattio/window"
+	"github.com/jaytaylor/html2text"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+const defaultLanguage = "en"
+
+var motleyReplacement = "�" // U+FFFD (decimal 65533) is the "replacement character".
+//var escapedFragmentReplacement = regexp.MustCompile("#!")
+//var titleReplacements = regexp.MustCompile("»")
+
+var titleDelimiters = []string{
+	"|",
+	" - ",
+	" — ",
+	"»",
+	":",
+}
+
+var aRelTagSelector = "a[rel=tag]"
+var aHrefTagSelector = [...]string{"/tag/", "/tags/", "/topic/", "?keyword"}
+
+//var langRegEx = "^[A-Za-z]{2}$"
+
+// ContentExtractor can parse the HTML and fetch various properties
+type ContentExtractor struct {
+	config Configuration
+}
+
+// NewExtractor returns a configured HTML parser
+func NewExtractor(config Configuration) ContentExtractor {
+	return ContentExtractor{
+		config: config,
+	}
+}
+
+//if the article has a title set in the source, use that
+func (extr *ContentExtractor) getTitleUnmodified(document *goquery.Document) string {
+	title := ""
+
+	titleElement := document.Find("title")
+	if titleElement != nil && titleElement.Size() > 0 {
+		title = titleElement.Text()
+	}
+
+	if title == "" {
+		ogTitleElement := document.Find(`meta[property="og:title"]`)
+		if ogTitleElement != nil && ogTitleElement.Size() > 0 {
+			title, _ = ogTitleElement.Attr("content")
+		}
+	}
+
+	if title == "" {
+		titleElement = document.Find("post-title,headline")
+		if titleElement == nil || titleElement.Size() == 0 {
+			return title
+		}
+		title = titleElement.Text()
+	}
+	return title
+}
+
+// GetTitleFromUnmodifiedTitle returns the title from the unmodified one
+func (extr *ContentExtractor) GetTitleFromUnmodifiedTitle(title string) string {
+	for _, delimiter := range titleDelimiters {
+		if strings.Contains(title, delimiter) {
+			title = extr.splitTitle(strings.Split(title, delimiter))
+			break
+		}
+	}
+
+	title = strings.Replace(title, motleyReplacement, "", -1)
+
+	if extr.config.debug {
+		log.Printf("Page title is %s\n", title)
+	}
+
+	return strings.TrimSpace(title)
+}
+
+// GetTitle returns the title set in the source, if the article has one
+func (extr *ContentExtractor) GetTitle(document *goquery.Document) string {
+	title := extr.getTitleUnmodified(document)
+	return extr.GetTitleFromUnmodifiedTitle(title)
+}
+
+func (extr *ContentExtractor) splitTitle(titles []string) string {
+	largeTextLength := 0
+	largeTextIndex := 0
+	for i, current := range titles {
+		if len(current) > largeTextLength {
+			largeTextLength = len(current)
+			largeTextIndex = i
+		}
+	}
+	title := titles[largeTextIndex]
+	title = strings.Replace(title, "»", "»", -1)
+	return title
+}
+
+// GetMetaLanguage returns the meta language set in the source, if the article has one
+func (extr *ContentExtractor) GetMetaLanguage(document *goquery.Document) string {
+	var language string
+	shtml := document.Find("html")
+	attr, _ := shtml.Attr("lang")
+	if attr == "" {
+		attr, _ = document.Attr("lang")
+	}
+	if attr == "" {
+		selection := document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
+			var exists bool
+			attr, exists = s.Attr("http-equiv")
+			if exists && attr == "content-language" {
+				return false
+			}
+			return true
+		})
+		if selection != nil {
+			attr, _ = selection.Attr("content")
+		}
+	}
+	idx := strings.LastIndex(attr, "-")
+	if idx == -1 {
+		language = attr
+	} else {
+		language = attr[0:idx]
+	}
+
+	_, ok := sw[language]
+
+	if language == "" || !ok {
+		language = extr.config.stopWords.SimpleLanguageDetector(shtml.Text())
+		if language == "" {
+			language = defaultLanguage
+		}
+	}
+
+	extr.config.targetLanguage = language
+	return language
+}
+
+// GetFavicon returns the favicon set in the source, if the article has one
+func (extr *ContentExtractor) GetFavicon(document *goquery.Document) string {
+	favicon := ""
+	document.Find("link").EachWithBreak(func(i int, s *goquery.Selection) bool {
+		attr, exists := s.Attr("rel")
+		if exists && strings.Contains(attr, "icon") {
+			favicon, _ = s.Attr("href")
+			return false
+		}
+		return true
+	})
+	return favicon
+}
+
+// GetMetaContentWithSelector returns the content attribute of meta tag matching the selector
+func (extr *ContentExtractor) GetMetaContentWithSelector(document *goquery.Document, selector string) string {
+	selection := document.Find(selector)
+	content, _ := selection.Attr("content")
+	return strings.TrimSpace(content)
+}
+
+// GetMetaContent returns the content attribute of meta tag with the given property name
+func (extr *ContentExtractor) GetMetaContent(document *goquery.Document, metaName string) string {
+	content := ""
+	document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
+		attr, exists := s.Attr("name")
+		if exists && attr == metaName {
+			content, _ = s.Attr("content")
+			return false
+		}
+		attr, exists = s.Attr("itemprop")
+		if exists && attr == metaName {
+			content, _ = s.Attr("content")
+			return false
+		}
+		return true
+	})
+	return content
+}
+
+// GetMetaContents returns all the meta tags as name->content pairs
+func (extr *ContentExtractor) GetMetaContents(document *goquery.Document, metaNames *set.Set) map[string]string {
+	contents := make(map[string]string)
+	counter := metaNames.Size()
+	document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
+		attr, exists := s.Attr("name")
+		if exists && metaNames.Has(attr) {
+			content, _ := s.Attr("content")
+			contents[attr] = content
+			counter--
+			if counter < 0 {
+				return false
+			}
+		}
+		return true
+	})
+	return contents
+}
+
+// GetMetaDescription returns the meta description set in the source, if the article has one
+func (extr *ContentExtractor) GetMetaDescription(document *goquery.Document) string {
+	return extr.GetMetaContent(document, "description")
+}
+
+// GetMetaKeywords returns the meta keywords set in the source, if the article has them
+func (extr *ContentExtractor) GetMetaKeywords(document *goquery.Document) string {
+	return extr.GetMetaContent(document, "keywords")
+}
+
+// GetMetaAuthor returns the meta author set in the source, if the article has one
+func (extr *ContentExtractor) GetMetaAuthor(document *goquery.Document) string {
+	return extr.GetMetaContent(document, "author")
+}
+
+// GetMetaContentLocation returns the meta content location set in the source, if the article has one
+func (extr *ContentExtractor) GetMetaContentLocation(document *goquery.Document) string {
+	return extr.GetMetaContent(document, "contentLocation")
+}
+
+// GetCanonicalLink returns the meta canonical link set in the source
+func (extr *ContentExtractor) GetCanonicalLink(document *goquery.Document) string {
+	metas := document.Find("link[rel=canonical]")
+	if metas.Length() > 0 {
+		meta := metas.First()
+		href, _ := meta.Attr("href")
+		href = strings.Trim(href, "\n")
+		href = strings.Trim(href, " ")
+		if href != "" {
+			return href
+		}
+	}
+	return ""
+}
+
+// GetDomain extracts the domain from a link
+func (extr *ContentExtractor) GetDomain(canonicalLink string) string {
+	u, err := url.Parse(canonicalLink)
+	if err == nil {
+		return u.Host
+	}
+	return ""
+}
+
+// GetTags returns the tags set in the source, if the article has them
+func (extr *ContentExtractor) GetTags(document *goquery.Document) *set.Set {
+	tags := set.New(set.ThreadSafe).(*set.Set)
+	selections := document.Find(aRelTagSelector)
+	selections.Each(func(i int, s *goquery.Selection) {
+		tags.Add(s.Text())
+	})
+	selections = document.Find("a")
+	selections.Each(func(i int, s *goquery.Selection) {
+		href, exists := s.Attr("href")
+		if exists {
+			for _, part := range aHrefTagSelector {
+				if strings.Contains(href, part) {
+					tags.Add(s.Text())
+				}
+			}
+		}
+	})
+
+	return tags
+}
+
+// GetPublishDate returns the publication date, if one can be located.
+func (extr *ContentExtractor) GetPublishDate(document *goquery.Document) *time.Time {
+	raw, err := document.Html()
+	if err != nil {
+		log.Printf("Error converting document HTML nodes to raw HTML: %s (publish date detection aborted)\n", err)
+		return nil
+	}
+
+	text, err := html2text.FromString(raw)
+	if err != nil {
+		log.Printf("Error converting document HTML to plaintext: %s (publish date detection aborted)\n", err)
+		return nil
+	}
+
+	text = strings.ToLower(text)
+
+	// Simplify months because the dateparse pkg only handles abbreviated.
+	for k, v := range map[string]string{
+		"january":  "jan",
+		"march":    "mar",
+		"february": "feb",
+		"april":    "apr",
+		// "may":       "may", // Pointless.
+		"june":      "jun",
+		"august":    "aug",
+		"september": "sep",
+		"sept":      "sep",
+		"october":   "oct",
+		"november":  "nov",
+		"december":  "dec",
+		"th,":       ",", // Strip day number suffixes.
+		"rd,":       ",",
+	} {
+		text = strings.Replace(text, k, v, -1)
+	}
+	text = strings.Replace(text, "\n", " ", -1)
+	text = regexp.MustCompile(" +").ReplaceAllString(text, " ")
+
+	tuple1 := strings.Split(text, " ")
+
+	var (
+		expr  = regexp.MustCompile("[0-9]")
+		ts    time.Time
+		found bool
+	)
+	for _, n := range []int{3, 4, 5, 2, 6} {
+		for _, win := range window.Rolling(tuple1, n) {
+			if !expr.MatchString(strings.Join(win, " ")) {
+				continue
+			}
+
+			input := strings.Join(win, " ")
+			ts, err = dateparse.ParseAny(input)
+			if err == nil && ts.Year() > 0 && ts.Month() > 0 && ts.Day() > 0 {
+				found = true
+				break
+			}
+
+			// Try injecting a comma for dateparse.
+			win[1] = win[1] + ","
+			input = strings.Join(win, " ")
+			ts, err = dateparse.ParseAny(input)
+			if err == nil && ts.Year() > 0 && ts.Month() > 0 && ts.Day() > 0 {
+				found = true
+				break
+			}
+		}
+		if found {
+			break
+		}
+	}
+
+	if found {
+		return &ts
+	}
+	return nil
+}
+
+// GetCleanTextAndLinks parses the main HTML node for text and links
+func (extr *ContentExtractor) GetCleanTextAndLinks(topNode *goquery.Selection, lang string) (string, []string) {
+	outputFormatter := new(outputFormatter)
+	outputFormatter.config = extr.config
+	return outputFormatter.getFormattedText(topNode, lang)
+}
+
+// CalculateBestNode checks for the HTML node most likely to contain the main content.
+//we're going to start looking for where the clusters of paragraphs are. We'll score a cluster based on the number of stopwords
+//and the number of consecutive paragraphs together, which should form the cluster of text that this node is around
+//also store on how high up the paragraphs are, comments are usually at the bottom and should get a lower score
+func (extr *ContentExtractor) CalculateBestNode(document *goquery.Document) *goquery.Selection {
+	var topNode *goquery.Selection
+	nodesToCheck := extr.nodesToCheck(document)
+	if extr.config.debug {
+		log.Printf("Nodes to check %d\n", len(nodesToCheck))
+	}
+	startingBoost := 1.0
+	cnt := 0
+	i := 0
+	parentNodes := set.New(set.ThreadSafe).(*set.Set)
+	nodesWithText := list.New()
+	for _, node := range nodesToCheck {
+		textNode := node.Text()
+		ws := extr.config.stopWords.stopWordsCount(extr.config.targetLanguage, textNode)
+		highLinkDensity := extr.isHighLinkDensity(node)
+		if ws.stopWordCount > 2 && !highLinkDensity {
+			nodesWithText.PushBack(node)
+		}
+	}
+	nodesNumber := nodesWithText.Len()
+	negativeScoring := 0
+	bottomNegativeScoring := float64(nodesNumber) * 0.25
+
+	if extr.config.debug {
+		log.Printf("About to inspect num of nodes with text %d\n", nodesNumber)
+	}
+
+	for n := nodesWithText.Front(); n != nil; n = n.Next() {
+		node := n.Value.(*goquery.Selection)
+		boostScore := 0.0
+		if extr.isBoostable(node) {
+			if cnt >= 0 {
+				boostScore = float64((1.0 / startingBoost) * 50)
+				startingBoost++
+			}
+		}
+
+		if nodesNumber > 15 {
+			if float64(nodesNumber-i) <= bottomNegativeScoring {
+				booster := bottomNegativeScoring - float64(nodesNumber-i)
+				boostScore = -math.Pow(booster, 2.0)
+				negScore := math.Abs(boostScore) + float64(negativeScoring)
+				if negScore > 40 {
+					boostScore = 5.0
+				}
+			}
+		}
+
+		if extr.config.debug {
+			log.Printf("Location Boost Score %1.5f on iteration %d id='%s' class='%s'\n", boostScore, i, extr.config.parser.name("id", node), extr.config.parser.name("class", node))
+		}
+		textNode := node.Text()
+		ws := extr.config.stopWords.stopWordsCount(extr.config.targetLanguage, textNode)
+		upScore := ws.stopWordCount + int(boostScore)
+		parentNode := node.Parent()
+		extr.updateScore(parentNode, upScore)
+		extr.updateNodeCount(parentNode, 1)
+		if !parentNodes.Has(parentNode) {
+			parentNodes.Add(parentNode)
+		}
+		parentParentNode := parentNode.Parent()
+		if parentParentNode != nil {
+			extr.updateNodeCount(parentParentNode, 1)
+			extr.updateScore(parentParentNode, upScore/2.0)
+			if !parentNodes.Has(parentParentNode) {
+				parentNodes.Add(parentParentNode)
+			}
+		}
+		cnt++
+		i++
+	}
+
+	topNodeScore := 0
+	parentNodesArray := parentNodes.List()
+	for _, p := range parentNodesArray {
+		e := p.(*goquery.Selection)
+		if extr.config.debug {
+			log.Printf("ParentNode: score=%s nodeCount=%s id='%s' class='%s'\n", extr.config.parser.name("gravityScore", e), extr.config.parser.name("gravityNodes", e), extr.config.parser.name("id", e), extr.config.parser.name("class", e))
+		}
+		score := extr.getScore(e)
+		if score >= topNodeScore {
+			topNode = e
+			topNodeScore = score
+		}
+		if topNode == nil {
+			topNode = e
+		}
+	}
+	return topNode
+}
+
+//returns the gravityScore as an integer from this node
+func (extr *ContentExtractor) getScore(node *goquery.Selection) int {
+	return extr.getNodeGravityScore(node)
+}
+
+func (extr *ContentExtractor) getNodeGravityScore(node *goquery.Selection) int {
+	grvScoreString, exists := node.Attr("gravityScore")
+	if !exists {
+		return 0
+	}
+	grvScore, err := strconv.Atoi(grvScoreString)
+	if err != nil {
+		return 0
+	}
+	return grvScore
+}
+
+//adds a score to the gravityScore Attribute we put on divs
+//we'll get the current score then add the score we're passing in to the current
+func (extr *ContentExtractor) updateScore(node *goquery.Selection, addToScore int) {
+	currentScore := 0
+	var err error
+	scoreString, _ := node.Attr("gravityScore")
+	if scoreString != "" {
+		currentScore, err = strconv.Atoi(scoreString)
+		if err != nil {
+			currentScore = 0
+		}
+	}
+	newScore := currentScore + addToScore
+	extr.config.parser.setAttr(node, "gravityScore", strconv.Itoa(newScore))
+}
+
+//stores how many decent nodes are under a parent node
+func (extr *ContentExtractor) updateNodeCount(node *goquery.Selection, addToCount int) {
+	currentScore := 0
+	var err error
+	scoreString, _ := node.Attr("gravityNodes")
+	if scoreString != "" {
+		currentScore, err = strconv.Atoi(scoreString)
+		if err != nil {
+			currentScore = 0
+		}
+	}
+	newScore := currentScore + addToCount
+	extr.config.parser.setAttr(node, "gravityNodes", strconv.Itoa(newScore))
+}
+
+//a lot of times the first paragraph might be the caption under an image so we'll want to make sure if we're going to
+//boost a parent node that it should be connected to other paragraphs, at least for the first n paragraphs
+//so we'll want to make sure that the next sibling is a paragraph and has at least some substantial weight to it
+func (extr *ContentExtractor) isBoostable(node *goquery.Selection) bool {
+	stepsAway := 0
+	next := node.Next()
+	for next != nil && stepsAway < node.Siblings().Length() {
+		currentNodeTag := node.Get(0).DataAtom.String()
+		if currentNodeTag == "p" {
+			if stepsAway >= 3 {
+				if extr.config.debug {
+					log.Println("Next paragraph is too far away, not boosting")
+				}
+				return false
+			}
+
+			paraText := node.Text()
+			ws := extr.config.stopWords.stopWordsCount(extr.config.targetLanguage, paraText)
+			if ws.stopWordCount > 5 {
+				if extr.config.debug {
+					log.Println("We're gonna boost this node, seems content")
+				}
+				return true
+			}
+		}
+
+		stepsAway++
+		next = next.Next()
+	}
+
+	return false
+}
+
+//returns a list of nodes we want to search on like paragraphs and tables
+func (extr *ContentExtractor) nodesToCheck(doc *goquery.Document) []*goquery.Selection {
+	var output []*goquery.Selection
+	tags := []string{"p", "pre", "td"}
+	for _, tag := range tags {
+		selections := doc.Children().Find(tag)
+		if selections != nil {
+			selections.Each(func(i int, s *goquery.Selection) {
+				output = append(output, s)
+			})
+		}
+	}
+	return output
+}
+
+//checks the density of links within a node, is there not much text and most of it contains bad links?
+//if so it's no good
+func (extr *ContentExtractor) isHighLinkDensity(node *goquery.Selection) bool {
+	links := node.Find("a")
+	if links == nil || links.Size() == 0 {
+		return false
+	}
+	text := node.Text()
+	words := strings.Split(text, " ")
+	nwords := len(words)
+	var sb []string
+	links.Each(func(i int, s *goquery.Selection) {
+		linkText := s.Text()
+		sb = append(sb, linkText)
+	})
+	linkText := strings.Join(sb, "")
+	linkWords := strings.Split(linkText, " ")
+	nlinkWords := len(linkWords)
+	nlinks := links.Size()
+	linkDivisor := float64(nlinkWords) / float64(nwords)
+	score := linkDivisor * float64(nlinks)
+
+	if extr.config.debug {
+		var logText string
+		if len(node.Text()) >= 51 {
+			logText = node.Text()[0:50]
+		} else {
+			logText = node.Text()
+		}
+		log.Printf("Calculated link density score as %1.5f for node %s\n", score, logText)
+	}
+	if score > 1.0 {
+		return true
+	}
+	return false
+}
+
+func (extr *ContentExtractor) isTableAndNoParaExist(selection *goquery.Selection) bool {
+	subParagraph := selection.Find("p")
+	subParagraph.Each(func(i int, s *goquery.Selection) {
+		txt := s.Text()
+		if len(txt) < 25 {
+			node := s.Get(0)
+			parent := node.Parent
+			parent.RemoveChild(node)
+		}
+	})
+
+	subParagraph2 := selection.Find("p")
+	if subParagraph2.Length() == 0 && selection.Get(0).DataAtom.String() != "td" {
+		return true
+	}
+	return false
+}
+
+func (extr *ContentExtractor) isNodescoreThresholdMet(node *goquery.Selection, e *goquery.Selection) bool {
+	topNodeScore := extr.getNodeGravityScore(node)
+	currentNodeScore := extr.getNodeGravityScore(e)
+	threasholdScore := float64(topNodeScore) * 0.08
+	if (float64(currentNodeScore) < threasholdScore) && e.Get(0).DataAtom.String() != "td" {
+		return false
+	}
+	return true
+}
+
+//we could have long articles that have tons of paragraphs so if we tried to calculate the base score against
+//the total text score of those paragraphs it would be unfair. So we need to normalize the score based on the average scoring
+//of the paragraphs within the top node. For example if our total score of 10 paragraphs was 1000 but each had an average value of
+//100 then 100 should be our base.
+func (extr *ContentExtractor) getSiblingsScore(topNode *goquery.Selection) int {
+	base := 100000
+	paragraphNumber := 0
+	paragraphScore := 0
+	nodesToCheck := topNode.Find("p")
+	nodesToCheck.Each(func(i int, s *goquery.Selection) {
+		textNode := s.Text()
+		ws := extr.config.stopWords.stopWordsCount(extr.config.targetLanguage, textNode)
+		highLinkDensity := extr.isHighLinkDensity(s)
+		if ws.stopWordCount > 2 && !highLinkDensity {
+			paragraphNumber++
+			paragraphScore += ws.stopWordCount
+		}
+	})
+	if paragraphNumber > 0 {
+		base = paragraphScore / paragraphNumber
+	}
+	return base
+}
+
+func (extr *ContentExtractor) getSiblingsContent(currentSibling *goquery.Selection, baselinescoreSiblingsPara float64) []*goquery.Selection {
+	var ps []*goquery.Selection
+	if currentSibling.Get(0).DataAtom.String() == "p" && len(currentSibling.Text()) > 0 {
+		ps = append(ps, currentSibling)
+		return ps
+	}
+
+	potentialParagraphs := currentSibling.Find("p")
+	potentialParagraphs.Each(func(i int, s *goquery.Selection) {
+		text := s.Text()
+		if len(text) > 0 {
+			ws := extr.config.stopWords.stopWordsCount(extr.config.targetLanguage, text)
+			paragraphScore := ws.stopWordCount
+			siblingBaselineScore := 0.30
+			highLinkDensity := extr.isHighLinkDensity(s)
+			score := siblingBaselineScore * baselinescoreSiblingsPara
+			if score < float64(paragraphScore) && !highLinkDensity {
+				node := new(html.Node)
+				node.Type = html.TextNode
+				node.Data = text
+				node.DataAtom = atom.P
+				nodes := make([]*html.Node, 1)
+				nodes[0] = node
+				newSelection := new(goquery.Selection)
+				newSelection.Nodes = nodes
+				ps = append(ps, newSelection)
+			}
+		}
+
+	})
+	return ps
+}
+
+func (extr *ContentExtractor) walkSiblings(node *goquery.Selection) []*goquery.Selection {
+	currentSibling := node.Prev()
+	var b []*goquery.Selection
+	for currentSibling.Length() != 0 {
+		b = append(b, currentSibling)
+		previousSibling := currentSibling.Prev()
+		currentSibling = previousSibling
+	}
+	return b
+}
+
+//adds any siblings that may have a decent score to this node
+func (extr *ContentExtractor) addSiblings(topNode *goquery.Selection) *goquery.Selection {
+	if extr.config.debug {
+		log.Println("Starting to add siblings")
+	}
+	baselinescoreSiblingsPara := extr.getSiblingsScore(topNode)
+	results := extr.walkSiblings(topNode)
+	for _, currentNode := range results {
+		ps := extr.getSiblingsContent(currentNode, float64(baselinescoreSiblingsPara))
+		for _, p := range ps {
+			nodes := make([]*html.Node, len(topNode.Nodes)+1)
+			nodes[0] = p.Get(0)
+			for i, node := range topNode.Nodes {
+				nodes[i+1] = node
+			}
+			topNode.Nodes = nodes
+		}
+	}
+	return topNode
+}
+
+//PostCleanup removes any divs that looks like non-content, clusters of links, or paras with no gusto
+func (extr *ContentExtractor) PostCleanup(targetNode *goquery.Selection) *goquery.Selection {
+	if extr.config.debug {
+		log.Println("Starting cleanup Node")
+	}
+	node := extr.addSiblings(targetNode)
+	children := node.Children()
+	children.Each(func(i int, s *goquery.Selection) {
+		tag := s.Get(0).DataAtom.String()
+		if tag != "p" {
+			if extr.config.debug {
+				log.Printf("CLEANUP  NODE: %s class: %s\n", extr.config.parser.name("id", s), extr.config.parser.name("class", s))
+			}
+			//if extr.isHighLinkDensity(s) || extr.isTableAndNoParaExist(s) || !extr.isNodescoreThresholdMet(node, s) {
+			if extr.isHighLinkDensity(s) {
+				extr.config.parser.removeNode(s)
+				return
+			}
+
+			subParagraph := s.Find("p")
+			subParagraph.Each(func(j int, e *goquery.Selection) {
+				if len(e.Text()) < 25 {
+					extr.config.parser.removeNode(e)
+				}
+			})
+
+			subParagraph2 := s.Find("p")
+			if subParagraph2.Length() == 0 && tag != "td" {
+				if extr.config.debug {
+					log.Println("Removing node because it doesn't have any paragraphs")
+				}
+				extr.config.parser.removeNode(s)
+			} else {
+				if extr.config.debug {
+					log.Println("Not removing TD node")
+				}
+			}
+			return
+		}
+	})
+	return node
+}

vendor/github.com/advancedlogic/GoOse/go.mod

@@ -0,0 +1,22 @@
+module github.com/advancedlogic/GoOse
+
+go 1.13
+
+require (
+	github.com/PuerkitoBio/goquery v1.4.1
+	github.com/andybalholm/cascadia v1.0.0 // indirect
+	github.com/araddon/dateparse v0.0.0-20180729174819-cfd92a431d0e
+	github.com/fatih/set v0.2.1
+	github.com/gigawattio/window v0.0.0-20180317192513-0f5467e35573
+	github.com/go-resty/resty/v2 v2.0.0
+	github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0
+	github.com/mattn/go-runewidth v0.0.3 // indirect
+	github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84 // indirect
+	github.com/pkg/errors v0.8.1
+	github.com/simplereach/timeutils v1.2.0 // indirect
+	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
+	github.com/stretchr/testify v1.4.0 // indirect
+	golang.org/x/net v0.0.0-20190628185345-da137c7871d7
+	golang.org/x/text v0.3.0
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
+)

vendor/github.com/advancedlogic/GoOse/go.sum

@@ -0,0 +1,44 @@
+github.com/PuerkitoBio/goquery v1.4.1 h1:smcIRGdYm/w7JSbcdeLHEMzxmsBQvl8lhf0dSw2nzMI=
+github.com/PuerkitoBio/goquery v1.4.1/go.mod h1:T9ezsOHcCrDCgA8aF1Cqr3sSYbO/xgdy8/R/XiIMAhA=
+github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
+github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/araddon/dateparse v0.0.0-20180729174819-cfd92a431d0e h1:s05JG2GwtJMHaPcXDpo4V35TFgyYZzNsmBlSkHPEbeg=
+github.com/araddon/dateparse v0.0.0-20180729174819-cfd92a431d0e/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fatih/set v0.2.1 h1:nn2CaJyknWE/6txyUDGwysr3G5QC6xWB/PtVjPBbeaA=
+github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI=
+github.com/gigawattio/window v0.0.0-20180317192513-0f5467e35573 h1:u8AQ9bPa9oC+8/A/jlWouakhIvkFfuxgIIRjiy8av7I=
+github.com/gigawattio/window v0.0.0-20180317192513-0f5467e35573/go.mod h1:eBvb3i++NHDH4Ugo9qCvMw8t0mTSctaEa5blJbWcNxs=
+github.com/go-resty/resty/v2 v2.0.0 h1:9Nq/U+V4xsoDnDa/iTrABDWUCuk3Ne92XFHPe6dKWUc=
+github.com/go-resty/resty/v2 v2.0.0/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8=
+github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 h1:xqgexXAGQgY3HAjNPSaCqn5Aahbo5TKsmhp8VRfr1iQ=
+github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
+github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4=
+github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84 h1:fiKJgB4JDUd43CApkmCeTSQlWjtTtABrU2qsgbuP0BI=
+github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/simplereach/timeutils v1.2.0 h1:btgOAlu9RW6de2r2qQiONhjgxdAG7BL6je0G6J/yPnA=
+github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8=
+github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo=
+github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

vendor/github.com/advancedlogic/GoOse/goose.go

@@ -0,0 +1,34 @@
+package goose
+
+import (
+	"github.com/pkg/errors"
+)
+
+// Goose is the main entry point of the program
+type Goose struct {
+	config Configuration
+}
+
+// New returns a new instance of the article extractor
+func New(args ...string) Goose {
+	return Goose{
+		config: GetDefaultConfiguration(args...),
+	}
+}
+
+// ExtractFromURL follows the URL, fetches the HTML page and returns an article object
+func (g Goose) ExtractFromURL(url string) (*Article, error) {
+	HtmlRequester := NewHtmlRequester(g.config)
+	html, err := HtmlRequester.fetchHTML(url)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get htnk from site")
+	}
+	cc := NewCrawler(g.config)
+	return cc.Crawl(html, url)
+}
+
+// ExtractFromRawHTML returns an article object from the raw HTML content
+func (g Goose) ExtractFromRawHTML(RawHTML string, url string) (*Article, error) {
+	cc := NewCrawler(g.config)
+	return cc.Crawl(RawHTML, url)
+}

vendor/github.com/advancedlogic/GoOse/goose.json

@@ -0,0 +1,37 @@
+/*
+This is a golang port of "Goose" originaly licensed to Gravity.com
+under one or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.
+
+Golang port was written by Antonio Linari
+
+Gravity.com licenses this file
+to you under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+//JSON Configuration file
+{
+  "localStoragePath": "",
+  "imagesMinBytes": 4500,
+  "enableImageFetching": true,
+  "useMetaLanguage": true,
+  "targetLanguage": "en",
+  "imageMagickConvertPath": "/usr/bin/convert",
+  "imageMagickIdentifyPath": "/usr/bin/identify",
+  "browserUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/534.52.7 (KHTML, like Gecko) Version/5.1.2 Safari/534.52.7",
+  "debug": false,
+  "extractPublishDate": false,
+  "additionalDataExtractor": false,
+  "stopWordsPath": "resources/stopwords"
+}

vendor/github.com/advancedlogic/GoOse/html.go

@@ -0,0 +1,47 @@
+package goose
+
+import (
+	resty "github.com/go-resty/resty/v2"
+	"github.com/pkg/errors"
+)
+
+type HtmlRequester interface {
+	fetchHTML(string) (string, error)
+}
+
+// Crawler can fetch the target HTML page
+type htmlrequester struct {
+	config Configuration
+}
+
+// NewCrawler returns a crawler object initialised with the URL and the [optional] raw HTML body
+func NewHtmlRequester(config Configuration) HtmlRequester {
+	return htmlrequester{
+		config: config,
+	}
+}
+
+func (hr htmlrequester) fetchHTML(url string) (string, error) {
+	client := resty.New()
+	client.SetTimeout(hr.config.timeout)
+	resp, err := client.R().
+		SetHeader("Content-Type", "text/html").
+		SetHeader("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.91 Safari/534.30").
+		Get(url)
+
+	if err != nil {
+		return "", errors.Wrap(err, "could not perform request on "+url)
+	}
+	if resp.IsError() {
+		return "", &badRequest{Message: "could not perform request with " + url + " status code " + string(resp.StatusCode())}
+	}
+	return resp.String(), nil
+}
+
+type badRequest struct {
+	Message string `json:"message,omitempty"`
+}
+
+func (BadRequest *badRequest) Error() string {
+	return "Required request fields are not filled"
+}

vendor/github.com/advancedlogic/GoOse/images.go

@@ -0,0 +1,328 @@
+package goose
+
+import (
+	"github.com/PuerkitoBio/goquery"
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+type candidate struct {
+	url     string
+	surface int
+	score   int
+}
+
+func (c *candidate) GetUrl() string {
+	return c.url
+}
+
+var largebig = regexp.MustCompile("(large|big|full)")
+
+var classRules = map[*regexp.Regexp]int{
+	regexp.MustCompile("(promo|ads|banner)"): -1}
+
+var rules = map[*regexp.Regexp]int{
+	largebig:                                   1,
+	regexp.MustCompile("upload"):               1,
+	regexp.MustCompile("media"):                1,
+	regexp.MustCompile("gravatar.com"):         -1,
+	regexp.MustCompile("feeds.feedburner.com"): -1,
+	regexp.MustCompile("(?i)icon"):             -1,
+	regexp.MustCompile("(?i)logo"):             -1,
+	regexp.MustCompile("(?i)spinner"):          -1,
+	regexp.MustCompile("(?i)loading"):          -1,
+	regexp.MustCompile("(?i)ads"):              -1,
+	regexp.MustCompile("badge"):                -1,
+	regexp.MustCompile("1x1"):                  -1,
+	regexp.MustCompile("pixel"):                -1,
+	regexp.MustCompile("thumbnail[s]*"):        -1,
+	regexp.MustCompile(".html|" +
+		".gif|" +
+		".ico|" +
+		"button|" +
+		"twitter.jpg|" +
+		"facebook.jpg|" +
+		"ap_buy_photo|" +
+		"digg.jpg|" +
+		"digg.png|" +
+		"delicious.png|" +
+		"facebook.png|" +
+		"reddit.jpg|" +
+		"doubleclick|" +
+		"diggthis|" +
+		"diggThis|" +
+		"adserver|" +
+		"/(ads|promos|banners)/|" +
+		"ec.atdmt.com|" +
+		"mediaplex.com|" +
+		"adsatt|" +
+		"view.atdmt"): -1}
+
+func getImageSrc(tag *goquery.Selection) string {
+	src, _ := tag.Attr("src")
+	// skip inline images
+	if strings.Contains(src, "data:image/") {
+		src = ""
+	}
+	if src == "" {
+		src, _ = tag.Attr("data-src")
+	}
+	if src == "" {
+		src, _ = tag.Attr("data-lazy-src")
+	}
+	return src
+}
+
+func score(tag *goquery.Selection) int {
+	src := getImageSrc(tag)
+	if src == "" {
+		return -1
+	}
+	tagScore := 0
+	for rule, score := range rules {
+		if rule.MatchString(src) {
+			tagScore += score
+		}
+	}
+
+	alt, exists := tag.Attr("alt")
+	if exists {
+		if strings.Contains(alt, "thumbnail") {
+			tagScore--
+		}
+	}
+
+	id, exists := tag.Attr("id")
+	if exists {
+		if id == "fbPhotoImage" {
+			tagScore++
+		}
+	}
+
+	class, exists := tag.Attr("class")
+	if exists {
+		for rule, score := range classRules {
+			if rule.MatchString(class) {
+				tagScore += score
+			}
+		}
+	}
+
+	return tagScore
+}
+
+// WebPageImageResolver fetches all candidate images from the HTML page
+func WebPageImageResolver(doc *goquery.Document) ([]candidate, int) {
+	imgs := doc.Find("img")
+
+	var candidates []candidate
+	significantSurface := 320 * 200
+	significantSurfaceCount := 0
+	src := ""
+	imgs.Each(func(i int, tag *goquery.Selection) {
+		var surface int
+		src = getImageSrc(tag)
+		if src == "" {
+			return
+		}
+
+		width, _ := tag.Attr("width")
+		height, _ := tag.Attr("height")
+		if width != "" {
+			w, _ := strconv.Atoi(width)
+			if height != "" {
+				h, _ := strconv.Atoi(height)
+				surface = w * h
+			} else {
+				surface = w
+			}
+		} else {
+			if height != "" {
+				surface, _ = strconv.Atoi(height)
+			} else {
+				surface = 0
+			}
+		}
+
+		if surface > significantSurface {
+			significantSurfaceCount++
+		}
+
+		tagscore := score(tag)
+		if tagscore >= 0 {
+			c := candidate{
+				url:     src,
+				surface: surface,
+				score:   score(tag),
+			}
+			candidates = append(candidates, c)
+		}
+	})
+
+	if len(candidates) == 0 {
+		return nil, 0
+	}
+
+	return candidates, significantSurfaceCount
+
+}
+
+// WebPageResolver fetches the main image from the HTML page
+func WebPageResolver(article *Article) string {
+	candidates, significantSurfaceCount := WebPageImageResolver(article.Doc)
+	if candidates == nil {
+		return ""
+	}
+	var bestCandidate candidate
+	var topImage string
+	if significantSurfaceCount > 0 {
+		bestCandidate = findBestCandidateFromSurface(candidates)
+	} else {
+		bestCandidate = findBestCandidateFromScore(candidates)
+	}
+
+	topImage = bestCandidate.url
+	a, err := url.Parse(topImage)
+	if err != nil {
+		return topImage
+	}
+	finalURL, err := url.Parse(article.FinalURL)
+	if err != nil {
+		return topImage
+	}
+	b := finalURL.ResolveReference(a)
+	topImage = b.String()
+
+	return topImage
+}
+
+func findBestCandidateFromSurface(candidates []candidate) candidate {
+	max := 0
+	var bestCandidate candidate
+	for _, candidate := range candidates {
+		surface := candidate.surface
+		if surface >= max {
+			max = surface
+			bestCandidate = candidate
+		}
+	}
+
+	return bestCandidate
+}
+
+func findBestCandidateFromScore(candidates []candidate) candidate {
+	max := 0
+	var bestCandidate candidate
+	for _, candidate := range candidates {
+		score := candidate.score
+		if score >= max {
+			max = score
+			bestCandidate = candidate
+		}
+	}
+
+	return bestCandidate
+}
+
+type ogTag struct {
+	tpe       string
+	attribute string
+	name      string
+	value     string
+}
+
+var ogTags = [4]ogTag{
+	{
+		tpe:       "facebook",
+		attribute: "property",
+		name:      "og:image",
+		value:     "content",
+	},
+	{
+		tpe:       "facebook",
+		attribute: "rel",
+		name:      "image_src",
+		value:     "href",
+	},
+	{
+		tpe:       "twitter",
+		attribute: "name",
+		name:      "twitter:image",
+		value:     "value",
+	},
+	{
+		tpe:       "twitter",
+		attribute: "name",
+		name:      "twitter:image",
+		value:     "content",
+	},
+}
+
+type ogImage struct {
+	url   string
+	tpe   string
+	score int
+}
+
+// OpenGraphResolver return OpenGraph properties
+func OpenGraphResolver(doc *goquery.Document) string {
+	meta := doc.Find("meta")
+	links := doc.Find("link")
+	var topImage string
+	meta = meta.Union(links)
+	var ogImages []ogImage
+	meta.Each(func(i int, tag *goquery.Selection) {
+		for _, ogTag := range ogTags {
+			attr, exist := tag.Attr(ogTag.attribute)
+			value, vexist := tag.Attr(ogTag.value)
+			if exist && attr == ogTag.name && vexist {
+				ogImage := ogImage{
+					url:   value,
+					tpe:   ogTag.tpe,
+					score: 0,
+				}
+
+				ogImages = append(ogImages, ogImage)
+			}
+		}
+	})
+	if len(ogImages) == 0 {
+		return ""
+	}
+	if len(ogImages) == 1 {
+		topImage = ogImages[0].url
+		goto IMAGE_FINALIZE
+	}
+	for _, ogImage := range ogImages {
+		if largebig.MatchString(ogImage.url) {
+			ogImage.score++
+		}
+		if ogImage.tpe == "twitter" {
+			ogImage.score++
+		}
+	}
+	topImage = findBestImageFromScore(ogImages).url
+IMAGE_FINALIZE:
+	if !strings.HasPrefix(topImage, "http") {
+		topImage = "http://" + topImage
+	}
+
+	return topImage
+}
+
+// assume that len(ogImages)>=2
+func findBestImageFromScore(ogImages []ogImage) ogImage {
+	max := 0
+	bestOGImage := ogImages[0]
+	for _, ogImage := range ogImages[1:] {
+		score := ogImage.score
+		if score > max {
+			max = score
+			bestOGImage = ogImage
+		}
+	}
+
+	return bestOGImage
+}

vendor/github.com/advancedlogic/GoOse/outputformatter.go

@@ -0,0 +1,180 @@
+package goose
+
+import (
+	"bytes"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"golang.org/x/net/html"
+)
+
+var normalizeWhitespaceRegexp = regexp.MustCompile(`[ \r\f\v\t]+`)
+var normalizeNl = regexp.MustCompile(`[\n]+`)
+var validURLRegex = regexp.MustCompile("^http[s]?://")
+
+type outputFormatter struct {
+	topNode  *goquery.Selection
+	config   Configuration
+	language string
+}
+
+func (formatter *outputFormatter) getLanguage(lang string) string {
+	if formatter.config.useMetaLanguage && "" != lang {
+		return lang
+	}
+	return formatter.config.targetLanguage
+}
+
+func (formatter *outputFormatter) getTopNode() *goquery.Selection {
+	return formatter.topNode
+}
+
+func (formatter *outputFormatter) getFormattedText(topNode *goquery.Selection, lang string) (output string, links []string) {
+	formatter.topNode = topNode
+	formatter.language = formatter.getLanguage(lang)
+	if formatter.language == "" {
+		formatter.language = formatter.config.targetLanguage
+	}
+	formatter.removeNegativescoresNodes()
+	links = formatter.linksToText()
+	formatter.replaceTagsWithText()
+	formatter.removeParagraphsWithFewWords()
+
+	output = formatter.getOutputText()
+	return output, links
+}
+
+func (formatter *outputFormatter) convertToText() string {
+	var txts []string
+	selections := formatter.topNode
+	selections.Each(func(i int, s *goquery.Selection) {
+		txt := s.Text()
+		if txt != "" {
+			// txt = txt //unescape
+			txtLis := strings.Trim(txt, "\n")
+			txts = append(txts, txtLis)
+		}
+	})
+	return strings.Join(txts, "\n\n")
+}
+
+// check if this is a valid URL
+func isValidURL(u string) bool {
+	return validURLRegex.MatchString(u)
+}
+
+func (formatter *outputFormatter) linksToText() []string {
+	var urlList []string
+	links := formatter.topNode.Find("a")
+	links.Each(func(i int, a *goquery.Selection) {
+		imgs := a.Find("img")
+		// ignore linked images
+		if imgs.Length() == 0 {
+			// save a list of URLs
+			url, _ := a.Attr("href")
+			if isValidURL(url) {
+				urlList = append(urlList, url)
+			}
+			// replace <a> tag with its text contents
+			replaceTagWithContents(a, whitelistedExtAtomTypes)
+
+			// see whether we can collapse the parent node now
+			replaceTagWithContents(a.Parent(), whitelistedTextAtomTypes)
+		}
+	})
+
+	return urlList
+}
+
+// Text gets the combined text contents of each element in the set of matched
+// elements, including their descendants.
+//
+// @see https://github.com/PuerkitoBio/goquery/blob/master/property.go
+func (formatter *outputFormatter) Text(s *goquery.Selection) string {
+	var buf bytes.Buffer
+
+	// Slightly optimized vs calling Each: no single selection object created
+	var f func(*html.Node)
+	f = func(n *html.Node) {
+		if n.Type == html.TextNode && 0 == n.DataAtom { // NB: had to add the DataAtom check to avoid printing text twice when a textual node embeds another textual node
+			// Keep newlines and spaces, like jQuery
+			buf.WriteString(n.Data)
+		}
+		if n.FirstChild != nil {
+			for c := n.FirstChild; c != nil; c = c.NextSibling {
+				f(c)
+			}
+		}
+	}
+	for _, n := range s.Nodes {
+		f(n)
+	}
+
+	return buf.String()
+}
+
+func (formatter *outputFormatter) getOutputText() string {
+	//out := formatter.topNode.Text()
+	out := formatter.Text(formatter.topNode)
+	out = normalizeWhitespaceRegexp.ReplaceAllString(out, " ")
+
+	strArr := strings.Split(out, "\n")
+	resArr := []string{}
+
+	for i, v := range strArr {
+		v = strings.TrimSpace(v)
+		if v != "" {
+			resArr = append(resArr, v)
+		} else if i > 2 && strArr[i-2] != "" {
+			resArr = append(resArr, "")
+		}
+	}
+
+	out = strings.Join(resArr, "\n")
+	out = normalizeNl.ReplaceAllString(out, "\n\n")
+
+	out = strings.TrimSpace(out)
+	return out
+}
+
+func (formatter *outputFormatter) removeNegativescoresNodes() {
+	gravityItems := formatter.topNode.Find("*[gravityScore]")
+	gravityItems.Each(func(i int, s *goquery.Selection) {
+		var score int
+		sscore, exists := s.Attr("gravityScore")
+		if exists {
+			score, _ = strconv.Atoi(sscore)
+			if score < 1 {
+				sNode := s.Get(0)
+				sNode.Parent.RemoveChild(sNode)
+			}
+		}
+
+	})
+}
+
+func (formatter *outputFormatter) replaceTagsWithText() {
+	for _, tag := range []string{"em", "strong", "b", "i", "span", "h1", "h2", "h3", "h4"} {
+		nodes := formatter.topNode.Find(tag)
+		nodes.Each(func(i int, node *goquery.Selection) {
+			replaceTagWithContents(node, whitelistedTextAtomTypes)
+		})
+	}
+}
+
+func (formatter *outputFormatter) removeParagraphsWithFewWords() {
+	language := formatter.language
+	if language == "" {
+		language = "en"
+	}
+	allNodes := formatter.topNode.Children()
+	allNodes.Each(func(i int, s *goquery.Selection) {
+		sw := formatter.config.stopWords.stopWordsCount(language, s.Text())
+		if sw.wordCount < 5 && s.Find("object").Length() == 0 && s.Find("em").Length() == 0 {
+			node := s.Get(0)
+			node.Parent.RemoveChild(node)
+		}
+	})
+}

vendor/github.com/advancedlogic/GoOse/parser.go

@@ -0,0 +1,90 @@
+package goose
+
+import (
+	"github.com/PuerkitoBio/goquery"
+	"golang.org/x/net/html"
+)
+
+// Parser is an HTML parser specialised in extraction of main content and other properties
+type Parser struct{}
+
+// NewParser returns an HTML parser
+func NewParser() *Parser {
+	return &Parser{}
+}
+
+func (p Parser) dropTag(selection *goquery.Selection) {
+	selection.Each(func(i int, s *goquery.Selection) {
+		replaceTagWithContents(s, whitelistedTextAtomTypes)
+	})
+}
+
+func (p Parser) indexOfAttribute(selection *goquery.Selection, attr string) int {
+	node := selection.Get(0)
+	for i, a := range node.Attr {
+		if a.Key == attr {
+			return i
+		}
+	}
+	return -1
+}
+
+func (p Parser) delAttr(selection *goquery.Selection, attr string) {
+	idx := p.indexOfAttribute(selection, attr)
+	if idx > -1 {
+		node := selection.Get(0)
+		node.Attr = append(node.Attr[:idx], node.Attr[idx+1:]...)
+	}
+}
+
+func (p Parser) getElementsByTags(div *goquery.Selection, tags []string) *goquery.Selection {
+	selection := new(goquery.Selection)
+	for _, tag := range tags {
+		selections := div.Find(tag)
+		if selections != nil {
+			selection = selection.Union(selections)
+		}
+	}
+	return selection
+}
+
+func (p Parser) clear(selection *goquery.Selection) {
+	selection.Nodes = make([]*html.Node, 0)
+}
+
+func (p Parser) removeNode(selection *goquery.Selection) {
+	if selection != nil {
+		node := selection.Get(0)
+		if node != nil && node.Parent != nil {
+			node.Parent.RemoveChild(node)
+		}
+	}
+}
+
+func (p Parser) name(selector string, selection *goquery.Selection) string {
+	value, exists := selection.Attr(selector)
+	if exists {
+		return value
+	}
+	return ""
+}
+
+func (p Parser) setAttr(selection *goquery.Selection, attr string, value string) {
+	if selection.Size() > 0 {
+		node := selection.Get(0)
+		var attrs []html.Attribute
+		for _, a := range node.Attr {
+			if a.Key != attr {
+				newAttr := new(html.Attribute)
+				newAttr.Key = a.Key
+				newAttr.Val = a.Val
+				attrs = append(attrs, *newAttr)
+			}
+		}
+		newAttr := new(html.Attribute)
+		newAttr.Key = attr
+		newAttr.Val = value
+		attrs = append(attrs, *newAttr)
+		node.Attr = attrs
+	}
+}

vendor/github.com/advancedlogic/GoOse/videos.go

@@ -0,0 +1,165 @@
+package goose
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/fatih/set"
+)
+
+// VideoExtractor can extract the main video from an HTML page
+type VideoExtractor struct {
+	article    *Article
+	config     Configuration
+	candidates *set.Set
+	movies     *set.Set
+}
+
+type video struct {
+	embedType string
+	provider  string
+	width     int
+	height    int
+	embedCode string
+	src       string
+}
+
+// NewVideoExtractor returns a new instance of a HTML video extractor
+func NewVideoExtractor() VideoExtractor {
+	return VideoExtractor{
+		candidates: set.New(set.ThreadSafe).(*set.Set),
+		movies:     set.New(set.ThreadSafe).(*set.Set),
+	}
+}
+
+var videoTags = [4]string{"iframe", "embed", "object", "video"}
+var videoProviders = [4]string{"youtube", "vimeo", "dailymotion", "kewego"}
+
+func (ve *VideoExtractor) getEmbedCode(node *goquery.Selection) string {
+	return node.Text()
+}
+
+func (ve *VideoExtractor) getWidth(node *goquery.Selection) int {
+	value, exists := node.Attr("width")
+	if exists {
+		nvalue, _ := strconv.Atoi(value)
+		return nvalue
+	}
+	return 0
+}
+
+func (ve *VideoExtractor) getHeight(node *goquery.Selection) int {
+	value, exists := node.Attr("height")
+	if exists {
+		nvalue, _ := strconv.Atoi(value)
+		return nvalue
+	}
+	return 0
+}
+
+func (ve *VideoExtractor) getSrc(node *goquery.Selection) string {
+	value, exists := node.Attr("src")
+	if exists {
+		return value
+	}
+	return ""
+}
+
+func (ve *VideoExtractor) getProvider(src string) string {
+	if src != "" {
+		for _, provider := range videoProviders {
+			if strings.Contains(src, provider) {
+				return provider
+			}
+		}
+	}
+	return ""
+}
+
+func (ve *VideoExtractor) getVideo(node *goquery.Selection) video {
+	src := ve.getSrc(node)
+	video := video{
+		embedCode: ve.getEmbedCode(node),
+		embedType: node.Get(0).DataAtom.String(),
+		width:     ve.getWidth(node),
+		height:    ve.getHeight(node),
+		src:       src,
+		provider:  ve.getProvider(src),
+	}
+	return video
+}
+
+func (ve *VideoExtractor) getIFrame(node *goquery.Selection) video {
+	return ve.getVideo(node)
+}
+
+func (ve *VideoExtractor) getVideoTag(node *goquery.Selection) video {
+	return video{}
+}
+
+func (ve *VideoExtractor) getEmbedTag(node *goquery.Selection) video {
+	parent := node.Parent()
+	if parent != nil {
+		parentTag := parent.Get(0).DataAtom.String()
+		if parentTag == "object" {
+			return ve.getObjectTag(node)
+		}
+	}
+	return ve.getVideo(node)
+}
+
+func (ve *VideoExtractor) getObjectTag(node *goquery.Selection) video {
+	childEmbedTag := node.Find("embed")
+	if ve.candidates.Has(childEmbedTag) {
+		ve.candidates.Remove(childEmbedTag)
+	}
+	srcNode := node.Find(`param[name="movie"]`)
+	if srcNode == nil || srcNode.Length() == 0 {
+		return video{}
+	}
+
+	src, _ := srcNode.Attr("value")
+	provider := ve.getProvider(src)
+	if provider == "" {
+		return video{}
+	}
+	video := ve.getVideo(node)
+	video.provider = provider
+	video.src = src
+	return video
+}
+
+// GetVideos returns the video tags embedded in the article
+func (ve *VideoExtractor) GetVideos(doc *goquery.Document) *set.Set {
+	var nodes *goquery.Selection
+	for _, videoTag := range videoTags {
+		tmpNodes := doc.Find(videoTag)
+		if nodes == nil {
+			nodes = tmpNodes
+		} else {
+			nodes.Union(tmpNodes)
+		}
+	}
+
+	nodes.Each(func(i int, node *goquery.Selection) {
+		tag := node.Get(0).DataAtom.String()
+		var movie video
+		switch tag {
+		case "video":
+			movie = ve.getVideoTag(node)
+		case "embed":
+			movie = ve.getEmbedTag(node)
+		case "object":
+			movie = ve.getObjectTag(node)
+		case "iframe":
+			movie = ve.getIFrame(node)
+		}
+
+		if movie.src != "" {
+			ve.movies.Add(movie)
+		}
+	})
+
+	return ve.movies
+}

vendor/github.com/advancedlogic/GoOse/wordstats.go

@@ -0,0 +1,39 @@
+package goose
+
+import (
+	"github.com/fatih/set"
+)
+
+//some word statistics
+type wordStats struct {
+	//total number of stopwords or good words that we can calculate
+	stopWordCount int
+	//total number of words on a node
+	wordCount int
+	//holds an actual list of the stop words we found
+	stopWords *set.Set
+}
+
+func (w *wordStats) getStopWords() *set.Set {
+	return w.stopWords
+}
+
+func (w *wordStats) setStopWords(stopWords *set.Set) {
+	w.stopWords = stopWords
+}
+
+func (w *wordStats) getStopWordCount() int {
+	return w.stopWordCount
+}
+
+func (w *wordStats) setStopWordCount(stopWordCount int) {
+	w.stopWordCount = stopWordCount
+}
+
+func (w *wordStats) getWordCount() int {
+	return w.wordCount
+}
+
+func (w *wordStats) setWordCount(wordCount int) {
+	w.wordCount = wordCount
+}

vendor/github.com/andybalholm/brotli/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

vendor/github.com/andybalholm/brotli/README.md

@@ -0,0 +1,5 @@
+This package is a brotli compressor and decompressor implemented in Go.
+It was translated from the reference implementation (https://github.com/google/brotli)
+with the `c2go` tool at https://github.com/andybalholm/c2go.
+
+I am using it in production with https://github.com/andybalholm/redwood.

vendor/github.com/andybalholm/brotli/backward_references.go

@@ -0,0 +1,177 @@
+package brotli
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Function to find backward reference copies. */
+
+func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uint {
+	if distance <= max_distance {
+		var distance_plus_3 uint = distance + 3
+		var offset0 uint = distance_plus_3 - uint(dist_cache[0])
+		var offset1 uint = distance_plus_3 - uint(dist_cache[1])
+		if distance == uint(dist_cache[0]) {
+			return 0
+		} else if distance == uint(dist_cache[1]) {
+			return 1
+		} else if offset0 < 7 {
+			return (0x9750468 >> (4 * offset0)) & 0xF
+		} else if offset1 < 7 {
+			return (0xFDB1ACE >> (4 * offset1)) & 0xF
+		} else if distance == uint(dist_cache[2]) {
+			return 2
+		} else if distance == uint(dist_cache[3]) {
+			return 3
+		}
+	}
+
+	return distance + numDistanceShortCodes - 1
+}
+
+/* "commands" points to the next output command to write to, "*num_commands" is
+   initially the total amount of commands output by previous
+   CreateBackwardReferences calls, and must be incremented by the amount written
+   by this call. */
+func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
+	var max_backward_limit uint = maxBackwardLimit(params.lgwin)
+	var orig_commands []command = commands
+	var insert_length uint = *last_insert_len
+	var pos_end uint = position + num_bytes
+	var store_end uint
+	if num_bytes >= hasher.StoreLookahead() {
+		store_end = position + num_bytes - hasher.StoreLookahead() + 1
+	} else {
+		store_end = position
+	}
+	var random_heuristics_window_size uint = literalSpreeLengthForSparseSearch(params)
+	var apply_random_heuristics uint = position + random_heuristics_window_size
+	var gap uint = 0
+	/* Set maximum distance, see section 9.1. of the spec. */
+
+	const kMinScore uint = scoreBase + 100
+
+	/* For speed up heuristics for random data. */
+
+	/* Minimum score to accept a backward reference. */
+	hasher.PrepareDistanceCache(dist_cache)
+	var sr2 hasherSearchResult
+	var sr hasherSearchResult
+
+	for position+hasher.HashTypeLength() < pos_end {
+		var max_length uint = pos_end - position
+		var max_distance uint = brotli_min_size_t(position, max_backward_limit)
+		sr.len = 0
+		sr.len_code_delta = 0
+		sr.distance = 0
+		sr.score = kMinScore
+		hasher.FindLongestMatch(&params.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position, max_length, max_distance, gap, params.dist.max_distance, &sr)
+		if sr.score > kMinScore {
+			/* Found a match. Let's look for something even better ahead. */
+			var delayed_backward_references_in_row int = 0
+			max_length--
+			for ; ; max_length-- {
+				var cost_diff_lazy uint = 175
+				if params.quality < minQualityForExtensiveReferenceSearch {
+					sr2.len = brotli_min_size_t(sr.len-1, max_length)
+				} else {
+					sr2.len = 0
+				}
+				sr2.len_code_delta = 0
+				sr2.distance = 0
+				sr2.score = kMinScore
+				max_distance = brotli_min_size_t(position+1, max_backward_limit)
+				hasher.FindLongestMatch(&params.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position+1, max_length, max_distance, gap, params.dist.max_distance, &sr2)
+				if sr2.score >= sr.score+cost_diff_lazy {
+					/* Ok, let's just write one byte for now and start a match from the
+					   next byte. */
+					position++
+
+					insert_length++
+					sr = sr2
+					delayed_backward_references_in_row++
+					if delayed_backward_references_in_row < 4 && position+hasher.HashTypeLength() < pos_end {
+						continue
+					}
+				}
+
+				break
+			}
+
+			apply_random_heuristics = position + 2*sr.len + random_heuristics_window_size
+			max_distance = brotli_min_size_t(position, max_backward_limit)
+			{
+				/* The first 16 codes are special short-codes,
+				   and the minimum offset is 1. */
+				var distance_code uint = computeDistanceCode(sr.distance, max_distance+gap, dist_cache)
+				if (sr.distance <= (max_distance + gap)) && distance_code > 0 {
+					dist_cache[3] = dist_cache[2]
+					dist_cache[2] = dist_cache[1]
+					dist_cache[1] = dist_cache[0]
+					dist_cache[0] = int(sr.distance)
+					hasher.PrepareDistanceCache(dist_cache)
+				}
+
+				initCommand(&commands[0], &params.dist, insert_length, sr.len, sr.len_code_delta, distance_code)
+				commands = commands[1:]
+			}
+
+			*num_literals += insert_length
+			insert_length = 0
+			/* Put the hash keys into the table, if there are enough bytes left.
+			   Depending on the hasher implementation, it can push all positions
+			   in the given range or only a subset of them.
+			   Avoid hash poisoning with RLE data. */
+			{
+				var range_start uint = position + 2
+				var range_end uint = brotli_min_size_t(position+sr.len, store_end)
+				if sr.distance < sr.len>>2 {
+					range_start = brotli_min_size_t(range_end, brotli_max_size_t(range_start, position+sr.len-(sr.distance<<2)))
+				}
+
+				hasher.StoreRange(ringbuffer, ringbuffer_mask, range_start, range_end)
+			}
+
+			position += sr.len
+		} else {
+			insert_length++
+			position++
+
+			/* If we have not seen matches for a long time, we can skip some
+			   match lookups. Unsuccessful match lookups are very very expensive
+			   and this kind of a heuristic speeds up compression quite
+			   a lot. */
+			if position > apply_random_heuristics {
+				/* Going through uncompressible data, jump. */
+				if position > apply_random_heuristics+4*random_heuristics_window_size {
+					var kMargin uint = brotli_max_size_t(hasher.StoreLookahead()-1, 4)
+					/* It is quite a long time since we saw a copy, so we assume
+					   that this data is not compressible, and store hashes less
+					   often. Hashes of non compressible data are less likely to
+					   turn out to be useful in the future, too, so we store less of
+					   them to not to flood out the hash table of good compressible
+					   data. */
+
+					var pos_jump uint = brotli_min_size_t(position+16, pos_end-kMargin)
+					for ; position < pos_jump; position += 4 {
+						hasher.Store(ringbuffer, ringbuffer_mask, position)
+						insert_length += 4
+					}
+				} else {
+					var kMargin uint = brotli_max_size_t(hasher.StoreLookahead()-1, 2)
+					var pos_jump uint = brotli_min_size_t(position+8, pos_end-kMargin)
+					for ; position < pos_jump; position += 2 {
+						hasher.Store(ringbuffer, ringbuffer_mask, position)
+						insert_length += 2
+					}
+				}
+			}
+		}
+	}
+
+	insert_length += pos_end - position
+	*last_insert_len = insert_length
+	*num_commands += uint(-cap(commands) + cap(orig_commands))
+}

vendor/github.com/andybalholm/brotli/backward_references_hq.go

@@ -0,0 +1,795 @@
+package brotli
+
+import "math"
+
+type zopfliNode struct {
+	length              uint32
+	distance            uint32
+	dcode_insert_length uint32
+	u                   struct {
+		cost     float32
+		next     uint32
+		shortcut uint32
+	}
+}
+
+const maxEffectiveDistanceAlphabetSize = 544
+
+const kInfinity float32 = 1.7e38 /* ~= 2 ^ 127 */
+
+var kDistanceCacheIndex = []uint32{0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1}
+
+var kDistanceCacheOffset = []int{0, 0, 0, 0, -1, 1, -2, 2, -3, 3, -1, 1, -2, 2, -3, 3}
+
+func initZopfliNodes(array []zopfliNode, length uint) {
+	var stub zopfliNode
+	var i uint
+	stub.length = 1
+	stub.distance = 0
+	stub.dcode_insert_length = 0
+	stub.u.cost = kInfinity
+	for i = 0; i < length; i++ {
+		array[i] = stub
+	}
+}
+
+func zopfliNodeCopyLength(self *zopfliNode) uint32 {
+	return self.length & 0x1FFFFFF
+}
+
+func zopfliNodeLengthCode(self *zopfliNode) uint32 {
+	var modifier uint32 = self.length >> 25
+	return zopfliNodeCopyLength(self) + 9 - modifier
+}
+
+func zopfliNodeCopyDistance(self *zopfliNode) uint32 {
+	return self.distance
+}
+
+func zopfliNodeDistanceCode(self *zopfliNode) uint32 {
+	var short_code uint32 = self.dcode_insert_length >> 27
+	if short_code == 0 {
+		return zopfliNodeCopyDistance(self) + numDistanceShortCodes - 1
+	} else {
+		return short_code - 1
+	}
+}
+
+func zopfliNodeCommandLength(self *zopfliNode) uint32 {
+	return zopfliNodeCopyLength(self) + (self.dcode_insert_length & 0x7FFFFFF)
+}
+
+/* Histogram based cost model for zopflification. */
+type zopfliCostModel struct {
+	cost_cmd_               [numCommandSymbols]float32
+	cost_dist_              []float32
+	distance_histogram_size uint32
+	literal_costs_          []float32
+	min_cost_cmd_           float32
+	num_bytes_              uint
+}
+
+func initZopfliCostModel(self *zopfliCostModel, dist *distanceParams, num_bytes uint) {
+	var distance_histogram_size uint32 = dist.alphabet_size
+	if distance_histogram_size > maxEffectiveDistanceAlphabetSize {
+		distance_histogram_size = maxEffectiveDistanceAlphabetSize
+	}
+
+	self.num_bytes_ = num_bytes
+	self.literal_costs_ = make([]float32, (num_bytes + 2))
+	self.cost_dist_ = make([]float32, (dist.alphabet_size))
+	self.distance_histogram_size = distance_histogram_size
+}
+
+func cleanupZopfliCostModel(self *zopfliCostModel) {
+	self.literal_costs_ = nil
+	self.cost_dist_ = nil
+}
+
+func setCost(histogram []uint32, histogram_size uint, literal_histogram bool, cost []float32) {
+	var sum uint = 0
+	var missing_symbol_sum uint
+	var log2sum float32
+	var missing_symbol_cost float32
+	var i uint
+	for i = 0; i < histogram_size; i++ {
+		sum += uint(histogram[i])
+	}
+
+	log2sum = float32(fastLog2(sum))
+	missing_symbol_sum = sum
+	if !literal_histogram {
+		for i = 0; i < histogram_size; i++ {
+			if histogram[i] == 0 {
+				missing_symbol_sum++
+			}
+		}
+	}
+
+	missing_symbol_cost = float32(fastLog2(missing_symbol_sum)) + 2
+	for i = 0; i < histogram_size; i++ {
+		if histogram[i] == 0 {
+			cost[i] = missing_symbol_cost
+			continue
+		}
+
+		/* Shannon bits for this symbol. */
+		cost[i] = log2sum - float32(fastLog2(uint(histogram[i])))
+
+		/* Cannot be coded with less than 1 bit */
+		if cost[i] < 1 {
+			cost[i] = 1
+		}
+	}
+}
+
+func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, num_commands uint, last_insert_len uint) {
+	var histogram_literal [numLiteralSymbols]uint32
+	var histogram_cmd [numCommandSymbols]uint32
+	var histogram_dist [maxEffectiveDistanceAlphabetSize]uint32
+	var cost_literal [numLiteralSymbols]float32
+	var pos uint = position - last_insert_len
+	var min_cost_cmd float32 = kInfinity
+	var i uint
+	var cost_cmd []float32 = self.cost_cmd_[:]
+	var literal_costs []float32
+
+	histogram_literal = [numLiteralSymbols]uint32{}
+	histogram_cmd = [numCommandSymbols]uint32{}
+	histogram_dist = [maxEffectiveDistanceAlphabetSize]uint32{}
+
+	for i = 0; i < num_commands; i++ {
+		var inslength uint = uint(commands[i].insert_len_)
+		var copylength uint = uint(commandCopyLen(&commands[i]))
+		var distcode uint = uint(commands[i].dist_prefix_) & 0x3FF
+		var cmdcode uint = uint(commands[i].cmd_prefix_)
+		var j uint
+
+		histogram_cmd[cmdcode]++
+		if cmdcode >= 128 {
+			histogram_dist[distcode]++
+		}
+
+		for j = 0; j < inslength; j++ {
+			histogram_literal[ringbuffer[(pos+j)&ringbuffer_mask]]++
+		}
+
+		pos += inslength + copylength
+	}
+
+	setCost(histogram_literal[:], numLiteralSymbols, true, cost_literal[:])
+	setCost(histogram_cmd[:], numCommandSymbols, false, cost_cmd)
+	setCost(histogram_dist[:], uint(self.distance_histogram_size), false, self.cost_dist_)
+
+	for i = 0; i < numCommandSymbols; i++ {
+		min_cost_cmd = brotli_min_float(min_cost_cmd, cost_cmd[i])
+	}
+
+	self.min_cost_cmd_ = min_cost_cmd
+	{
+		literal_costs = self.literal_costs_
+		var literal_carry float32 = 0.0
+		var num_bytes uint = self.num_bytes_
+		literal_costs[0] = 0.0
+		for i = 0; i < num_bytes; i++ {
+			literal_carry += cost_literal[ringbuffer[(position+i)&ringbuffer_mask]]
+			literal_costs[i+1] = literal_costs[i] + literal_carry
+			literal_carry -= literal_costs[i+1] - literal_costs[i]
+		}
+	}
+}
+
+func zopfliCostModelSetFromLiteralCosts(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint) {
+	var literal_costs []float32 = self.literal_costs_
+	var literal_carry float32 = 0.0
+	var cost_dist []float32 = self.cost_dist_
+	var cost_cmd []float32 = self.cost_cmd_[:]
+	var num_bytes uint = self.num_bytes_
+	var i uint
+	estimateBitCostsForLiterals(position, num_bytes, ringbuffer_mask, ringbuffer, literal_costs[1:])
+	literal_costs[0] = 0.0
+	for i = 0; i < num_bytes; i++ {
+		literal_carry += literal_costs[i+1]
+		literal_costs[i+1] = literal_costs[i] + literal_carry
+		literal_carry -= literal_costs[i+1] - literal_costs[i]
+	}
+
+	for i = 0; i < numCommandSymbols; i++ {
+		cost_cmd[i] = float32(fastLog2(uint(11 + uint32(i))))
+	}
+
+	for i = 0; uint32(i) < self.distance_histogram_size; i++ {
+		cost_dist[i] = float32(fastLog2(uint(20 + uint32(i))))
+	}
+
+	self.min_cost_cmd_ = float32(fastLog2(11))
+}
+
+func zopfliCostModelGetCommandCost(self *zopfliCostModel, cmdcode uint16) float32 {
+	return self.cost_cmd_[cmdcode]
+}
+
+func zopfliCostModelGetDistanceCost(self *zopfliCostModel, distcode uint) float32 {
+	return self.cost_dist_[distcode]
+}
+
+func zopfliCostModelGetLiteralCosts(self *zopfliCostModel, from uint, to uint) float32 {
+	return self.literal_costs_[to] - self.literal_costs_[from]
+}
+
+func zopfliCostModelGetMinCostCmd(self *zopfliCostModel) float32 {
+	return self.min_cost_cmd_
+}
+
+/* REQUIRES: len >= 2, start_pos <= pos */
+/* REQUIRES: cost < kInfinity, nodes[start_pos].cost < kInfinity */
+/* Maintains the "ZopfliNode array invariant". */
+func updateZopfliNode(nodes []zopfliNode, pos uint, start_pos uint, len uint, len_code uint, dist uint, short_code uint, cost float32) {
+	var next *zopfliNode = &nodes[pos+len]
+	next.length = uint32(len | (len+9-len_code)<<25)
+	next.distance = uint32(dist)
+	next.dcode_insert_length = uint32(short_code<<27 | (pos - start_pos))
+	next.u.cost = cost
+}
+
+type posData struct {
+	pos            uint
+	distance_cache [4]int
+	costdiff       float32
+	cost           float32
+}
+
+/* Maintains the smallest 8 cost difference together with their positions */
+type startPosQueue struct {
+	q_   [8]posData
+	idx_ uint
+}
+
+func initStartPosQueue(self *startPosQueue) {
+	self.idx_ = 0
+}
+
+func startPosQueueSize(self *startPosQueue) uint {
+	return brotli_min_size_t(self.idx_, 8)
+}
+
+func startPosQueuePush(self *startPosQueue, posdata *posData) {
+	var offset uint = ^(self.idx_) & 7
+	self.idx_++
+	var len uint = startPosQueueSize(self)
+	var i uint
+	var q []posData = self.q_[:]
+	q[offset] = *posdata
+
+	/* Restore the sorted order. In the list of |len| items at most |len - 1|
+	   adjacent element comparisons / swaps are required. */
+	for i = 1; i < len; i++ {
+		if q[offset&7].costdiff > q[(offset+1)&7].costdiff {
+			var tmp posData = q[offset&7]
+			q[offset&7] = q[(offset+1)&7]
+			q[(offset+1)&7] = tmp
+		}
+
+		offset++
+	}
+}
+
+func startPosQueueAt(self *startPosQueue, k uint) *posData {
+	return &self.q_[(k-self.idx_)&7]
+}
+
+/* Returns the minimum possible copy length that can improve the cost of any */
+/* future position. */
+func computeMinimumCopyLength(start_cost float32, nodes []zopfliNode, num_bytes uint, pos uint) uint {
+	var min_cost float32 = start_cost
+	var len uint = 2
+	var next_len_bucket uint = 4
+	/* Compute the minimum possible cost of reaching any future position. */
+
+	var next_len_offset uint = 10
+	for pos+len <= num_bytes && nodes[pos+len].u.cost <= min_cost {
+		/* We already reached (pos + len) with no more cost than the minimum
+		   possible cost of reaching anything from this pos, so there is no point in
+		   looking for lengths <= len. */
+		len++
+
+		if len == next_len_offset {
+			/* We reached the next copy length code bucket, so we add one more
+			   extra bit to the minimum cost. */
+			min_cost += 1.0
+
+			next_len_offset += next_len_bucket
+			next_len_bucket *= 2
+		}
+	}
+
+	return uint(len)
+}
+
+/* REQUIRES: nodes[pos].cost < kInfinity
+   REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */
+func computeDistanceShortcut(block_start uint, pos uint, max_backward_limit uint, gap uint, nodes []zopfliNode) uint32 {
+	var clen uint = uint(zopfliNodeCopyLength(&nodes[pos]))
+	var ilen uint = uint(nodes[pos].dcode_insert_length & 0x7FFFFFF)
+	var dist uint = uint(zopfliNodeCopyDistance(&nodes[pos]))
+
+	/* Since |block_start + pos| is the end position of the command, the copy part
+	   starts from |block_start + pos - clen|. Distances that are greater than
+	   this or greater than |max_backward_limit| + |gap| are static dictionary
+	   references, and do not update the last distances.
+	   Also distance code 0 (last distance) does not update the last distances. */
+	if pos == 0 {
+		return 0
+	} else if dist+clen <= block_start+pos+gap && dist <= max_backward_limit+gap && zopfliNodeDistanceCode(&nodes[pos]) > 0 {
+		return uint32(pos)
+	} else {
+		return nodes[pos-clen-ilen].u.shortcut
+	}
+}
+
+/* Fills in dist_cache[0..3] with the last four distances (as defined by
+   Section 4. of the Spec) that would be used at (block_start + pos) if we
+   used the shortest path of commands from block_start, computed from
+   nodes[0..pos]. The last four distances at block_start are in
+   starting_dist_cache[0..3].
+   REQUIRES: nodes[pos].cost < kInfinity
+   REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */
+func computeDistanceCache(pos uint, starting_dist_cache []int, nodes []zopfliNode, dist_cache []int) {
+	var idx int = 0
+	var p uint = uint(nodes[pos].u.shortcut)
+	for idx < 4 && p > 0 {
+		var ilen uint = uint(nodes[p].dcode_insert_length & 0x7FFFFFF)
+		var clen uint = uint(zopfliNodeCopyLength(&nodes[p]))
+		var dist uint = uint(zopfliNodeCopyDistance(&nodes[p]))
+		dist_cache[idx] = int(dist)
+		idx++
+
+		/* Because of prerequisite, p >= clen + ilen >= 2. */
+		p = uint(nodes[p-clen-ilen].u.shortcut)
+	}
+
+	for ; idx < 4; idx++ {
+		dist_cache[idx] = starting_dist_cache[0]
+		starting_dist_cache = starting_dist_cache[1:]
+	}
+}
+
+/* Maintains "ZopfliNode array invariant" and pushes node to the queue, if it
+   is eligible. */
+func evaluateNode(block_start uint, pos uint, max_backward_limit uint, gap uint, starting_dist_cache []int, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) {
+	/* Save cost, because ComputeDistanceCache invalidates it. */
+	var node_cost float32 = nodes[pos].u.cost
+	nodes[pos].u.shortcut = computeDistanceShortcut(block_start, pos, max_backward_limit, gap, nodes)
+	if node_cost <= zopfliCostModelGetLiteralCosts(model, 0, pos) {
+		var posdata posData
+		posdata.pos = pos
+		posdata.cost = node_cost
+		posdata.costdiff = node_cost - zopfliCostModelGetLiteralCosts(model, 0, pos)
+		computeDistanceCache(pos, starting_dist_cache, nodes, posdata.distance_cache[:])
+		startPosQueuePush(queue, &posdata)
+	}
+}
+
+/* Returns longest copy length. */
+func updateNodes(num_bytes uint, block_start uint, pos uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, max_backward_limit uint, starting_dist_cache []int, num_matches uint, matches []backwardMatch, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) uint {
+	var cur_ix uint = block_start + pos
+	var cur_ix_masked uint = cur_ix & ringbuffer_mask
+	var max_distance uint = brotli_min_size_t(cur_ix, max_backward_limit)
+	var max_len uint = num_bytes - pos
+	var max_zopfli_len uint = maxZopfliLen(params)
+	var max_iters uint = maxZopfliCandidates(params)
+	var min_len uint
+	var result uint = 0
+	var k uint
+	var gap uint = 0
+
+	evaluateNode(block_start, pos, max_backward_limit, gap, starting_dist_cache, model, queue, nodes)
+	{
+		var posdata *posData = startPosQueueAt(queue, 0)
+		var min_cost float32 = (posdata.cost + zopfliCostModelGetMinCostCmd(model) + zopfliCostModelGetLiteralCosts(model, posdata.pos, pos))
+		min_len = computeMinimumCopyLength(min_cost, nodes, num_bytes, pos)
+	}
+
+	/* Go over the command starting positions in order of increasing cost
+	   difference. */
+	for k = 0; k < max_iters && k < startPosQueueSize(queue); k++ {
+		var posdata *posData = startPosQueueAt(queue, k)
+		var start uint = posdata.pos
+		var inscode uint16 = getInsertLengthCode(pos - start)
+		var start_costdiff float32 = posdata.costdiff
+		var base_cost float32 = start_costdiff + float32(getInsertExtra(inscode)) + zopfliCostModelGetLiteralCosts(model, 0, pos)
+		var best_len uint = min_len - 1
+		var j uint = 0
+		/* Look for last distance matches using the distance cache from this
+		   starting position. */
+		for ; j < numDistanceShortCodes && best_len < max_len; j++ {
+			var idx uint = uint(kDistanceCacheIndex[j])
+			var backward uint = uint(posdata.distance_cache[idx] + kDistanceCacheOffset[j])
+			var prev_ix uint = cur_ix - backward
+			var len uint = 0
+			var continuation byte = ringbuffer[cur_ix_masked+best_len]
+			if cur_ix_masked+best_len > ringbuffer_mask {
+				break
+			}
+
+			if backward > max_distance+gap {
+				/* Word dictionary -> ignore. */
+				continue
+			}
+
+			if backward <= max_distance {
+				/* Regular backward reference. */
+				if prev_ix >= cur_ix {
+					continue
+				}
+
+				prev_ix &= ringbuffer_mask
+				if prev_ix+best_len > ringbuffer_mask || continuation != ringbuffer[prev_ix+best_len] {
+					continue
+				}
+
+				len = findMatchLengthWithLimit(ringbuffer[prev_ix:], ringbuffer[cur_ix_masked:], max_len)
+			} else {
+				continue
+			}
+			{
+				var dist_cost float32 = base_cost + zopfliCostModelGetDistanceCost(model, j)
+				var l uint
+				for l = best_len + 1; l <= len; l++ {
+					var copycode uint16 = getCopyLengthCode(l)
+					var cmdcode uint16 = combineLengthCodes(inscode, copycode, j == 0)
+					var tmp float32
+					if cmdcode < 128 {
+						tmp = base_cost
+					} else {
+						tmp = dist_cost
+					}
+					var cost float32 = tmp + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode)
+					if cost < nodes[pos+l].u.cost {
+						updateZopfliNode(nodes, pos, start, l, l, backward, j+1, cost)
+						result = brotli_max_size_t(result, l)
+					}
+
+					best_len = l
+				}
+			}
+		}
+
+		/* At higher iterations look only for new last distance matches, since
+		   looking only for new command start positions with the same distances
+		   does not help much. */
+		if k >= 2 {
+			continue
+		}
+		{
+			/* Loop through all possible copy lengths at this position. */
+			var len uint = min_len
+			for j = 0; j < num_matches; j++ {
+				var match backwardMatch = matches[j]
+				var dist uint = uint(match.distance)
+				var is_dictionary_match bool = (dist > max_distance+gap)
+				var dist_code uint = dist + numDistanceShortCodes - 1
+				var dist_symbol uint16
+				var distextra uint32
+				var distnumextra uint32
+				var dist_cost float32
+				var max_match_len uint
+				/* We already tried all possible last distance matches, so we can use
+				   normal distance code here. */
+				prefixEncodeCopyDistance(dist_code, uint(params.dist.num_direct_distance_codes), uint(params.dist.distance_postfix_bits), &dist_symbol, &distextra)
+
+				distnumextra = uint32(dist_symbol) >> 10
+				dist_cost = base_cost + float32(distnumextra) + zopfliCostModelGetDistanceCost(model, uint(dist_symbol)&0x3FF)
+
+				/* Try all copy lengths up until the maximum copy length corresponding
+				   to this distance. If the distance refers to the static dictionary, or
+				   the maximum length is long enough, try only one maximum length. */
+				max_match_len = backwardMatchLength(&match)
+
+				if len < max_match_len && (is_dictionary_match || max_match_len > max_zopfli_len) {
+					len = max_match_len
+				}
+
+				for ; len <= max_match_len; len++ {
+					var len_code uint
+					if is_dictionary_match {
+						len_code = backwardMatchLengthCode(&match)
+					} else {
+						len_code = len
+					}
+					var copycode uint16 = getCopyLengthCode(len_code)
+					var cmdcode uint16 = combineLengthCodes(inscode, copycode, false)
+					var cost float32 = dist_cost + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode)
+					if cost < nodes[pos+len].u.cost {
+						updateZopfliNode(nodes, pos, start, uint(len), len_code, dist, 0, cost)
+						result = brotli_max_size_t(result, uint(len))
+					}
+				}
+			}
+		}
+	}
+
+	return result
+}
+
+func computeShortestPathFromNodes(num_bytes uint, nodes []zopfliNode) uint {
+	var index uint = num_bytes
+	var num_commands uint = 0
+	for nodes[index].dcode_insert_length&0x7FFFFFF == 0 && nodes[index].length == 1 {
+		index--
+	}
+	nodes[index].u.next = math.MaxUint32
+	for index != 0 {
+		var len uint = uint(zopfliNodeCommandLength(&nodes[index]))
+		index -= uint(len)
+		nodes[index].u.next = uint32(len)
+		num_commands++
+	}
+
+	return num_commands
+}
+
+/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
+func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands []command, num_literals *uint) {
+	var max_backward_limit uint = maxBackwardLimit(params.lgwin)
+	var pos uint = 0
+	var offset uint32 = nodes[0].u.next
+	var i uint
+	var gap uint = 0
+	for i = 0; offset != math.MaxUint32; i++ {
+		var next *zopfliNode = &nodes[uint32(pos)+offset]
+		var copy_length uint = uint(zopfliNodeCopyLength(next))
+		var insert_length uint = uint(next.dcode_insert_length & 0x7FFFFFF)
+		pos += insert_length
+		offset = next.u.next
+		if i == 0 {
+			insert_length += *last_insert_len
+			*last_insert_len = 0
+		}
+		{
+			var distance uint = uint(zopfliNodeCopyDistance(next))
+			var len_code uint = uint(zopfliNodeLengthCode(next))
+			var max_distance uint = brotli_min_size_t(block_start+pos, max_backward_limit)
+			var is_dictionary bool = (distance > max_distance+gap)
+			var dist_code uint = uint(zopfliNodeDistanceCode(next))
+			initCommand(&commands[i], &params.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code)
+
+			if !is_dictionary && dist_code > 0 {
+				dist_cache[3] = dist_cache[2]
+				dist_cache[2] = dist_cache[1]
+				dist_cache[1] = dist_cache[0]
+				dist_cache[0] = int(distance)
+			}
+		}
+
+		*num_literals += insert_length
+		pos += copy_length
+	}
+
+	*last_insert_len += num_bytes - pos
+}
+
+func zopfliIterate(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, gap uint, dist_cache []int, model *zopfliCostModel, num_matches []uint32, matches []backwardMatch, nodes []zopfliNode) uint {
+	var max_backward_limit uint = maxBackwardLimit(params.lgwin)
+	var max_zopfli_len uint = maxZopfliLen(params)
+	var queue startPosQueue
+	var cur_match_pos uint = 0
+	var i uint
+	nodes[0].length = 0
+	nodes[0].u.cost = 0
+	initStartPosQueue(&queue)
+	for i = 0; i+3 < num_bytes; i++ {
+		var skip uint = updateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask, params, max_backward_limit, dist_cache, uint(num_matches[i]), matches[cur_match_pos:], model, &queue, nodes)
+		if skip < longCopyQuickStep {
+			skip = 0
+		}
+		cur_match_pos += uint(num_matches[i])
+		if num_matches[i] == 1 && backwardMatchLength(&matches[cur_match_pos-1]) > max_zopfli_len {
+			skip = brotli_max_size_t(backwardMatchLength(&matches[cur_match_pos-1]), skip)
+		}
+
+		if skip > 1 {
+			skip--
+			for skip != 0 {
+				i++
+				if i+3 >= num_bytes {
+					break
+				}
+				evaluateNode(position, i, max_backward_limit, gap, dist_cache, model, &queue, nodes)
+				cur_match_pos += uint(num_matches[i])
+				skip--
+			}
+		}
+	}
+
+	return computeShortestPathFromNodes(num_bytes, nodes)
+}
+
+/* Computes the shortest path of commands from position to at most
+   position + num_bytes.
+
+   On return, path->size() is the number of commands found and path[i] is the
+   length of the i-th command (copy length plus insert length).
+   Note that the sum of the lengths of all commands can be less than num_bytes.
+
+   On return, the nodes[0..num_bytes] array will have the following
+   "ZopfliNode array invariant":
+   For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then
+     (1) nodes[i].copy_length() >= 2
+     (2) nodes[i].command_length() <= i and
+     (3) nodes[i - nodes[i].command_length()].cost < kInfinity
+
+ REQUIRES: nodes != nil and len(nodes) >= num_bytes + 1 */
+func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, dist_cache []int, hasher *h10, nodes []zopfliNode) uint {
+	var max_backward_limit uint = maxBackwardLimit(params.lgwin)
+	var max_zopfli_len uint = maxZopfliLen(params)
+	var model zopfliCostModel
+	var queue startPosQueue
+	var matches [2 * (maxNumMatchesH10 + 64)]backwardMatch
+	var store_end uint
+	if num_bytes >= hasher.StoreLookahead() {
+		store_end = position + num_bytes - hasher.StoreLookahead() + 1
+	} else {
+		store_end = position
+	}
+	var i uint
+	var gap uint = 0
+	var lz_matches_offset uint = 0
+	nodes[0].length = 0
+	nodes[0].u.cost = 0
+	initZopfliCostModel(&model, &params.dist, num_bytes)
+	zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask)
+	initStartPosQueue(&queue)
+	for i = 0; i+hasher.HashTypeLength()-1 < num_bytes; i++ {
+		var pos uint = position + i
+		var max_distance uint = brotli_min_size_t(pos, max_backward_limit)
+		var skip uint
+		var num_matches uint
+		num_matches = findAllMatchesH10(hasher, &params.dictionary, ringbuffer, ringbuffer_mask, pos, num_bytes-i, max_distance, gap, params, matches[lz_matches_offset:])
+		if num_matches > 0 && backwardMatchLength(&matches[num_matches-1]) > max_zopfli_len {
+			matches[0] = matches[num_matches-1]
+			num_matches = 1
+		}
+
+		skip = updateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask, params, max_backward_limit, dist_cache, num_matches, matches[:], &model, &queue, nodes)
+		if skip < longCopyQuickStep {
+			skip = 0
+		}
+		if num_matches == 1 && backwardMatchLength(&matches[0]) > max_zopfli_len {
+			skip = brotli_max_size_t(backwardMatchLength(&matches[0]), skip)
+		}
+
+		if skip > 1 {
+			/* Add the tail of the copy to the hasher. */
+			hasher.StoreRange(ringbuffer, ringbuffer_mask, pos+1, brotli_min_size_t(pos+skip, store_end))
+
+			skip--
+			for skip != 0 {
+				i++
+				if i+hasher.HashTypeLength()-1 >= num_bytes {
+					break
+				}
+				evaluateNode(position, i, max_backward_limit, gap, dist_cache, &model, &queue, nodes)
+				skip--
+			}
+		}
+	}
+
+	cleanupZopfliCostModel(&model)
+	return computeShortestPathFromNodes(num_bytes, nodes)
+}
+
+func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
+	var nodes []zopfliNode
+	nodes = make([]zopfliNode, (num_bytes + 1))
+	initZopfliNodes(nodes, num_bytes+1)
+	*num_commands += zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes)
+	zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals)
+	nodes = nil
+}
+
+func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
+	var max_backward_limit uint = maxBackwardLimit(params.lgwin)
+	var num_matches []uint32 = make([]uint32, num_bytes)
+	var matches_size uint = 4 * num_bytes
+	var store_end uint
+	if num_bytes >= hasher.StoreLookahead() {
+		store_end = position + num_bytes - hasher.StoreLookahead() + 1
+	} else {
+		store_end = position
+	}
+	var cur_match_pos uint = 0
+	var i uint
+	var orig_num_literals uint
+	var orig_last_insert_len uint
+	var orig_dist_cache [4]int
+	var orig_num_commands uint
+	var model zopfliCostModel
+	var nodes []zopfliNode
+	var matches []backwardMatch = make([]backwardMatch, matches_size)
+	var gap uint = 0
+	var shadow_matches uint = 0
+	var new_array []backwardMatch
+	for i = 0; i+hasher.HashTypeLength()-1 < num_bytes; i++ {
+		var pos uint = position + i
+		var max_distance uint = brotli_min_size_t(pos, max_backward_limit)
+		var max_length uint = num_bytes - i
+		var num_found_matches uint
+		var cur_match_end uint
+		var j uint
+
+		/* Ensure that we have enough free slots. */
+		if matches_size < cur_match_pos+maxNumMatchesH10+shadow_matches {
+			var new_size uint = matches_size
+			if new_size == 0 {
+				new_size = cur_match_pos + maxNumMatchesH10 + shadow_matches
+			}
+
+			for new_size < cur_match_pos+maxNumMatchesH10+shadow_matches {
+				new_size *= 2
+			}
+
+			new_array = make([]backwardMatch, new_size)
+			if matches_size != 0 {
+				copy(new_array, matches[:matches_size])
+			}
+
+			matches = new_array
+			matches_size = new_size
+		}
+
+		num_found_matches = findAllMatchesH10(hasher.(*h10), &params.dictionary, ringbuffer, ringbuffer_mask, pos, max_length, max_distance, gap, params, matches[cur_match_pos+shadow_matches:])
+		cur_match_end = cur_match_pos + num_found_matches
+		for j = cur_match_pos; j+1 < cur_match_end; j++ {
+			assert(backwardMatchLength(&matches[j]) <= backwardMatchLength(&matches[j+1]))
+		}
+
+		num_matches[i] = uint32(num_found_matches)
+		if num_found_matches > 0 {
+			var match_len uint = backwardMatchLength(&matches[cur_match_end-1])
+			if match_len > maxZopfliLenQuality11 {
+				var skip uint = match_len - 1
+				matches[cur_match_pos] = matches[cur_match_end-1]
+				cur_match_pos++
+				num_matches[i] = 1
+
+				/* Add the tail of the copy to the hasher. */
+				hasher.StoreRange(ringbuffer, ringbuffer_mask, pos+1, brotli_min_size_t(pos+match_len, store_end))
+				var pos uint = i
+				for i := 0; i < int(skip); i++ {
+					num_matches[pos+1:][i] = 0
+				}
+				i += skip
+			} else {
+				cur_match_pos = cur_match_end
+			}
+		}
+	}
+
+	orig_num_literals = *num_literals
+	orig_last_insert_len = *last_insert_len
+	copy(orig_dist_cache[:], dist_cache[:4])
+	orig_num_commands = *num_commands
+	nodes = make([]zopfliNode, (num_bytes + 1))
+	initZopfliCostModel(&model, &params.dist, num_bytes)
+	for i = 0; i < 2; i++ {
+		initZopfliNodes(nodes, num_bytes+1)
+		if i == 0 {
+			zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask)
+		} else {
+			zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, commands, *num_commands-orig_num_commands, orig_last_insert_len)
+		}
+
+		*num_commands = orig_num_commands
+		*num_literals = orig_num_literals
+		*last_insert_len = orig_last_insert_len
+		copy(dist_cache, orig_dist_cache[:4])
+		*num_commands += zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes)
+		zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals)
+	}
+
+	cleanupZopfliCostModel(&model)
+	nodes = nil
+	matches = nil
+	num_matches = nil
+}

vendor/github.com/andybalholm/brotli/bit_cost.go

@@ -0,0 +1,436 @@
+package brotli
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Functions to estimate the bit cost of Huffman trees. */
+func shannonEntropy(population []uint32, size uint, total *uint) float64 {
+	var sum uint = 0
+	var retval float64 = 0
+	var population_end []uint32 = population[size:]
+	var p uint
+	for -cap(population) < -cap(population_end) {
+		p = uint(population[0])
+		population = population[1:]
+		sum += p
+		retval -= float64(p) * fastLog2(p)
+	}
+
+	if sum != 0 {
+		retval += float64(sum) * fastLog2(sum)
+	}
+	*total = sum
+	return retval
+}
+
+func bitsEntropy(population []uint32, size uint) float64 {
+	var sum uint
+	var retval float64 = shannonEntropy(population, size, &sum)
+	if retval < float64(sum) {
+		/* At least one bit per literal is needed. */
+		retval = float64(sum)
+	}
+
+	return retval
+}
+
+const kOneSymbolHistogramCost float64 = 12
+const kTwoSymbolHistogramCost float64 = 20
+const kThreeSymbolHistogramCost float64 = 28
+const kFourSymbolHistogramCost float64 = 37
+
+func populationCostLiteral(histogram *histogramLiteral) float64 {
+	var data_size uint = histogramDataSizeLiteral()
+	var count int = 0
+	var s [5]uint
+	var bits float64 = 0.0
+	var i uint
+	if histogram.total_count_ == 0 {
+		return kOneSymbolHistogramCost
+	}
+
+	for i = 0; i < data_size; i++ {
+		if histogram.data_[i] > 0 {
+			s[count] = i
+			count++
+			if count > 4 {
+				break
+			}
+		}
+	}
+
+	if count == 1 {
+		return kOneSymbolHistogramCost
+	}
+
+	if count == 2 {
+		return kTwoSymbolHistogramCost + float64(histogram.total_count_)
+	}
+
+	if count == 3 {
+		var histo0 uint32 = histogram.data_[s[0]]
+		var histo1 uint32 = histogram.data_[s[1]]
+		var histo2 uint32 = histogram.data_[s[2]]
+		var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2))
+		return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax)
+	}
+
+	if count == 4 {
+		var histo [4]uint32
+		var h23 uint32
+		var histomax uint32
+		for i = 0; i < 4; i++ {
+			histo[i] = histogram.data_[s[i]]
+		}
+
+		/* Sort */
+		for i = 0; i < 4; i++ {
+			var j uint
+			for j = i + 1; j < 4; j++ {
+				if histo[j] > histo[i] {
+					var tmp uint32 = histo[j]
+					histo[j] = histo[i]
+					histo[i] = tmp
+				}
+			}
+		}
+
+		h23 = histo[2] + histo[3]
+		histomax = brotli_max_uint32_t(h23, histo[0])
+		return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax)
+	}
+	{
+		var max_depth uint = 1
+		var depth_histo = [codeLengthCodes]uint32{0}
+		/* In this loop we compute the entropy of the histogram and simultaneously
+		   build a simplified histogram of the code length codes where we use the
+		   zero repeat code 17, but we don't use the non-zero repeat code 16. */
+
+		var log2total float64 = fastLog2(histogram.total_count_)
+		for i = 0; i < data_size; {
+			if histogram.data_[i] > 0 {
+				var log2p float64 = log2total - fastLog2(uint(histogram.data_[i]))
+				/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
+				   = log2(total_count) - log2(count(symbol)) */
+
+				var depth uint = uint(log2p + 0.5)
+				/* Approximate the bit depth by round(-log2(P(symbol))) */
+				bits += float64(histogram.data_[i]) * log2p
+
+				if depth > 15 {
+					depth = 15
+				}
+
+				if depth > max_depth {
+					max_depth = depth
+				}
+
+				depth_histo[depth]++
+				i++
+			} else {
+				var reps uint32 = 1
+				/* Compute the run length of zeros and add the appropriate number of 0
+				   and 17 code length codes to the code length code histogram. */
+
+				var k uint
+				for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ {
+					reps++
+				}
+
+				i += uint(reps)
+				if i == data_size {
+					/* Don't add any cost for the last zero run, since these are encoded
+					   only implicitly. */
+					break
+				}
+
+				if reps < 3 {
+					depth_histo[0] += reps
+				} else {
+					reps -= 2
+					for reps > 0 {
+						depth_histo[repeatZeroCodeLength]++
+
+						/* Add the 3 extra bits for the 17 code length code. */
+						bits += 3
+
+						reps >>= 3
+					}
+				}
+			}
+		}
+
+		/* Add the estimated encoding cost of the code length code histogram. */
+		bits += float64(18 + 2*max_depth)
+
+		/* Add the entropy of the code length code histogram. */
+		bits += bitsEntropy(depth_histo[:], codeLengthCodes)
+	}
+
+	return bits
+}
+
+func populationCostCommand(histogram *histogramCommand) float64 {
+	var data_size uint = histogramDataSizeCommand()
+	var count int = 0
+	var s [5]uint
+	var bits float64 = 0.0
+	var i uint
+	if histogram.total_count_ == 0 {
+		return kOneSymbolHistogramCost
+	}
+
+	for i = 0; i < data_size; i++ {
+		if histogram.data_[i] > 0 {
+			s[count] = i
+			count++
+			if count > 4 {
+				break
+			}
+		}
+	}
+
+	if count == 1 {
+		return kOneSymbolHistogramCost
+	}
+
+	if count == 2 {
+		return kTwoSymbolHistogramCost + float64(histogram.total_count_)
+	}
+
+	if count == 3 {
+		var histo0 uint32 = histogram.data_[s[0]]
+		var histo1 uint32 = histogram.data_[s[1]]
+		var histo2 uint32 = histogram.data_[s[2]]
+		var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2))
+		return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax)
+	}
+
+	if count == 4 {
+		var histo [4]uint32
+		var h23 uint32
+		var histomax uint32
+		for i = 0; i < 4; i++ {
+			histo[i] = histogram.data_[s[i]]
+		}
+
+		/* Sort */
+		for i = 0; i < 4; i++ {
+			var j uint
+			for j = i + 1; j < 4; j++ {
+				if histo[j] > histo[i] {
+					var tmp uint32 = histo[j]
+					histo[j] = histo[i]
+					histo[i] = tmp
+				}
+			}
+		}
+
+		h23 = histo[2] + histo[3]
+		histomax = brotli_max_uint32_t(h23, histo[0])
+		return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax)
+	}
+	{
+		var max_depth uint = 1
+		var depth_histo = [codeLengthCodes]uint32{0}
+		/* In this loop we compute the entropy of the histogram and simultaneously
+		   build a simplified histogram of the code length codes where we use the
+		   zero repeat code 17, but we don't use the non-zero repeat code 16. */
+
+		var log2total float64 = fastLog2(histogram.total_count_)
+		for i = 0; i < data_size; {
+			if histogram.data_[i] > 0 {
+				var log2p float64 = log2total - fastLog2(uint(histogram.data_[i]))
+				/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
+				   = log2(total_count) - log2(count(symbol)) */
+
+				var depth uint = uint(log2p + 0.5)
+				/* Approximate the bit depth by round(-log2(P(symbol))) */
+				bits += float64(histogram.data_[i]) * log2p
+
+				if depth > 15 {
+					depth = 15
+				}
+
+				if depth > max_depth {
+					max_depth = depth
+				}
+
+				depth_histo[depth]++
+				i++
+			} else {
+				var reps uint32 = 1
+				/* Compute the run length of zeros and add the appropriate number of 0
+				   and 17 code length codes to the code length code histogram. */
+
+				var k uint
+				for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ {
+					reps++
+				}
+
+				i += uint(reps)
+				if i == data_size {
+					/* Don't add any cost for the last zero run, since these are encoded
+					   only implicitly. */
+					break
+				}
+
+				if reps < 3 {
+					depth_histo[0] += reps
+				} else {
+					reps -= 2
+					for reps > 0 {
+						depth_histo[repeatZeroCodeLength]++
+
+						/* Add the 3 extra bits for the 17 code length code. */
+						bits += 3
+
+						reps >>= 3
+					}
+				}
+			}
+		}
+
+		/* Add the estimated encoding cost of the code length code histogram. */
+		bits += float64(18 + 2*max_depth)
+
+		/* Add the entropy of the code length code histogram. */
+		bits += bitsEntropy(depth_histo[:], codeLengthCodes)
+	}
+
+	return bits
+}
+
+func populationCostDistance(histogram *histogramDistance) float64 {
+	var data_size uint = histogramDataSizeDistance()
+	var count int = 0
+	var s [5]uint
+	var bits float64 = 0.0
+	var i uint
+	if histogram.total_count_ == 0 {
+		return kOneSymbolHistogramCost
+	}
+
+	for i = 0; i < data_size; i++ {
+		if histogram.data_[i] > 0 {
+			s[count] = i
+			count++
+			if count > 4 {
+				break
+			}
+		}
+	}
+
+	if count == 1 {
+		return kOneSymbolHistogramCost
+	}
+
+	if count == 2 {
+		return kTwoSymbolHistogramCost + float64(histogram.total_count_)
+	}
+
+	if count == 3 {
+		var histo0 uint32 = histogram.data_[s[0]]
+		var histo1 uint32 = histogram.data_[s[1]]
+		var histo2 uint32 = histogram.data_[s[2]]
+		var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2))
+		return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax)
+	}
+
+	if count == 4 {
+		var histo [4]uint32
+		var h23 uint32
+		var histomax uint32
+		for i = 0; i < 4; i++ {
+			histo[i] = histogram.data_[s[i]]
+		}
+
+		/* Sort */
+		for i = 0; i < 4; i++ {
+			var j uint
+			for j = i + 1; j < 4; j++ {
+				if histo[j] > histo[i] {
+					var tmp uint32 = histo[j]
+					histo[j] = histo[i]
+					histo[i] = tmp
+				}
+			}
+		}
+
+		h23 = histo[2] + histo[3]
+		histomax = brotli_max_uint32_t(h23, histo[0])
+		return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax)
+	}
+	{
+		var max_depth uint = 1
+		var depth_histo = [codeLengthCodes]uint32{0}
+		/* In this loop we compute the entropy of the histogram and simultaneously
+		   build a simplified histogram of the code length codes where we use the
+		   zero repeat code 17, but we don't use the non-zero repeat code 16. */
+
+		var log2total float64 = fastLog2(histogram.total_count_)
+		for i = 0; i < data_size; {
+			if histogram.data_[i] > 0 {
+				var log2p float64 = log2total - fastLog2(uint(histogram.data_[i]))
+				/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
+				   = log2(total_count) - log2(count(symbol)) */
+
+				var depth uint = uint(log2p + 0.5)
+				/* Approximate the bit depth by round(-log2(P(symbol))) */
+				bits += float64(histogram.data_[i]) * log2p
+
+				if depth > 15 {
+					depth = 15
+				}
+
+				if depth > max_depth {
+					max_depth = depth
+				}
+
+				depth_histo[depth]++
+				i++
+			} else {
+				var reps uint32 = 1
+				/* Compute the run length of zeros and add the appropriate number of 0
+				   and 17 code length codes to the code length code histogram. */
+
+				var k uint
+				for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ {
+					reps++
+				}
+
+				i += uint(reps)
+				if i == data_size {
+					/* Don't add any cost for the last zero run, since these are encoded
+					   only implicitly. */
+					break
+				}
+
+				if reps < 3 {
+					depth_histo[0] += reps
+				} else {
+					reps -= 2
+					for reps > 0 {
+						depth_histo[repeatZeroCodeLength]++
+
+						/* Add the 3 extra bits for the 17 code length code. */
+						bits += 3
+
+						reps >>= 3
+					}
+				}
+			}
+		}
+
+		/* Add the estimated encoding cost of the code length code histogram. */
+		bits += float64(18 + 2*max_depth)
+
+		/* Add the entropy of the code length code histogram. */
+		bits += bitsEntropy(depth_histo[:], codeLengthCodes)
+	}
+
+	return bits
+}

vendor/github.com/andybalholm/brotli/bit_reader.go

@@ -0,0 +1,266 @@
+package brotli
+
+import "encoding/binary"
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Bit reading helpers */
+
+const shortFillBitWindowRead = (8 >> 1)
+
+var kBitMask = [33]uint32{
+	0x00000000,
+	0x00000001,
+	0x00000003,
+	0x00000007,
+	0x0000000F,
+	0x0000001F,
+	0x0000003F,
+	0x0000007F,
+	0x000000FF,
+	0x000001FF,
+	0x000003FF,
+	0x000007FF,
+	0x00000FFF,
+	0x00001FFF,
+	0x00003FFF,
+	0x00007FFF,
+	0x0000FFFF,
+	0x0001FFFF,
+	0x0003FFFF,
+	0x0007FFFF,
+	0x000FFFFF,
+	0x001FFFFF,
+	0x003FFFFF,
+	0x007FFFFF,
+	0x00FFFFFF,
+	0x01FFFFFF,
+	0x03FFFFFF,
+	0x07FFFFFF,
+	0x0FFFFFFF,
+	0x1FFFFFFF,
+	0x3FFFFFFF,
+	0x7FFFFFFF,
+	0xFFFFFFFF,
+}
+
+func bitMask(n uint32) uint32 {
+	return kBitMask[n]
+}
+
+type bitReader struct {
+	val_      uint64
+	bit_pos_  uint32
+	input     []byte
+	input_len uint
+	byte_pos  uint
+}
+
+type bitReaderState struct {
+	val_      uint64
+	bit_pos_  uint32
+	input     []byte
+	input_len uint
+	byte_pos  uint
+}
+
+/* Initializes the BrotliBitReader fields. */
+
+/* Ensures that accumulator is not empty.
+   May consume up to sizeof(brotli_reg_t) - 1 bytes of input.
+   Returns false if data is required but there is no input available.
+   For BROTLI_ALIGNED_READ this function also prepares bit reader for aligned
+   reading. */
+func bitReaderSaveState(from *bitReader, to *bitReaderState) {
+	to.val_ = from.val_
+	to.bit_pos_ = from.bit_pos_
+	to.input = from.input
+	to.input_len = from.input_len
+	to.byte_pos = from.byte_pos
+}
+
+func bitReaderRestoreState(to *bitReader, from *bitReaderState) {
+	to.val_ = from.val_
+	to.bit_pos_ = from.bit_pos_
+	to.input = from.input
+	to.input_len = from.input_len
+	to.byte_pos = from.byte_pos
+}
+
+func getAvailableBits(br *bitReader) uint32 {
+	return 64 - br.bit_pos_
+}
+
+/* Returns amount of unread bytes the bit reader still has buffered from the
+   BrotliInput, including whole bytes in br->val_. */
+func getRemainingBytes(br *bitReader) uint {
+	return uint(uint32(br.input_len-br.byte_pos) + (getAvailableBits(br) >> 3))
+}
+
+/* Checks if there is at least |num| bytes left in the input ring-buffer
+   (excluding the bits remaining in br->val_). */
+func checkInputAmount(br *bitReader, num uint) bool {
+	return br.input_len-br.byte_pos >= num
+}
+
+/* Guarantees that there are at least |n_bits| + 1 bits in accumulator.
+   Precondition: accumulator contains at least 1 bit.
+   |n_bits| should be in the range [1..24] for regular build. For portable
+   non-64-bit little-endian build only 16 bits are safe to request. */
+func fillBitWindow(br *bitReader, n_bits uint32) {
+	if br.bit_pos_ >= 32 {
+		br.val_ >>= 32
+		br.bit_pos_ ^= 32 /* here same as -= 32 because of the if condition */
+		br.val_ |= (uint64(binary.LittleEndian.Uint32(br.input[br.byte_pos:]))) << 32
+		br.byte_pos += 4
+	}
+}
+
+/* Mostly like BrotliFillBitWindow, but guarantees only 16 bits and reads no
+   more than BROTLI_SHORT_FILL_BIT_WINDOW_READ bytes of input. */
+func fillBitWindow16(br *bitReader) {
+	fillBitWindow(br, 17)
+}
+
+/* Tries to pull one byte of input to accumulator.
+   Returns false if there is no input available. */
+func pullByte(br *bitReader) bool {
+	if br.byte_pos == br.input_len {
+		return false
+	}
+
+	br.val_ >>= 8
+	br.val_ |= (uint64(br.input[br.byte_pos])) << 56
+	br.bit_pos_ -= 8
+	br.byte_pos++
+	return true
+}
+
+/* Returns currently available bits.
+   The number of valid bits could be calculated by BrotliGetAvailableBits. */
+func getBitsUnmasked(br *bitReader) uint64 {
+	return br.val_ >> br.bit_pos_
+}
+
+/* Like BrotliGetBits, but does not mask the result.
+   The result contains at least 16 valid bits. */
+func get16BitsUnmasked(br *bitReader) uint32 {
+	fillBitWindow(br, 16)
+	return uint32(getBitsUnmasked(br))
+}
+
+/* Returns the specified number of bits from |br| without advancing bit
+   position. */
+func getBits(br *bitReader, n_bits uint32) uint32 {
+	fillBitWindow(br, n_bits)
+	return uint32(getBitsUnmasked(br)) & bitMask(n_bits)
+}
+
+/* Tries to peek the specified amount of bits. Returns false, if there
+   is not enough input. */
+func safeGetBits(br *bitReader, n_bits uint32, val *uint32) bool {
+	for getAvailableBits(br) < n_bits {
+		if !pullByte(br) {
+			return false
+		}
+	}
+
+	*val = uint32(getBitsUnmasked(br)) & bitMask(n_bits)
+	return true
+}
+
+/* Advances the bit pos by |n_bits|. */
+func dropBits(br *bitReader, n_bits uint32) {
+	br.bit_pos_ += n_bits
+}
+
+func bitReaderUnload(br *bitReader) {
+	var unused_bytes uint32 = getAvailableBits(br) >> 3
+	var unused_bits uint32 = unused_bytes << 3
+	br.byte_pos -= uint(unused_bytes)
+	if unused_bits == 64 {
+		br.val_ = 0
+	} else {
+		br.val_ <<= unused_bits
+	}
+
+	br.bit_pos_ += unused_bits
+}
+
+/* Reads the specified number of bits from |br| and advances the bit pos.
+   Precondition: accumulator MUST contain at least |n_bits|. */
+func takeBits(br *bitReader, n_bits uint32, val *uint32) {
+	*val = uint32(getBitsUnmasked(br)) & bitMask(n_bits)
+	dropBits(br, n_bits)
+}
+
+/* Reads the specified number of bits from |br| and advances the bit pos.
+   Assumes that there is enough input to perform BrotliFillBitWindow. */
+func readBits(br *bitReader, n_bits uint32) uint32 {
+	var val uint32
+	fillBitWindow(br, n_bits)
+	takeBits(br, n_bits, &val)
+	return val
+}
+
+/* Tries to read the specified amount of bits. Returns false, if there
+   is not enough input. |n_bits| MUST be positive. */
+func safeReadBits(br *bitReader, n_bits uint32, val *uint32) bool {
+	for getAvailableBits(br) < n_bits {
+		if !pullByte(br) {
+			return false
+		}
+	}
+
+	takeBits(br, n_bits, val)
+	return true
+}
+
+/* Advances the bit reader position to the next byte boundary and verifies
+   that any skipped bits are set to zero. */
+func bitReaderJumpToByteBoundary(br *bitReader) bool {
+	var pad_bits_count uint32 = getAvailableBits(br) & 0x7
+	var pad_bits uint32 = 0
+	if pad_bits_count != 0 {
+		takeBits(br, pad_bits_count, &pad_bits)
+	}
+
+	return pad_bits == 0
+}
+
+/* Copies remaining input bytes stored in the bit reader to the output. Value
+   |num| may not be larger than BrotliGetRemainingBytes. The bit reader must be
+   warmed up again after this. */
+func copyBytes(dest []byte, br *bitReader, num uint) {
+	for getAvailableBits(br) >= 8 && num > 0 {
+		dest[0] = byte(getBitsUnmasked(br))
+		dropBits(br, 8)
+		dest = dest[1:]
+		num--
+	}
+
+	copy(dest, br.input[br.byte_pos:][:num])
+	br.byte_pos += num
+}
+
+func initBitReader(br *bitReader) {
+	br.val_ = 0
+	br.bit_pos_ = 64
+}
+
+func warmupBitReader(br *bitReader) bool {
+	/* Fixing alignment after unaligned BrotliFillWindow would result accumulator
+	   overflow. If unalignment is caused by BrotliSafeReadBits, then there is
+	   enough space in accumulator to fix alignment. */
+	if getAvailableBits(br) == 0 {
+		if !pullByte(br) {
+			return false
+		}
+	}
+
+	return true
+}

vendor/github.com/andybalholm/brotli/block_splitter.go

@@ -0,0 +1,153 @@
+package brotli
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Block split point selection utilities. */
+
+type blockSplit struct {
+	num_types          uint
+	num_blocks         uint
+	types              []byte
+	lengths            []uint32
+	types_alloc_size   uint
+	lengths_alloc_size uint
+}
+
+const (
+	kMaxLiteralHistograms        uint    = 100
+	kMaxCommandHistograms        uint    = 50
+	kLiteralBlockSwitchCost      float64 = 28.1
+	kCommandBlockSwitchCost      float64 = 13.5
+	kDistanceBlockSwitchCost     float64 = 14.6
+	kLiteralStrideLength         uint    = 70
+	kCommandStrideLength         uint    = 40
+	kSymbolsPerLiteralHistogram  uint    = 544
+	kSymbolsPerCommandHistogram  uint    = 530
+	kSymbolsPerDistanceHistogram uint    = 544
+	kMinLengthForBlockSplitting  uint    = 128
+	kIterMulForRefining          uint    = 2
+	kMinItersForRefining         uint    = 100
+)
+
+func countLiterals(cmds []command, num_commands uint) uint {
+	var total_length uint = 0
+	/* Count how many we have. */
+
+	var i uint
+	for i = 0; i < num_commands; i++ {
+		total_length += uint(cmds[i].insert_len_)
+	}
+
+	return total_length
+}
+
+func copyLiteralsToByteArray(cmds []command, num_commands uint, data []byte, offset uint, mask uint, literals []byte) {
+	var pos uint = 0
+	var from_pos uint = offset & mask
+	var i uint
+	for i = 0; i < num_commands; i++ {
+		var insert_len uint = uint(cmds[i].insert_len_)
+		if from_pos+insert_len > mask {
+			var head_size uint = mask + 1 - from_pos
+			copy(literals[pos:], data[from_pos:][:head_size])
+			from_pos = 0
+			pos += head_size
+			insert_len -= head_size
+		}
+
+		if insert_len > 0 {
+			copy(literals[pos:], data[from_pos:][:insert_len])
+			pos += insert_len
+		}
+
+		from_pos = uint((uint32(from_pos+insert_len) + commandCopyLen(&cmds[i])) & uint32(mask))
+	}
+}
+
+func myRand(seed *uint32) uint32 {
+	/* Initial seed should be 7. In this case, loop length is (1 << 29). */
+	*seed *= 16807
+
+	return *seed
+}
+
+func bitCost(count uint) float64 {
+	if count == 0 {
+		return -2.0
+	} else {
+		return fastLog2(count)
+	}
+}
+
+const histogramsPerBatch = 64
+
+const clustersPerBatch = 16
+
+func initBlockSplit(self *blockSplit) {
+	self.num_types = 0
+	self.num_blocks = 0
+	self.types = nil
+	self.lengths = nil
+	self.types_alloc_size = 0
+	self.lengths_alloc_size = 0
+}
+
+func destroyBlockSplit(self *blockSplit) {
+	self.types = nil
+	self.lengths = nil
+}
+
+func splitBlock(cmds []command, num_commands uint, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) {
+	{
+		var literals_count uint = countLiterals(cmds, num_commands)
+		var literals []byte = make([]byte, literals_count)
+
+		/* Create a continuous array of literals. */
+		copyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals)
+
+		/* Create the block split on the array of literals.
+		   Literal histograms have alphabet size 256. */
+		splitByteVectorLiteral(literals, literals_count, kSymbolsPerLiteralHistogram, kMaxLiteralHistograms, kLiteralStrideLength, kLiteralBlockSwitchCost, params, literal_split)
+
+		literals = nil
+	}
+	{
+		var insert_and_copy_codes []uint16 = make([]uint16, num_commands)
+		/* Compute prefix codes for commands. */
+
+		var i uint
+		for i = 0; i < num_commands; i++ {
+			insert_and_copy_codes[i] = cmds[i].cmd_prefix_
+		}
+
+		/* Create the block split on the array of command prefixes. */
+		splitByteVectorCommand(insert_and_copy_codes, num_commands, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split)
+
+		/* TODO: reuse for distances? */
+
+		insert_and_copy_codes = nil
+	}
+	{
+		var distance_prefixes []uint16 = make([]uint16, num_commands)
+		var j uint = 0
+		/* Create a continuous array of distance prefixes. */
+
+		var i uint
+		for i = 0; i < num_commands; i++ {
+			var cmd *command = &cmds[i]
+			if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
+				distance_prefixes[j] = cmd.dist_prefix_ & 0x3FF
+				j++
+			}
+		}
+
+		/* Create the block split on the array of distance prefixes. */
+		splitByteVectorDistance(distance_prefixes, j, kSymbolsPerDistanceHistogram, kMaxCommandHistograms, kCommandStrideLength, kDistanceBlockSwitchCost, params, dist_split)
+
+		distance_prefixes = nil
+	}
+}

vendor/github.com/andybalholm/brotli/block_splitter_command.go

@@ -0,0 +1,433 @@
+package brotli
+
+import "math"
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+func initialEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) {
+	var seed uint32 = 7
+	var block_length uint = length / num_histograms
+	var i uint
+	clearHistogramsCommand(histograms, num_histograms)
+	for i = 0; i < num_histograms; i++ {
+		var pos uint = length * i / num_histograms
+		if i != 0 {
+			pos += uint(myRand(&seed) % uint32(block_length))
+		}
+
+		if pos+stride >= length {
+			pos = length - stride - 1
+		}
+
+		histogramAddVectorCommand(&histograms[i], data[pos:], stride)
+	}
+}
+
+func randomSampleCommand(seed *uint32, data []uint16, length uint, stride uint, sample *histogramCommand) {
+	var pos uint = 0
+	if stride >= length {
+		stride = length
+	} else {
+		pos = uint(myRand(seed) % uint32(length-stride+1))
+	}
+
+	histogramAddVectorCommand(sample, data[pos:], stride)
+}
+
+func refineEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) {
+	var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining
+	var seed uint32 = 7
+	var iter uint
+	iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms
+	for iter = 0; iter < iters; iter++ {
+		var sample histogramCommand
+		histogramClearCommand(&sample)
+		randomSampleCommand(&seed, data, length, stride, &sample)
+		histogramAddHistogramCommand(&histograms[iter%num_histograms], &sample)
+	}
+}
+
+/* Assigns a block id from the range [0, num_histograms) to each data element
+   in data[0..length) and fills in block_id[0..length) with the assigned values.
+   Returns the number of blocks, i.e. one plus the number of block switches. */
+func findBlocksCommand(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramCommand, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint {
+	var data_size uint = histogramDataSizeCommand()
+	var bitmaplen uint = (num_histograms + 7) >> 3
+	var num_blocks uint = 1
+	var i uint
+	var j uint
+	assert(num_histograms <= 256)
+	if num_histograms <= 1 {
+		for i = 0; i < length; i++ {
+			block_id[i] = 0
+		}
+
+		return 1
+	}
+
+	for i := 0; i < int(data_size*num_histograms); i++ {
+		insert_cost[i] = 0
+	}
+	for i = 0; i < num_histograms; i++ {
+		insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_)))
+	}
+
+	for i = data_size; i != 0; {
+		i--
+		for j = 0; j < num_histograms; j++ {
+			insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i]))
+		}
+	}
+
+	for i := 0; i < int(num_histograms); i++ {
+		cost[i] = 0
+	}
+	for i := 0; i < int(length*bitmaplen); i++ {
+		switch_signal[i] = 0
+	}
+
+	/* After each iteration of this loop, cost[k] will contain the difference
+	   between the minimum cost of arriving at the current byte position using
+	   entropy code k, and the minimum cost of arriving at the current byte
+	   position. This difference is capped at the block switch cost, and if it
+	   reaches block switch cost, it means that when we trace back from the last
+	   position, we need to switch here. */
+	for i = 0; i < length; i++ {
+		var byte_ix uint = i
+		var ix uint = byte_ix * bitmaplen
+		var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms
+		var min_cost float64 = 1e99
+		var block_switch_cost float64 = block_switch_bitcost
+		var k uint
+		for k = 0; k < num_histograms; k++ {
+			/* We are coding the symbol in data[byte_ix] with entropy code k. */
+			cost[k] += insert_cost[insert_cost_ix+k]
+
+			if cost[k] < min_cost {
+				min_cost = cost[k]
+				block_id[byte_ix] = byte(k)
+			}
+		}
+
+		/* More blocks for the beginning. */
+		if byte_ix < 2000 {
+			block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000
+		}
+
+		for k = 0; k < num_histograms; k++ {
+			cost[k] -= min_cost
+			if cost[k] >= block_switch_cost {
+				var mask byte = byte(1 << (k & 7))
+				cost[k] = block_switch_cost
+				assert(k>>3 < bitmaplen)
+				switch_signal[ix+(k>>3)] |= mask
+				/* Trace back from the last position and switch at the marked places. */
+			}
+		}
+	}
+	{
+		var byte_ix uint = length - 1
+		var ix uint = byte_ix * bitmaplen
+		var cur_id byte = block_id[byte_ix]
+		for byte_ix > 0 {
+			var mask byte = byte(1 << (cur_id & 7))
+			assert(uint(cur_id)>>3 < bitmaplen)
+			byte_ix--
+			ix -= bitmaplen
+			if switch_signal[ix+uint(cur_id>>3)]&mask != 0 {
+				if cur_id != block_id[byte_ix] {
+					cur_id = block_id[byte_ix]
+					num_blocks++
+				}
+			}
+
+			block_id[byte_ix] = cur_id
+		}
+	}
+
+	return num_blocks
+}
+
+var remapBlockIdsCommand_kInvalidId uint16 = 256
+
+func remapBlockIdsCommand(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint {
+	var next_id uint16 = 0
+	var i uint
+	for i = 0; i < num_histograms; i++ {
+		new_id[i] = remapBlockIdsCommand_kInvalidId
+	}
+
+	for i = 0; i < length; i++ {
+		assert(uint(block_ids[i]) < num_histograms)
+		if new_id[block_ids[i]] == remapBlockIdsCommand_kInvalidId {
+			new_id[block_ids[i]] = next_id
+			next_id++
+		}
+	}
+
+	for i = 0; i < length; i++ {
+		block_ids[i] = byte(new_id[block_ids[i]])
+		assert(uint(block_ids[i]) < num_histograms)
+	}
+
+	assert(uint(next_id) <= num_histograms)
+	return uint(next_id)
+}
+
+func buildBlockHistogramsCommand(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []histogramCommand) {
+	var i uint
+	clearHistogramsCommand(histograms, num_histograms)
+	for i = 0; i < length; i++ {
+		histogramAddCommand(&histograms[block_ids[i]], uint(data[i]))
+	}
+}
+
+var clusterBlocksCommand_kInvalidIndex uint32 = math.MaxUint32
+
+func clusterBlocksCommand(data []uint16, length uint, num_blocks uint, block_ids []byte, split *blockSplit) {
+	var histogram_symbols []uint32 = make([]uint32, num_blocks)
+	var block_lengths []uint32 = make([]uint32, num_blocks)
+	var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch
+	var all_histograms_size uint = 0
+	var all_histograms_capacity uint = expected_num_clusters
+	var all_histograms []histogramCommand = make([]histogramCommand, all_histograms_capacity)
+	var cluster_size_size uint = 0
+	var cluster_size_capacity uint = expected_num_clusters
+	var cluster_size []uint32 = make([]uint32, cluster_size_capacity)
+	var num_clusters uint = 0
+	var histograms []histogramCommand = make([]histogramCommand, brotli_min_size_t(num_blocks, histogramsPerBatch))
+	var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2
+	var pairs_capacity uint = max_num_pairs + 1
+	var pairs []histogramPair = make([]histogramPair, pairs_capacity)
+	var pos uint = 0
+	var clusters []uint32
+	var num_final_clusters uint
+	var new_index []uint32
+	var i uint
+	var sizes = [histogramsPerBatch]uint32{0}
+	var new_clusters = [histogramsPerBatch]uint32{0}
+	var symbols = [histogramsPerBatch]uint32{0}
+	var remap = [histogramsPerBatch]uint32{0}
+
+	for i := 0; i < int(num_blocks); i++ {
+		block_lengths[i] = 0
+	}
+	{
+		var block_idx uint = 0
+		for i = 0; i < length; i++ {
+			assert(block_idx < num_blocks)
+			block_lengths[block_idx]++
+			if i+1 == length || block_ids[i] != block_ids[i+1] {
+				block_idx++
+			}
+		}
+
+		assert(block_idx == num_blocks)
+	}
+
+	for i = 0; i < num_blocks; i += histogramsPerBatch {
+		var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch)
+		var num_new_clusters uint
+		var j uint
+		for j = 0; j < num_to_combine; j++ {
+			var k uint
+			histogramClearCommand(&histograms[j])
+			for k = 0; uint32(k) < block_lengths[i+j]; k++ {
+				histogramAddCommand(&histograms[j], uint(data[pos]))
+				pos++
+			}
+
+			histograms[j].bit_cost_ = populationCostCommand(&histograms[j])
+			new_clusters[j] = uint32(j)
+			symbols[j] = uint32(j)
+			sizes[j] = 1
+		}
+
+		num_new_clusters = histogramCombineCommand(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs)
+		if all_histograms_capacity < (all_histograms_size + num_new_clusters) {
+			var _new_size uint
+			if all_histograms_capacity == 0 {
+				_new_size = all_histograms_size + num_new_clusters
+			} else {
+				_new_size = all_histograms_capacity
+			}
+			var new_array []histogramCommand
+			for _new_size < (all_histograms_size + num_new_clusters) {
+				_new_size *= 2
+			}
+			new_array = make([]histogramCommand, _new_size)
+			if all_histograms_capacity != 0 {
+				copy(new_array, all_histograms[:all_histograms_capacity])
+			}
+
+			all_histograms = new_array
+			all_histograms_capacity = _new_size
+		}
+
+		brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters)
+		for j = 0; j < num_new_clusters; j++ {
+			all_histograms[all_histograms_size] = histograms[new_clusters[j]]
+			all_histograms_size++
+			cluster_size[cluster_size_size] = sizes[new_clusters[j]]
+			cluster_size_size++
+			remap[new_clusters[j]] = uint32(j)
+		}
+
+		for j = 0; j < num_to_combine; j++ {
+			histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]]
+		}
+
+		num_clusters += num_new_clusters
+		assert(num_clusters == cluster_size_size)
+		assert(num_clusters == all_histograms_size)
+	}
+
+	histograms = nil
+
+	max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
+	if pairs_capacity < max_num_pairs+1 {
+		pairs = nil
+		pairs = make([]histogramPair, (max_num_pairs + 1))
+	}
+
+	clusters = make([]uint32, num_clusters)
+	for i = 0; i < num_clusters; i++ {
+		clusters[i] = uint32(i)
+	}
+
+	num_final_clusters = histogramCombineCommand(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs)
+	pairs = nil
+	cluster_size = nil
+
+	new_index = make([]uint32, num_clusters)
+	for i = 0; i < num_clusters; i++ {
+		new_index[i] = clusterBlocksCommand_kInvalidIndex
+	}
+	pos = 0
+	{
+		var next_index uint32 = 0
+		for i = 0; i < num_blocks; i++ {
+			var histo histogramCommand
+			var j uint
+			var best_out uint32
+			var best_bits float64
+			histogramClearCommand(&histo)
+			for j = 0; uint32(j) < block_lengths[i]; j++ {
+				histogramAddCommand(&histo, uint(data[pos]))
+				pos++
+			}
+
+			if i == 0 {
+				best_out = histogram_symbols[0]
+			} else {
+				best_out = histogram_symbols[i-1]
+			}
+			best_bits = histogramBitCostDistanceCommand(&histo, &all_histograms[best_out])
+			for j = 0; j < num_final_clusters; j++ {
+				var cur_bits float64 = histogramBitCostDistanceCommand(&histo, &all_histograms[clusters[j]])
+				if cur_bits < best_bits {
+					best_bits = cur_bits
+					best_out = clusters[j]
+				}
+			}
+
+			histogram_symbols[i] = best_out
+			if new_index[best_out] == clusterBlocksCommand_kInvalidIndex {
+				new_index[best_out] = next_index
+				next_index++
+			}
+		}
+	}
+
+	clusters = nil
+	all_histograms = nil
+	brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks)
+	brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks)
+	{
+		var cur_length uint32 = 0
+		var block_idx uint = 0
+		var max_type byte = 0
+		for i = 0; i < num_blocks; i++ {
+			cur_length += block_lengths[i]
+			if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] {
+				var id byte = byte(new_index[histogram_symbols[i]])
+				split.types[block_idx] = id
+				split.lengths[block_idx] = cur_length
+				max_type = brotli_max_uint8_t(max_type, id)
+				cur_length = 0
+				block_idx++
+			}
+		}
+
+		split.num_blocks = block_idx
+		split.num_types = uint(max_type) + 1
+	}
+
+	new_index = nil
+	block_lengths = nil
+	histogram_symbols = nil
+}
+
+func splitByteVectorCommand(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
+	var data_size uint = histogramDataSizeCommand()
+	var num_histograms uint = length/literals_per_histogram + 1
+	var histograms []histogramCommand
+	if num_histograms > max_histograms {
+		num_histograms = max_histograms
+	}
+
+	if length == 0 {
+		split.num_types = 1
+		return
+	} else if length < kMinLengthForBlockSplitting {
+		brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1)
+		brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1)
+		split.num_types = 1
+		split.types[split.num_blocks] = 0
+		split.lengths[split.num_blocks] = uint32(length)
+		split.num_blocks++
+		return
+	}
+
+	histograms = make([]histogramCommand, num_histograms)
+
+	/* Find good entropy codes. */
+	initialEntropyCodesCommand(data, length, sampling_stride_length, num_histograms, histograms)
+
+	refineEntropyCodesCommand(data, length, sampling_stride_length, num_histograms, histograms)
+	{
+		var block_ids []byte = make([]byte, length)
+		var num_blocks uint = 0
+		var bitmaplen uint = (num_histograms + 7) >> 3
+		var insert_cost []float64 = make([]float64, (data_size * num_histograms))
+		var cost []float64 = make([]float64, num_histograms)
+		var switch_signal []byte = make([]byte, (length * bitmaplen))
+		var new_id []uint16 = make([]uint16, num_histograms)
+		var iters uint
+		if params.quality < hqZopflificationQuality {
+			iters = 3
+		} else {
+			iters = 10
+		}
+		/* Find a good path through literals with the good entropy codes. */
+
+		var i uint
+		for i = 0; i < iters; i++ {
+			num_blocks = findBlocksCommand(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids)
+			num_histograms = remapBlockIdsCommand(block_ids, length, new_id, num_histograms)
+			buildBlockHistogramsCommand(data, length, block_ids, num_histograms, histograms)
+		}
+
+		insert_cost = nil
+		cost = nil
+		switch_signal = nil
+		new_id = nil
+		histograms = nil
+		clusterBlocksCommand(data, length, num_blocks, block_ids, split)
+		block_ids = nil
+	}
+}

vendor/github.com/andybalholm/brotli/block_splitter_distance.go

@@ -0,0 +1,433 @@
+package brotli
+
+import "math"
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+func initialEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) {
+	var seed uint32 = 7
+	var block_length uint = length / num_histograms
+	var i uint
+	clearHistogramsDistance(histograms, num_histograms)
+	for i = 0; i < num_histograms; i++ {
+		var pos uint = length * i / num_histograms
+		if i != 0 {
+			pos += uint(myRand(&seed) % uint32(block_length))
+		}
+
+		if pos+stride >= length {
+			pos = length - stride - 1
+		}
+
+		histogramAddVectorDistance(&histograms[i], data[pos:], stride)
+	}
+}
+
+func randomSampleDistance(seed *uint32, data []uint16, length uint, stride uint, sample *histogramDistance) {
+	var pos uint = 0
+	if stride >= length {
+		stride = length
+	} else {
+		pos = uint(myRand(seed) % uint32(length-stride+1))
+	}
+
+	histogramAddVectorDistance(sample, data[pos:], stride)
+}
+
+func refineEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) {
+	var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining
+	var seed uint32 = 7
+	var iter uint
+	iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms
+	for iter = 0; iter < iters; iter++ {
+		var sample histogramDistance
+		histogramClearDistance(&sample)
+		randomSampleDistance(&seed, data, length, stride, &sample)
+		histogramAddHistogramDistance(&histograms[iter%num_histograms], &sample)
+	}
+}
+
+/* Assigns a block id from the range [0, num_histograms) to each data element
+   in data[0..length) and fills in block_id[0..length) with the assigned values.
+   Returns the number of blocks, i.e. one plus the number of block switches. */
+func findBlocksDistance(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramDistance, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint {
+	var data_size uint = histogramDataSizeDistance()
+	var bitmaplen uint = (num_histograms + 7) >> 3
+	var num_blocks uint = 1
+	var i uint
+	var j uint
+	assert(num_histograms <= 256)
+	if num_histograms <= 1 {
+		for i = 0; i < length; i++ {
+			block_id[i] = 0
+		}
+
+		return 1
+	}
+
+	for i := 0; i < int(data_size*num_histograms); i++ {
+		insert_cost[i] = 0
+	}
+	for i = 0; i < num_histograms; i++ {
+		insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_)))
+	}
+
+	for i = data_size; i != 0; {
+		i--
+		for j = 0; j < num_histograms; j++ {
+			insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i]))
+		}
+	}
+
+	for i := 0; i < int(num_histograms); i++ {
+		cost[i] = 0
+	}
+	for i := 0; i < int(length*bitmaplen); i++ {
+		switch_signal[i] = 0
+	}
+
+	/* After each iteration of this loop, cost[k] will contain the difference
+	   between the minimum cost of arriving at the current byte position using
+	   entropy code k, and the minimum cost of arriving at the current byte
+	   position. This difference is capped at the block switch cost, and if it
+	   reaches block switch cost, it means that when we trace back from the last
+	   position, we need to switch here. */
+	for i = 0; i < length; i++ {
+		var byte_ix uint = i
+		var ix uint = byte_ix * bitmaplen
+		var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms
+		var min_cost float64 = 1e99
+		var block_switch_cost float64 = block_switch_bitcost
+		var k uint
+		for k = 0; k < num_histograms; k++ {
+			/* We are coding the symbol in data[byte_ix] with entropy code k. */
+			cost[k] += insert_cost[insert_cost_ix+k]
+
+			if cost[k] < min_cost {
+				min_cost = cost[k]
+				block_id[byte_ix] = byte(k)
+			}
+		}
+
+		/* More blocks for the beginning. */
+		if byte_ix < 2000 {
+			block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000
+		}
+
+		for k = 0; k < num_histograms; k++ {
+			cost[k] -= min_cost
+			if cost[k] >= block_switch_cost {
+				var mask byte = byte(1 << (k & 7))
+				cost[k] = block_switch_cost
+				assert(k>>3 < bitmaplen)
+				switch_signal[ix+(k>>3)] |= mask
+				/* Trace back from the last position and switch at the marked places. */
+			}
+		}
+	}
+	{
+		var byte_ix uint = length - 1
+		var ix uint = byte_ix * bitmaplen
+		var cur_id byte = block_id[byte_ix]
+		for byte_ix > 0 {
+			var mask byte = byte(1 << (cur_id & 7))
+			assert(uint(cur_id)>>3 < bitmaplen)
+			byte_ix--
+			ix -= bitmaplen
+			if switch_signal[ix+uint(cur_id>>3)]&mask != 0 {
+				if cur_id != block_id[byte_ix] {
+					cur_id = block_id[byte_ix]
+					num_blocks++
+				}
+			}
+
+			block_id[byte_ix] = cur_id
+		}
+	}
+
+	return num_blocks
+}
+
+var remapBlockIdsDistance_kInvalidId uint16 = 256
+
+func remapBlockIdsDistance(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint {
+	var next_id uint16 = 0
+	var i uint
+	for i = 0; i < num_histograms; i++ {
+		new_id[i] = remapBlockIdsDistance_kInvalidId
+	}
+
+	for i = 0; i < length; i++ {
+		assert(uint(block_ids[i]) < num_histograms)
+		if new_id[block_ids[i]] == remapBlockIdsDistance_kInvalidId {
+			new_id[block_ids[i]] = next_id
+			next_id++
+		}
+	}
+
+	for i = 0; i < length; i++ {
+		block_ids[i] = byte(new_id[block_ids[i]])
+		assert(uint(block_ids[i]) < num_histograms)
+	}
+
+	assert(uint(next_id) <= num_histograms)
+	return uint(next_id)
+}
+
+func buildBlockHistogramsDistance(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []histogramDistance) {
+	var i uint
+	clearHistogramsDistance(histograms, num_histograms)
+	for i = 0; i < length; i++ {
+		histogramAddDistance(&histograms[block_ids[i]], uint(data[i]))
+	}
+}
+
+var clusterBlocksDistance_kInvalidIndex uint32 = math.MaxUint32
+
+func clusterBlocksDistance(data []uint16, length uint, num_blocks uint, block_ids []byte, split *blockSplit) {
+	var histogram_symbols []uint32 = make([]uint32, num_blocks)
+	var block_lengths []uint32 = make([]uint32, num_blocks)
+	var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch
+	var all_histograms_size uint = 0
+	var all_histograms_capacity uint = expected_num_clusters
+	var all_histograms []histogramDistance = make([]histogramDistance, all_histograms_capacity)
+	var cluster_size_size uint = 0
+	var cluster_size_capacity uint = expected_num_clusters
+	var cluster_size []uint32 = make([]uint32, cluster_size_capacity)
+	var num_clusters uint = 0
+	var histograms []histogramDistance = make([]histogramDistance, brotli_min_size_t(num_blocks, histogramsPerBatch))
+	var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2
+	var pairs_capacity uint = max_num_pairs + 1
+	var pairs []histogramPair = make([]histogramPair, pairs_capacity)
+	var pos uint = 0
+	var clusters []uint32
+	var num_final_clusters uint
+	var new_index []uint32
+	var i uint
+	var sizes = [histogramsPerBatch]uint32{0}
+	var new_clusters = [histogramsPerBatch]uint32{0}
+	var symbols = [histogramsPerBatch]uint32{0}
+	var remap = [histogramsPerBatch]uint32{0}
+
+	for i := 0; i < int(num_blocks); i++ {
+		block_lengths[i] = 0
+	}
+	{
+		var block_idx uint = 0
+		for i = 0; i < length; i++ {
+			assert(block_idx < num_blocks)
+			block_lengths[block_idx]++
+			if i+1 == length || block_ids[i] != block_ids[i+1] {
+				block_idx++
+			}
+		}
+
+		assert(block_idx == num_blocks)
+	}
+
+	for i = 0; i < num_blocks; i += histogramsPerBatch {
+		var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch)
+		var num_new_clusters uint
+		var j uint
+		for j = 0; j < num_to_combine; j++ {
+			var k uint
+			histogramClearDistance(&histograms[j])
+			for k = 0; uint32(k) < block_lengths[i+j]; k++ {
+				histogramAddDistance(&histograms[j], uint(data[pos]))
+				pos++
+			}
+
+			histograms[j].bit_cost_ = populationCostDistance(&histograms[j])
+			new_clusters[j] = uint32(j)
+			symbols[j] = uint32(j)
+			sizes[j] = 1
+		}
+
+		num_new_clusters = histogramCombineDistance(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs)
+		if all_histograms_capacity < (all_histograms_size + num_new_clusters) {
+			var _new_size uint
+			if all_histograms_capacity == 0 {
+				_new_size = all_histograms_size + num_new_clusters
+			} else {
+				_new_size = all_histograms_capacity
+			}
+			var new_array []histogramDistance
+			for _new_size < (all_histograms_size + num_new_clusters) {
+				_new_size *= 2
+			}
+			new_array = make([]histogramDistance, _new_size)
+			if all_histograms_capacity != 0 {
+				copy(new_array, all_histograms[:all_histograms_capacity])
+			}
+
+			all_histograms = new_array
+			all_histograms_capacity = _new_size
+		}
+
+		brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters)
+		for j = 0; j < num_new_clusters; j++ {
+			all_histograms[all_histograms_size] = histograms[new_clusters[j]]
+			all_histograms_size++
+			cluster_size[cluster_size_size] = sizes[new_clusters[j]]
+			cluster_size_size++
+			remap[new_clusters[j]] = uint32(j)
+		}
+
+		for j = 0; j < num_to_combine; j++ {
+			histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]]
+		}
+
+		num_clusters += num_new_clusters
+		assert(num_clusters == cluster_size_size)
+		assert(num_clusters == all_histograms_size)
+	}
+
+	histograms = nil
+
+	max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
+	if pairs_capacity < max_num_pairs+1 {
+		pairs = nil
+		pairs = make([]histogramPair, (max_num_pairs + 1))
+	}
+
+	clusters = make([]uint32, num_clusters)
+	for i = 0; i < num_clusters; i++ {
+		clusters[i] = uint32(i)
+	}
+
+	num_final_clusters = histogramCombineDistance(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs)
+	pairs = nil
+	cluster_size = nil
+
+	new_index = make([]uint32, num_clusters)
+	for i = 0; i < num_clusters; i++ {
+		new_index[i] = clusterBlocksDistance_kInvalidIndex
+	}
+	pos = 0
+	{
+		var next_index uint32 = 0
+		for i = 0; i < num_blocks; i++ {
+			var histo histogramDistance
+			var j uint
+			var best_out uint32
+			var best_bits float64
+			histogramClearDistance(&histo)
+			for j = 0; uint32(j) < block_lengths[i]; j++ {
+				histogramAddDistance(&histo, uint(data[pos]))
+				pos++
+			}
+
+			if i == 0 {
+				best_out = histogram_symbols[0]
+			} else {
+				best_out = histogram_symbols[i-1]
+			}
+			best_bits = histogramBitCostDistanceDistance(&histo, &all_histograms[best_out])
+			for j = 0; j < num_final_clusters; j++ {
+				var cur_bits float64 = histogramBitCostDistanceDistance(&histo, &all_histograms[clusters[j]])
+				if cur_bits < best_bits {
+					best_bits = cur_bits
+					best_out = clusters[j]
+				}
+			}
+
+			histogram_symbols[i] = best_out
+			if new_index[best_out] == clusterBlocksDistance_kInvalidIndex {
+				new_index[best_out] = next_index
+				next_index++
+			}
+		}
+	}
+
+	clusters = nil
+	all_histograms = nil
+	brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks)
+	brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks)
+	{
+		var cur_length uint32 = 0
+		var block_idx uint = 0
+		var max_type byte = 0
+		for i = 0; i < num_blocks; i++ {
+			cur_length += block_lengths[i]
+			if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] {
+				var id byte = byte(new_index[histogram_symbols[i]])
+				split.types[block_idx] = id
+				split.lengths[block_idx] = cur_length
+				max_type = brotli_max_uint8_t(max_type, id)
+				cur_length = 0
+				block_idx++
+			}
+		}
+
+		split.num_blocks = block_idx
+		split.num_types = uint(max_type) + 1
+	}
+
+	new_index = nil
+	block_lengths = nil
+	histogram_symbols = nil
+}
+
+func splitByteVectorDistance(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
+	var data_size uint = histogramDataSizeDistance()
+	var num_histograms uint = length/literals_per_histogram + 1
+	var histograms []histogramDistance
+	if num_histograms > max_histograms {
+		num_histograms = max_histograms
+	}
+
+	if length == 0 {
+		split.num_types = 1
+		return
+	} else if length < kMinLengthForBlockSplitting {
+		brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1)
+		brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1)
+		split.num_types = 1
+		split.types[split.num_blocks] = 0
+		split.lengths[split.num_blocks] = uint32(length)
+		split.num_blocks++
+		return
+	}
+
+	histograms = make([]histogramDistance, num_histograms)
+
+	/* Find good entropy codes. */
+	initialEntropyCodesDistance(data, length, sampling_stride_length, num_histograms, histograms)
+
+	refineEntropyCodesDistance(data, length, sampling_stride_length, num_histograms, histograms)
+	{
+		var block_ids []byte = make([]byte, length)
+		var num_blocks uint = 0
+		var bitmaplen uint = (num_histograms + 7) >> 3
+		var insert_cost []float64 = make([]float64, (data_size * num_histograms))
+		var cost []float64 = make([]float64, num_histograms)
+		var switch_signal []byte = make([]byte, (length * bitmaplen))
+		var new_id []uint16 = make([]uint16, num_histograms)
+		var iters uint
+		if params.quality < hqZopflificationQuality {
+			iters = 3
+		} else {
+			iters = 10
+		}
+		/* Find a good path through literals with the good entropy codes. */
+
+		var i uint
+		for i = 0; i < iters; i++ {
+			num_blocks = findBlocksDistance(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids)
+			num_histograms = remapBlockIdsDistance(block_ids, length, new_id, num_histograms)
+			buildBlockHistogramsDistance(data, length, block_ids, num_histograms, histograms)
+		}
+
+		insert_cost = nil
+		cost = nil
+		switch_signal = nil
+		new_id = nil
+		histograms = nil
+		clusterBlocksDistance(data, length, num_blocks, block_ids, split)
+		block_ids = nil
+	}
+}

vendor/github.com/andybalholm/brotli/block_splitter_literal.go

@@ -0,0 +1,433 @@
+package brotli
+
+import "math"
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+func initialEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) {
+	var seed uint32 = 7
+	var block_length uint = length / num_histograms
+	var i uint
+	clearHistogramsLiteral(histograms, num_histograms)
+	for i = 0; i < num_histograms; i++ {
+		var pos uint = length * i / num_histograms
+		if i != 0 {
+			pos += uint(myRand(&seed) % uint32(block_length))
+		}
+
+		if pos+stride >= length {
+			pos = length - stride - 1
+		}
+
+		histogramAddVectorLiteral(&histograms[i], data[pos:], stride)
+	}
+}
+
+func randomSampleLiteral(seed *uint32, data []byte, length uint, stride uint, sample *histogramLiteral) {
+	var pos uint = 0
+	if stride >= length {
+		stride = length
+	} else {
+		pos = uint(myRand(seed) % uint32(length-stride+1))
+	}
+
+	histogramAddVectorLiteral(sample, data[pos:], stride)
+}
+
+func refineEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) {
+	var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining
+	var seed uint32 = 7
+	var iter uint
+	iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms
+	for iter = 0; iter < iters; iter++ {
+		var sample histogramLiteral
+		histogramClearLiteral(&sample)
+		randomSampleLiteral(&seed, data, length, stride, &sample)
+		histogramAddHistogramLiteral(&histograms[iter%num_histograms], &sample)
+	}
+}
+
+/* Assigns a block id from the range [0, num_histograms) to each data element
+   in data[0..length) and fills in block_id[0..length) with the assigned values.
+   Returns the number of blocks, i.e. one plus the number of block switches. */
+func findBlocksLiteral(data []byte, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramLiteral, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint {
+	var data_size uint = histogramDataSizeLiteral()
+	var bitmaplen uint = (num_histograms + 7) >> 3
+	var num_blocks uint = 1
+	var i uint
+	var j uint
+	assert(num_histograms <= 256)
+	if num_histograms <= 1 {
+		for i = 0; i < length; i++ {
+			block_id[i] = 0
+		}
+
+		return 1
+	}
+
+	for i := 0; i < int(data_size*num_histograms); i++ {
+		insert_cost[i] = 0
+	}
+	for i = 0; i < num_histograms; i++ {
+		insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_)))
+	}
+
+	for i = data_size; i != 0; {
+		i--
+		for j = 0; j < num_histograms; j++ {
+			insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i]))
+		}
+	}
+
+	for i := 0; i < int(num_histograms); i++ {
+		cost[i] = 0
+	}
+	for i := 0; i < int(length*bitmaplen); i++ {
+		switch_signal[i] = 0
+	}
+
+	/* After each iteration of this loop, cost[k] will contain the difference
+	   between the minimum cost of arriving at the current byte position using
+	   entropy code k, and the minimum cost of arriving at the current byte
+	   position. This difference is capped at the block switch cost, and if it
+	   reaches block switch cost, it means that when we trace back from the last
+	   position, we need to switch here. */
+	for i = 0; i < length; i++ {
+		var byte_ix uint = i
+		var ix uint = byte_ix * bitmaplen
+		var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms
+		var min_cost float64 = 1e99
+		var block_switch_cost float64 = block_switch_bitcost
+		var k uint
+		for k = 0; k < num_histograms; k++ {
+			/* We are coding the symbol in data[byte_ix] with entropy code k. */
+			cost[k] += insert_cost[insert_cost_ix+k]
+
+			if cost[k] < min_cost {
+				min_cost = cost[k]
+				block_id[byte_ix] = byte(k)
+			}
+		}
+
+		/* More blocks for the beginning. */
+		if byte_ix < 2000 {
+			block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000
+		}
+
+		for k = 0; k < num_histograms; k++ {
+			cost[k] -= min_cost
+			if cost[k] >= block_switch_cost {
+				var mask byte = byte(1 << (k & 7))
+				cost[k] = block_switch_cost
+				assert(k>>3 < bitmaplen)
+				switch_signal[ix+(k>>3)] |= mask
+				/* Trace back from the last position and switch at the marked places. */
+			}
+		}
+	}
+	{
+		var byte_ix uint = length - 1
+		var ix uint = byte_ix * bitmaplen
+		var cur_id byte = block_id[byte_ix]
+		for byte_ix > 0 {
+			var mask byte = byte(1 << (cur_id & 7))
+			assert(uint(cur_id)>>3 < bitmaplen)
+			byte_ix--
+			ix -= bitmaplen
+			if switch_signal[ix+uint(cur_id>>3)]&mask != 0 {
+				if cur_id != block_id[byte_ix] {
+					cur_id = block_id[byte_ix]
+					num_blocks++
+				}
+			}
+
+			block_id[byte_ix] = cur_id
+		}
+	}
+
+	return num_blocks
+}
+
+var remapBlockIdsLiteral_kInvalidId uint16 = 256
+
+func remapBlockIdsLiteral(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint {
+	var next_id uint16 = 0
+	var i uint
+	for i = 0; i < num_histograms; i++ {
+		new_id[i] = remapBlockIdsLiteral_kInvalidId
+	}
+
+	for i = 0; i < length; i++ {
+		assert(uint(block_ids[i]) < num_histograms)
+		if new_id[block_ids[i]] == remapBlockIdsLiteral_kInvalidId {
+			new_id[block_ids[i]] = next_id
+			next_id++
+		}
+	}
+
+	for i = 0; i < length; i++ {
+		block_ids[i] = byte(new_id[block_ids[i]])
+		assert(uint(block_ids[i]) < num_histograms)
+	}
+
+	assert(uint(next_id) <= num_histograms)
+	return uint(next_id)
+}
+
+func buildBlockHistogramsLiteral(data []byte, length uint, block_ids []byte, num_histograms uint, histograms []histogramLiteral) {
+	var i uint
+	clearHistogramsLiteral(histograms, num_histograms)
+	for i = 0; i < length; i++ {
+		histogramAddLiteral(&histograms[block_ids[i]], uint(data[i]))
+	}
+}
+
+var clusterBlocksLiteral_kInvalidIndex uint32 = math.MaxUint32
+
+func clusterBlocksLiteral(data []byte, length uint, num_blocks uint, block_ids []byte, split *blockSplit) {
+	var histogram_symbols []uint32 = make([]uint32, num_blocks)
+	var block_lengths []uint32 = make([]uint32, num_blocks)
+	var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch
+	var all_histograms_size uint = 0
+	var all_histograms_capacity uint = expected_num_clusters
+	var all_histograms []histogramLiteral = make([]histogramLiteral, all_histograms_capacity)
+	var cluster_size_size uint = 0
+	var cluster_size_capacity uint = expected_num_clusters
+	var cluster_size []uint32 = make([]uint32, cluster_size_capacity)
+	var num_clusters uint = 0
+	var histograms []histogramLiteral = make([]histogramLiteral, brotli_min_size_t(num_blocks, histogramsPerBatch))
+	var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2
+	var pairs_capacity uint = max_num_pairs + 1
+	var pairs []histogramPair = make([]histogramPair, pairs_capacity)
+	var pos uint = 0
+	var clusters []uint32
+	var num_final_clusters uint
+	var new_index []uint32
+	var i uint
+	var sizes = [histogramsPerBatch]uint32{0}
+	var new_clusters = [histogramsPerBatch]uint32{0}
+	var symbols = [histogramsPerBatch]uint32{0}
+	var remap = [histogramsPerBatch]uint32{0}
+
+	for i := 0; i < int(num_blocks); i++ {
+		block_lengths[i] = 0
+	}
+	{
+		var block_idx uint = 0
+		for i = 0; i < length; i++ {
+			assert(block_idx < num_blocks)
+			block_lengths[block_idx]++
+			if i+1 == length || block_ids[i] != block_ids[i+1] {
+				block_idx++
+			}
+		}
+
+		assert(block_idx == num_blocks)
+	}
+
+	for i = 0; i < num_blocks; i += histogramsPerBatch {
+		var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch)
+		var num_new_clusters uint
+		var j uint
+		for j = 0; j < num_to_combine; j++ {
+			var k uint
+			histogramClearLiteral(&histograms[j])
+			for k = 0; uint32(k) < block_lengths[i+j]; k++ {
+				histogramAddLiteral(&histograms[j], uint(data[pos]))
+				pos++
+			}
+
+			histograms[j].bit_cost_ = populationCostLiteral(&histograms[j])
+			new_clusters[j] = uint32(j)
+			symbols[j] = uint32(j)
+			sizes[j] = 1
+		}
+
+		num_new_clusters = histogramCombineLiteral(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs)
+		if all_histograms_capacity < (all_histograms_size + num_new_clusters) {
+			var _new_size uint
+			if all_histograms_capacity == 0 {
+				_new_size = all_histograms_size + num_new_clusters
+			} else {
+				_new_size = all_histograms_capacity
+			}
+			var new_array []histogramLiteral
+			for _new_size < (all_histograms_size + num_new_clusters) {
+				_new_size *= 2
+			}
+			new_array = make([]histogramLiteral, _new_size)
+			if all_histograms_capacity != 0 {
+				copy(new_array, all_histograms[:all_histograms_capacity])
+			}
+
+			all_histograms = new_array
+			all_histograms_capacity = _new_size
+		}
+
+		brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters)
+		for j = 0; j < num_new_clusters; j++ {
+			all_histograms[all_histograms_size] = histograms[new_clusters[j]]
+			all_histograms_size++
+			cluster_size[cluster_size_size] = sizes[new_clusters[j]]
+			cluster_size_size++
+			remap[new_clusters[j]] = uint32(j)
+		}
+
+		for j = 0; j < num_to_combine; j++ {
+			histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]]
+		}
+
+		num_clusters += num_new_clusters
+		assert(num_clusters == cluster_size_size)
+		assert(num_clusters == all_histograms_size)
+	}
+
+	histograms = nil
+
+	max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
+	if pairs_capacity < max_num_pairs+1 {
+		pairs = nil
+		pairs = make([]histogramPair, (max_num_pairs + 1))
+	}
+
+	clusters = make([]uint32, num_clusters)
+	for i = 0; i < num_clusters; i++ {
+		clusters[i] = uint32(i)
+	}
+
+	num_final_clusters = histogramCombineLiteral(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs)
+	pairs = nil
+	cluster_size = nil
+
+	new_index = make([]uint32, num_clusters)
+	for i = 0; i < num_clusters; i++ {
+		new_index[i] = clusterBlocksLiteral_kInvalidIndex
+	}
+	pos = 0
+	{
+		var next_index uint32 = 0
+		for i = 0; i < num_blocks; i++ {
+			var histo histogramLiteral
+			var j uint
+			var best_out uint32
+			var best_bits float64
+			histogramClearLiteral(&histo)
+			for j = 0; uint32(j) < block_lengths[i]; j++ {
+				histogramAddLiteral(&histo, uint(data[pos]))
+				pos++
+			}
+
+			if i == 0 {
+				best_out = histogram_symbols[0]
+			} else {
+				best_out = histogram_symbols[i-1]
+			}
+			best_bits = histogramBitCostDistanceLiteral(&histo, &all_histograms[best_out])
+			for j = 0; j < num_final_clusters; j++ {
+				var cur_bits float64 = histogramBitCostDistanceLiteral(&histo, &all_histograms[clusters[j]])
+				if cur_bits < best_bits {
+					best_bits = cur_bits
+					best_out = clusters[j]
+				}
+			}
+
+			histogram_symbols[i] = best_out
+			if new_index[best_out] == clusterBlocksLiteral_kInvalidIndex {
+				new_index[best_out] = next_index
+				next_index++
+			}
+		}
+	}
+
+	clusters = nil
+	all_histograms = nil
+	brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks)
+	brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks)
+	{
+		var cur_length uint32 = 0
+		var block_idx uint = 0
+		var max_type byte = 0
+		for i = 0; i < num_blocks; i++ {
+			cur_length += block_lengths[i]
+			if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] {
+				var id byte = byte(new_index[histogram_symbols[i]])
+				split.types[block_idx] = id
+				split.lengths[block_idx] = cur_length
+				max_type = brotli_max_uint8_t(max_type, id)
+				cur_length = 0
+				block_idx++
+			}
+		}
+
+		split.num_blocks = block_idx
+		split.num_types = uint(max_type) + 1
+	}
+
+	new_index = nil
+	block_lengths = nil
+	histogram_symbols = nil
+}
+
+func splitByteVectorLiteral(data []byte, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
+	var data_size uint = histogramDataSizeLiteral()
+	var num_histograms uint = length/literals_per_histogram + 1
+	var histograms []histogramLiteral
+	if num_histograms > max_histograms {
+		num_histograms = max_histograms
+	}
+
+	if length == 0 {
+		split.num_types = 1
+		return
+	} else if length < kMinLengthForBlockSplitting {
+		brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1)
+		brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1)
+		split.num_types = 1
+		split.types[split.num_blocks] = 0
+		split.lengths[split.num_blocks] = uint32(length)
+		split.num_blocks++
+		return
+	}
+
+	histograms = make([]histogramLiteral, num_histograms)
+
+	/* Find good entropy codes. */
+	initialEntropyCodesLiteral(data, length, sampling_stride_length, num_histograms, histograms)
+
+	refineEntropyCodesLiteral(data, length, sampling_stride_length, num_histograms, histograms)
+	{
+		var block_ids []byte = make([]byte, length)
+		var num_blocks uint = 0
+		var bitmaplen uint = (num_histograms + 7) >> 3
+		var insert_cost []float64 = make([]float64, (data_size * num_histograms))
+		var cost []float64 = make([]float64, num_histograms)
+		var switch_signal []byte = make([]byte, (length * bitmaplen))
+		var new_id []uint16 = make([]uint16, num_histograms)
+		var iters uint
+		if params.quality < hqZopflificationQuality {
+			iters = 3
+		} else {
+			iters = 10
+		}
+		/* Find a good path through literals with the good entropy codes. */
+
+		var i uint
+		for i = 0; i < iters; i++ {
+			num_blocks = findBlocksLiteral(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids)
+			num_histograms = remapBlockIdsLiteral(block_ids, length, new_id, num_histograms)
+			buildBlockHistogramsLiteral(data, length, block_ids, num_histograms, histograms)
+		}
+
+		insert_cost = nil
+		cost = nil
+		switch_signal = nil
+		new_id = nil
+		histograms = nil
+		clusterBlocksLiteral(data, length, num_blocks, block_ids, split)
+		block_ids = nil
+	}
+}

vendor/github.com/andybalholm/brotli/cluster.go

@@ -0,0 +1,30 @@
+package brotli
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Functions for clustering similar histograms together. */
+
+type histogramPair struct {
+	idx1       uint32
+	idx2       uint32
+	cost_combo float64
+	cost_diff  float64
+}
+
+func histogramPairIsLess(p1 *histogramPair, p2 *histogramPair) bool {
+	if p1.cost_diff != p2.cost_diff {
+		return p1.cost_diff > p2.cost_diff
+	}
+
+	return (p1.idx2 - p1.idx1) > (p2.idx2 - p2.idx1)
+}
+
+/* Returns entropy reduction of the context map when we combine two clusters. */
+func clusterCostDiff(size_a uint, size_b uint) float64 {
+	var size_c uint = size_a + size_b
+	return float64(size_a)*fastLog2(size_a) + float64(size_b)*fastLog2(size_b) - float64(size_c)*fastLog2(size_c)
+}

vendor/github.com/andybalholm/brotli/cluster_command.go

@@ -0,0 +1,326 @@
+package brotli
+
+import "math"
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if
+   it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */
+func compareAndPushToQueueCommand(out []histogramCommand, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) {
+	var is_good_pair bool = false
+	var p histogramPair
+	p.idx2 = 0
+	p.idx1 = p.idx2
+	p.cost_combo = 0
+	p.cost_diff = p.cost_combo
+	if idx1 == idx2 {
+		return
+	}
+
+	if idx2 < idx1 {
+		var t uint32 = idx2
+		idx2 = idx1
+		idx1 = t
+	}
+
+	p.idx1 = idx1
+	p.idx2 = idx2
+	p.cost_diff = 0.5 * clusterCostDiff(uint(cluster_size[idx1]), uint(cluster_size[idx2]))
+	p.cost_diff -= out[idx1].bit_cost_
+	p.cost_diff -= out[idx2].bit_cost_
+
+	if out[idx1].total_count_ == 0 {
+		p.cost_combo = out[idx2].bit_cost_
+		is_good_pair = true
+	} else if out[idx2].total_count_ == 0 {
+		p.cost_combo = out[idx1].bit_cost_
+		is_good_pair = true
+	} else {
+		var threshold float64
+		if *num_pairs == 0 {
+			threshold = 1e99
+		} else {
+			threshold = brotli_max_double(0.0, pairs[0].cost_diff)
+		}
+		var combo histogramCommand = out[idx1]
+		var cost_combo float64
+		histogramAddHistogramCommand(&combo, &out[idx2])
+		cost_combo = populationCostCommand(&combo)
+		if cost_combo < threshold-p.cost_diff {
+			p.cost_combo = cost_combo
+			is_good_pair = true
+		}
+	}
+
+	if is_good_pair {
+		p.cost_diff += p.cost_combo
+		if *num_pairs > 0 && histogramPairIsLess(&pairs[0], &p) {
+			/* Replace the top of the queue if needed. */
+			if *num_pairs < max_num_pairs {
+				pairs[*num_pairs] = pairs[0]
+				(*num_pairs)++
+			}
+
+			pairs[0] = p
+		} else if *num_pairs < max_num_pairs {
+			pairs[*num_pairs] = p
+			(*num_pairs)++
+		}
+	}
+}
+
+func histogramCombineCommand(out []histogramCommand, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint {
+	var cost_diff_threshold float64 = 0.0
+	var min_cluster_size uint = 1
+	var num_pairs uint = 0
+	{
+		/* We maintain a vector of histogram pairs, with the property that the pair
+		   with the maximum bit cost reduction is the first. */
+		var idx1 uint
+		for idx1 = 0; idx1 < num_clusters; idx1++ {
+			var idx2 uint
+			for idx2 = idx1 + 1; idx2 < num_clusters; idx2++ {
+				compareAndPushToQueueCommand(out, cluster_size, clusters[idx1], clusters[idx2], max_num_pairs, pairs[0:], &num_pairs)
+			}
+		}
+	}
+
+	for num_clusters > min_cluster_size {
+		var best_idx1 uint32
+		var best_idx2 uint32
+		var i uint
+		if pairs[0].cost_diff >= cost_diff_threshold {
+			cost_diff_threshold = 1e99
+			min_cluster_size = max_clusters
+			continue
+		}
+
+		/* Take the best pair from the top of heap. */
+		best_idx1 = pairs[0].idx1
+
+		best_idx2 = pairs[0].idx2
+		histogramAddHistogramCommand(&out[best_idx1], &out[best_idx2])
+		out[best_idx1].bit_cost_ = pairs[0].cost_combo
+		cluster_size[best_idx1] += cluster_size[best_idx2]
+		for i = 0; i < symbols_size; i++ {
+			if symbols[i] == best_idx2 {
+				symbols[i] = best_idx1
+			}
+		}
+
+		for i = 0; i < num_clusters; i++ {
+			if clusters[i] == best_idx2 {
+				copy(clusters[i:], clusters[i+1:][:num_clusters-i-1])
+				break
+			}
+		}
+
+		num_clusters--
+		{
+			/* Remove pairs intersecting the just combined best pair. */
+			var copy_to_idx uint = 0
+			for i = 0; i < num_pairs; i++ {
+				var p *histogramPair = &pairs[i]
+				if p.idx1 == best_idx1 || p.idx2 == best_idx1 || p.idx1 == best_idx2 || p.idx2 == best_idx2 {
+					/* Remove invalid pair from the queue. */
+					continue
+				}
+
+				if histogramPairIsLess(&pairs[0], p) {
+					/* Replace the top of the queue if needed. */
+					var front histogramPair = pairs[0]
+					pairs[0] = *p
+					pairs[copy_to_idx] = front
+				} else {
+					pairs[copy_to_idx] = *p
+				}
+
+				copy_to_idx++
+			}
+
+			num_pairs = copy_to_idx
+		}
+
+		/* Push new pairs formed with the combined histogram to the heap. */
+		for i = 0; i < num_clusters; i++ {
+			compareAndPushToQueueCommand(out, cluster_size, best_idx1, clusters[i], max_num_pairs, pairs[0:], &num_pairs)
+		}
+	}
+
+	return num_clusters
+}
+
+/* What is the bit cost of moving histogram from cur_symbol to candidate. */
+func histogramBitCostDistanceCommand(histogram *histogramCommand, candidate *histogramCommand) float64 {
+	if histogram.total_count_ == 0 {
+		return 0.0
+	} else {
+		var tmp histogramCommand = *histogram
+		histogramAddHistogramCommand(&tmp, candidate)
+		return populationCostCommand(&tmp) - candidate.bit_cost_
+	}
+}
+
+/* Find the best 'out' histogram for each of the 'in' histograms.
+   When called, clusters[0..num_clusters) contains the unique values from
+   symbols[0..in_size), but this property is not preserved in this function.
+   Note: we assume that out[]->bit_cost_ is already up-to-date. */
+func histogramRemapCommand(in []histogramCommand, in_size uint, clusters []uint32, num_clusters uint, out []histogramCommand, symbols []uint32) {
+	var i uint
+	for i = 0; i < in_size; i++ {
+		var best_out uint32
+		if i == 0 {
+			best_out = symbols[0]
+		} else {
+			best_out = symbols[i-1]
+		}
+		var best_bits float64 = histogramBitCostDistanceCommand(&in[i], &out[best_out])
+		var j uint
+		for j = 0; j < num_clusters; j++ {
+			var cur_bits float64 = histogramBitCostDistanceCommand(&in[i], &out[clusters[j]])
+			if cur_bits < best_bits {
+				best_bits = cur_bits
+				best_out = clusters[j]
+			}
+		}
+
+		symbols[i] = best_out
+	}
+
+	/* Recompute each out based on raw and symbols. */
+	for i = 0; i < num_clusters; i++ {
+		histogramClearCommand(&out[clusters[i]])
+	}
+
+	for i = 0; i < in_size; i++ {
+		histogramAddHistogramCommand(&out[symbols[i]], &in[i])
+	}
+}
+
+/* Reorders elements of the out[0..length) array and changes values in
+   symbols[0..length) array in the following way:
+     * when called, symbols[] contains indexes into out[], and has N unique
+       values (possibly N < length)
+     * on return, symbols'[i] = f(symbols[i]) and
+                  out'[symbols'[i]] = out[symbols[i]], for each 0 <= i < length,
+       where f is a bijection between the range of symbols[] and [0..N), and
+       the first occurrences of values in symbols'[i] come in consecutive
+       increasing order.
+   Returns N, the number of unique values in symbols[]. */
+
+var histogramReindexCommand_kInvalidIndex uint32 = math.MaxUint32
+
+func histogramReindexCommand(out []histogramCommand, symbols []uint32, length uint) uint {
+	var new_index []uint32 = make([]uint32, length)
+	var next_index uint32
+	var tmp []histogramCommand
+	var i uint
+	for i = 0; i < length; i++ {
+		new_index[i] = histogramReindexCommand_kInvalidIndex
+	}
+
+	next_index = 0
+	for i = 0; i < length; i++ {
+		if new_index[symbols[i]] == histogramReindexCommand_kInvalidIndex {
+			new_index[symbols[i]] = next_index
+			next_index++
+		}
+	}
+
+	/* TODO: by using idea of "cycle-sort" we can avoid allocation of
+	   tmp and reduce the number of copying by the factor of 2. */
+	tmp = make([]histogramCommand, next_index)
+
+	next_index = 0
+	for i = 0; i < length; i++ {
+		if new_index[symbols[i]] == next_index {
+			tmp[next_index] = out[symbols[i]]
+			next_index++
+		}
+
+		symbols[i] = new_index[symbols[i]]
+	}
+
+	new_index = nil
+	for i = 0; uint32(i) < next_index; i++ {
+		out[i] = tmp[i]
+	}
+
+	tmp = nil
+	return uint(next_index)
+}
+
+func clusterHistogramsCommand(in []histogramCommand, in_size uint, max_histograms uint, out []histogramCommand, out_size *uint, histogram_symbols []uint32) {
+	var cluster_size []uint32 = make([]uint32, in_size)
+	var clusters []uint32 = make([]uint32, in_size)
+	var num_clusters uint = 0
+	var max_input_histograms uint = 64
+	var pairs_capacity uint = max_input_histograms * max_input_histograms / 2
+	var pairs []histogramPair = make([]histogramPair, (pairs_capacity + 1))
+	var i uint
+
+	/* For the first pass of clustering, we allow all pairs. */
+	for i = 0; i < in_size; i++ {
+		cluster_size[i] = 1
+	}
+
+	for i = 0; i < in_size; i++ {
+		out[i] = in[i]
+		out[i].bit_cost_ = populationCostCommand(&in[i])
+		histogram_symbols[i] = uint32(i)
+	}
+
+	for i = 0; i < in_size; i += max_input_histograms {
+		var num_to_combine uint = brotli_min_size_t(in_size-i, max_input_histograms)
+		var num_new_clusters uint
+		var j uint
+		for j = 0; j < num_to_combine; j++ {
+			clusters[num_clusters+j] = uint32(i + j)
+		}
+
+		num_new_clusters = histogramCombineCommand(out, cluster_size, histogram_symbols[i:], clusters[num_clusters:], pairs, num_to_combine, num_to_combine, max_histograms, pairs_capacity)
+		num_clusters += num_new_clusters
+	}
+	{
+		/* For the second pass, we limit the total number of histogram pairs.
+		   After this limit is reached, we only keep searching for the best pair. */
+		var max_num_pairs uint = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
+		if pairs_capacity < (max_num_pairs + 1) {
+			var _new_size uint
+			if pairs_capacity == 0 {
+				_new_size = max_num_pairs + 1
+			} else {
+				_new_size = pairs_capacity
+			}
+			var new_array []histogramPair
+			for _new_size < (max_num_pairs + 1) {
+				_new_size *= 2
+			}
+			new_array = make([]histogramPair, _new_size)
+			if pairs_capacity != 0 {
+				copy(new_array, pairs[:pairs_capacity])
+			}
+
+			pairs = new_array
+			pairs_capacity = _new_size
+		}
+
+		/* Collapse similar histograms. */
+		num_clusters = histogramCombineCommand(out, cluster_size, histogram_symbols, clusters, pairs, num_clusters, in_size, max_histograms, max_num_pairs)
+	}
+
+	pairs = nil
+	cluster_size = nil
+
+	/* Find the optimal map from original histograms to the final ones. */
+	histogramRemapCommand(in, in_size, clusters, num_clusters, out, histogram_symbols)
+
+	clusters = nil
+
+	/* Convert the context map to a canonical form. */
+	*out_size = histogramReindexCommand(out, histogram_symbols, in_size)
+}

vendor/github.com/andybalholm/brotli/cluster_distance.go

@@ -0,0 +1,326 @@
+package brotli
+
+import "math"
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if
+   it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */
+func compareAndPushToQueueDistance(out []histogramDistance, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) {
+	var is_good_pair bool = false
+	var p histogramPair
+	p.idx2 = 0
+	p.idx1 = p.idx2
+	p.cost_combo = 0
+	p.cost_diff = p.cost_combo
+	if idx1 == idx2 {
+		return
+	}
+
+	if idx2 < idx1 {
+		var t uint32 = idx2
+		idx2 = idx1
+		idx1 = t
+	}
+
+	p.idx1 = idx1
+	p.idx2 = idx2
+	p.cost_diff = 0.5 * clusterCostDiff(uint(cluster_size[idx1]), uint(cluster_size[idx2]))
+	p.cost_diff -= out[idx1].bit_cost_
+	p.cost_diff -= out[idx2].bit_cost_
+
+	if out[idx1].total_count_ == 0 {
+		p.cost_combo = out[idx2].bit_cost_
+		is_good_pair = true
+	} else if out[idx2].total_count_ == 0 {
+		p.cost_combo = out[idx1].bit_cost_
+		is_good_pair = true
+	} else {
+		var threshold float64
+		if *num_pairs == 0 {
+			threshold = 1e99
+		} else {
+			threshold = brotli_max_double(0.0, pairs[0].cost_diff)
+		}
+		var combo histogramDistance = out[idx1]
+		var cost_combo float64
+		histogramAddHistogramDistance(&combo, &out[idx2])
+		cost_combo = populationCostDistance(&combo)
+		if cost_combo < threshold-p.cost_diff {
+			p.cost_combo = cost_combo
+			is_good_pair = true
+		}
+	}
+
+	if is_good_pair {
+		p.cost_diff += p.cost_combo
+		if *num_pairs > 0 && histogramPairIsLess(&pairs[0], &p) {
+			/* Replace the top of the queue if needed. */
+			if *num_pairs < max_num_pairs {
+				pairs[*num_pairs] = pairs[0]
+				(*num_pairs)++
+			}
+
+			pairs[0] = p
+		} else if *num_pairs < max_num_pairs {
+			pairs[*num_pairs] = p
+			(*num_pairs)++
+		}
+	}
+}
+
+func histogramCombineDistance(out []histogramDistance, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint {
+	var cost_diff_threshold float64 = 0.0
+	var min_cluster_size uint = 1
+	var num_pairs uint = 0
+	{
+		/* We maintain a vector of histogram pairs, with the property that the pair
+		   with the maximum bit cost reduction is the first. */
+		var idx1 uint
+		for idx1 = 0; idx1 < num_clusters; idx1++ {
+			var idx2 uint
+			for idx2 = idx1 + 1; idx2 < num_clusters; idx2++ {
+				compareAndPushToQueueDistance(out, cluster_size, clusters[idx1], clusters[idx2], max_num_pairs, pairs[0:], &num_pairs)
+			}
+		}
+	}
+
+	for num_clusters > min_cluster_size {
+		var best_idx1 uint32
+		var best_idx2 uint32
+		var i uint
+		if pairs[0].cost_diff >= cost_diff_threshold {
+			cost_diff_threshold = 1e99
+			min_cluster_size = max_clusters
+			continue
+		}
+
+		/* Take the best pair from the top of heap. */
+		best_idx1 = pairs[0].idx1
+
+		best_idx2 = pairs[0].idx2
+		histogramAddHistogramDistance(&out[best_idx1], &out[best_idx2])
+		out[best_idx1].bit_cost_ = pairs[0].cost_combo
+		cluster_size[best_idx1] += cluster_size[best_idx2]
+		for i = 0; i < symbols_size; i++ {
+			if symbols[i] == best_idx2 {
+				symbols[i] = best_idx1
+			}
+		}
+
+		for i = 0; i < num_clusters; i++ {
+			if clusters[i] == best_idx2 {
+				copy(clusters[i:], clusters[i+1:][:num_clusters-i-1])
+				break
+			}
+		}
+
+		num_clusters--
+		{
+			/* Remove pairs intersecting the just combined best pair. */
+			var copy_to_idx uint = 0
+			for i = 0; i < num_pairs; i++ {
+				var p *histogramPair = &pairs[i]
+				if p.idx1 == best_idx1 || p.idx2 == best_idx1 || p.idx1 == best_idx2 || p.idx2 == best_idx2 {
+					/* Remove invalid pair from the queue. */
+					continue
+				}
+
+				if histogramPairIsLess(&pairs[0], p) {
+					/* Replace the top of the queue if needed. */
+					var front histogramPair = pairs[0]
+					pairs[0] = *p
+					pairs[copy_to_idx] = front
+				} else {
+					pairs[copy_to_idx] = *p
+				}
+
+				copy_to_idx++
+			}
+
+			num_pairs = copy_to_idx
+		}
+
+		/* Push new pairs formed with the combined histogram to the heap. */
+		for i = 0; i < num_clusters; i++ {
+			compareAndPushToQueueDistance(out, cluster_size, best_idx1, clusters[i], max_num_pairs, pairs[0:], &num_pairs)
+		}
+	}
+
+	return num_clusters
+}
+
+/* What is the bit cost of moving histogram from cur_symbol to candidate. */
+func histogramBitCostDistanceDistance(histogram *histogramDistance, candidate *histogramDistance) float64 {
+	if histogram.total_count_ == 0 {
+		return 0.0
+	} else {
+		var tmp histogramDistance = *histogram
+		histogramAddHistogramDistance(&tmp, candidate)
+		return populationCostDistance(&tmp) - candidate.bit_cost_
+	}
+}
+
+/* Find the best 'out' histogram for each of the 'in' histograms.
+   When called, clusters[0..num_clusters) contains the unique values from
+   symbols[0..in_size), but this property is not preserved in this function.
+   Note: we assume that out[]->bit_cost_ is already up-to-date. */
+func histogramRemapDistance(in []histogramDistance, in_size uint, clusters []uint32, num_clusters uint, out []histogramDistance, symbols []uint32) {
+	var i uint
+	for i = 0; i < in_size; i++ {
+		var best_out uint32
+		if i == 0 {
+			best_out = symbols[0]
+		} else {
+			best_out = symbols[i-1]
+		}
+		var best_bits float64 = histogramBitCostDistanceDistance(&in[i], &out[best_out])
+		var j uint
+		for j = 0; j < num_clusters; j++ {
+			var cur_bits float64 = histogramBitCostDistanceDistance(&in[i], &out[clusters[j]])
+			if cur_bits < best_bits {
+				best_bits = cur_bits
+				best_out = clusters[j]
+			}
+		}
+
+		symbols[i] = best_out
+	}
+
+	/* Recompute each out based on raw and symbols. */
+	for i = 0; i < num_clusters; i++ {
+		histogramClearDistance(&out[clusters[i]])
+	}
+
+	for i = 0; i < in_size; i++ {
+		histogramAddHistogramDistance(&out[symbols[i]], &in[i])
+	}
+}
+
+/* Reorders elements of the out[0..length) array and changes values in
+   symbols[0..length) array in the following way:
+     * when called, symbols[] contains indexes into out[], and has N unique
+       values (possibly N < length)
+     * on return, symbols'[i] = f(symbols[i]) and
+                  out'[symbols'[i]] = out[symbols[i]], for each 0 <= i < length,
+       where f is a bijection between the range of symbols[] and [0..N), and
+       the first occurrences of values in symbols'[i] come in consecutive
+       increasing order.
+   Returns N, the number of unique values in symbols[]. */
+
+var histogramReindexDistance_kInvalidIndex uint32 = math.MaxUint32
+
+func histogramReindexDistance(out []histogramDistance, symbols []uint32, length uint) uint {
+	var new_index []uint32 = make([]uint32, length)
+	var next_index uint32
+	var tmp []histogramDistance
+	var i uint
+	for i = 0; i < length; i++ {
+		new_index[i] = histogramReindexDistance_kInvalidIndex
+	}
+
+	next_index = 0
+	for i = 0; i < length; i++ {
+		if new_index[symbols[i]] == histogramReindexDistance_kInvalidIndex {
+			new_index[symbols[i]] = next_index
+			next_index++
+		}
+	}
+
+	/* TODO: by using idea of "cycle-sort" we can avoid allocation of
+	   tmp and reduce the number of copying by the factor of 2. */
+	tmp = make([]histogramDistance, next_index)
+
+	next_index = 0
+	for i = 0; i < length; i++ {
+		if new_index[symbols[i]] == next_index {
+			tmp[next_index] = out[symbols[i]]
+			next_index++
+		}
+
+		symbols[i] = new_index[symbols[i]]
+	}
+
+	new_index = nil
+	for i = 0; uint32(i) < next_index; i++ {
+		out[i] = tmp[i]
+	}
+
+	tmp = nil
+	return uint(next_index)
+}
+
+func clusterHistogramsDistance(in []histogramDistance, in_size uint, max_histograms uint, out []histogramDistance, out_size *uint, histogram_symbols []uint32) {
+	var cluster_size []uint32 = make([]uint32, in_size)
+	var clusters []uint32 = make([]uint32, in_size)
+	var num_clusters uint = 0
+	var max_input_histograms uint = 64
+	var pairs_capacity uint = max_input_histograms * max_input_histograms / 2
+	var pairs []histogramPair = make([]histogramPair, (pairs_capacity + 1))
+	var i uint
+
+	/* For the first pass of clustering, we allow all pairs. */
+	for i = 0; i < in_size; i++ {
+		cluster_size[i] = 1
+	}
+
+	for i = 0; i < in_size; i++ {
+		out[i] = in[i]
+		out[i].bit_cost_ = populationCostDistance(&in[i])
+		histogram_symbols[i] = uint32(i)
+	}
+
+	for i = 0; i < in_size; i += max_input_histograms {
+		var num_to_combine uint = brotli_min_size_t(in_size-i, max_input_histograms)
+		var num_new_clusters uint
+		var j uint
+		for j = 0; j < num_to_combine; j++ {
+			clusters[num_clusters+j] = uint32(i + j)
+		}
+
+		num_new_clusters = histogramCombineDistance(out, cluster_size, histogram_symbols[i:], clusters[num_clusters:], pairs, num_to_combine, num_to_combine, max_histograms, pairs_capacity)
+		num_clusters += num_new_clusters
+	}
+	{
+		/* For the second pass, we limit the total number of histogram pairs.
+		   After this limit is reached, we only keep searching for the best pair. */
+		var max_num_pairs uint = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
+		if pairs_capacity < (max_num_pairs + 1) {
+			var _new_size uint
+			if pairs_capacity == 0 {
+				_new_size = max_num_pairs + 1
+			} else {
+				_new_size = pairs_capacity
+			}
+			var new_array []histogramPair
+			for _new_size < (max_num_pairs + 1) {
+				_new_size *= 2
+			}
+			new_array = make([]histogramPair, _new_size)
+			if pairs_capacity != 0 {
+				copy(new_array, pairs[:pairs_capacity])
+			}
+
+			pairs = new_array
+			pairs_capacity = _new_size
+		}
+
+		/* Collapse similar histograms. */
+		num_clusters = histogramCombineDistance(out, cluster_size, histogram_symbols, clusters, pairs, num_clusters, in_size, max_histograms, max_num_pairs)
+	}
+
+	pairs = nil
+	cluster_size = nil
+
+	/* Find the optimal map from original histograms to the final ones. */
+	histogramRemapDistance(in, in_size, clusters, num_clusters, out, histogram_symbols)
+
+	clusters = nil
+
+	/* Convert the context map to a canonical form. */
+	*out_size = histogramReindexDistance(out, histogram_symbols, in_size)
+}

vendor/github.com/andybalholm/brotli/cluster_literal.go

@@ -0,0 +1,326 @@
+package brotli
+
+import "math"
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if
+   it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */
+func compareAndPushToQueueLiteral(out []histogramLiteral, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) {
+	var is_good_pair bool = false
+	var p histogramPair
+	p.idx2 = 0
+	p.idx1 = p.idx2
+	p.cost_combo = 0
+	p.cost_diff = p.cost_combo
+	if idx1 == idx2 {
+		return
+	}
+
+	if idx2 < idx1 {
+		var t uint32 = idx2
+		idx2 = idx1
+		idx1 = t
+	}
+
+	p.idx1 = idx1
+	p.idx2 = idx2
+	p.cost_diff = 0.5 * clusterCostDiff(uint(cluster_size[idx1]), uint(cluster_size[idx2]))
+	p.cost_diff -= out[idx1].bit_cost_
+	p.cost_diff -= out[idx2].bit_cost_
+
+	if out[idx1].total_count_ == 0 {
+		p.cost_combo = out[idx2].bit_cost_
+		is_good_pair = true
+	} else if out[idx2].total_count_ == 0 {
+		p.cost_combo = out[idx1].bit_cost_
+		is_good_pair = true
+	} else {
+		var threshold float64
+		if *num_pairs == 0 {
+			threshold = 1e99
+		} else {
+			threshold = brotli_max_double(0.0, pairs[0].cost_diff)
+		}
+		var combo histogramLiteral = out[idx1]
+		var cost_combo float64
+		histogramAddHistogramLiteral(&combo, &out[idx2])
+		cost_combo = populationCostLiteral(&combo)
+		if cost_combo < threshold-p.cost_diff {
+			p.cost_combo = cost_combo
+			is_good_pair = true
+		}
+	}
+
+	if is_good_pair {
+		p.cost_diff += p.cost_combo
+		if *num_pairs > 0 && histogramPairIsLess(&pairs[0], &p) {
+			/* Replace the top of the queue if needed. */
+			if *num_pairs < max_num_pairs {
+				pairs[*num_pairs] = pairs[0]
+				(*num_pairs)++
+			}
+
+			pairs[0] = p
+		} else if *num_pairs < max_num_pairs {
+			pairs[*num_pairs] = p
+			(*num_pairs)++
+		}
+	}
+}
+
+func histogramCombineLiteral(out []histogramLiteral, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint {
+	var cost_diff_threshold float64 = 0.0
+	var min_cluster_size uint = 1
+	var num_pairs uint = 0
+	{
+		/* We maintain a vector of histogram pairs, with the property that the pair
+		   with the maximum bit cost reduction is the first. */
+		var idx1 uint
+		for idx1 = 0; idx1 < num_clusters; idx1++ {
+			var idx2 uint
+			for idx2 = idx1 + 1; idx2 < num_clusters; idx2++ {
+				compareAndPushToQueueLiteral(out, cluster_size, clusters[idx1], clusters[idx2], max_num_pairs, pairs[0:], &num_pairs)
+			}
+		}
+	}
+
+	for num_clusters > min_cluster_size {
+		var best_idx1 uint32
+		var best_idx2 uint32
+		var i uint
+		if pairs[0].cost_diff >= cost_diff_threshold {
+			cost_diff_threshold = 1e99
+			min_cluster_size = max_clusters
+			continue
+		}
+
+		/* Take the best pair from the top of heap. */
+		best_idx1 = pairs[0].idx1
+
+		best_idx2 = pairs[0].idx2
+		histogramAddHistogramLiteral(&out[best_idx1], &out[best_idx2])
+		out[best_idx1].bit_cost_ = pairs[0].cost_combo
+		cluster_size[best_idx1] += cluster_size[best_idx2]
+		for i = 0; i < symbols_size; i++ {
+			if symbols[i] == best_idx2 {
+				symbols[i] = best_idx1
+			}
+		}
+
+		for i = 0; i < num_clusters; i++ {
+			if clusters[i] == best_idx2 {
+				copy(clusters[i:], clusters[i+1:][:num_clusters-i-1])
+				break
+			}
+		}
+
+		num_clusters--
+		{
+			/* Remove pairs intersecting the just combined best pair. */
+			var copy_to_idx uint = 0
+			for i = 0; i < num_pairs; i++ {
+				var p *histogramPair = &pairs[i]
+				if p.idx1 == best_idx1 || p.idx2 == best_idx1 || p.idx1 == best_idx2 || p.idx2 == best_idx2 {
+					/* Remove invalid pair from the queue. */
+					continue
+				}
+
+				if histogramPairIsLess(&pairs[0], p) {
+					/* Replace the top of the queue if needed. */
+					var front histogramPair = pairs[0]
+					pairs[0] = *p
+					pairs[copy_to_idx] = front
+				} else {
+					pairs[copy_to_idx] = *p
+				}
+
+				copy_to_idx++
+			}
+
+			num_pairs = copy_to_idx
+		}
+
+		/* Push new pairs formed with the combined histogram to the heap. */
+		for i = 0; i < num_clusters; i++ {
+			compareAndPushToQueueLiteral(out, cluster_size, best_idx1, clusters[i], max_num_pairs, pairs[0:], &num_pairs)
+		}
+	}
+
+	return num_clusters
+}
+
+/* What is the bit cost of moving histogram from cur_symbol to candidate. */
+func histogramBitCostDistanceLiteral(histogram *histogramLiteral, candidate *histogramLiteral) float64 {
+	if histogram.total_count_ == 0 {
+		return 0.0
+	} else {
+		var tmp histogramLiteral = *histogram
+		histogramAddHistogramLiteral(&tmp, candidate)
+		return populationCostLiteral(&tmp) - candidate.bit_cost_
+	}
+}
+
+/* Find the best 'out' histogram for each of the 'in' histograms.
+   When called, clusters[0..num_clusters) contains the unique values from
+   symbols[0..in_size), but this property is not preserved in this function.
+   Note: we assume that out[]->bit_cost_ is already up-to-date. */
+func histogramRemapLiteral(in []histogramLiteral, in_size uint, clusters []uint32, num_clusters uint, out []histogramLiteral, symbols []uint32) {
+	var i uint
+	for i = 0; i < in_size; i++ {
+		var best_out uint32
+		if i == 0 {
+			best_out = symbols[0]
+		} else {
+			best_out = symbols[i-1]
+		}
+		var best_bits float64 = histogramBitCostDistanceLiteral(&in[i], &out[best_out])
+		var j uint
+		for j = 0; j < num_clusters; j++ {
+			var cur_bits float64 = histogramBitCostDistanceLiteral(&in[i], &out[clusters[j]])
+			if cur_bits < best_bits {
+				best_bits = cur_bits
+				best_out = clusters[j]
+			}
+		}
+
+		symbols[i] = best_out
+	}
+
+	/* Recompute each out based on raw and symbols. */
+	for i = 0; i < num_clusters; i++ {
+		histogramClearLiteral(&out[clusters[i]])
+	}
+
+	for i = 0; i < in_size; i++ {
+		histogramAddHistogramLiteral(&out[symbols[i]], &in[i])
+	}
+}
+
+/* Reorders elements of the out[0..length) array and changes values in
+   symbols[0..length) array in the following way:
+     * when called, symbols[] contains indexes into out[], and has N unique
+       values (possibly N < length)
+     * on return, symbols'[i] = f(symbols[i]) and
+                  out'[symbols'[i]] = out[symbols[i]], for each 0 <= i < length,
+       where f is a bijection between the range of symbols[] and [0..N), and
+       the first occurrences of values in symbols'[i] come in consecutive
+       increasing order.
+   Returns N, the number of unique values in symbols[]. */
+
+var histogramReindexLiteral_kInvalidIndex uint32 = math.MaxUint32
+
+func histogramReindexLiteral(out []histogramLiteral, symbols []uint32, length uint) uint {
+	var new_index []uint32 = make([]uint32, length)
+	var next_index uint32
+	var tmp []histogramLiteral
+	var i uint
+	for i = 0; i < length; i++ {
+		new_index[i] = histogramReindexLiteral_kInvalidIndex
+	}
+
+	next_index = 0
+	for i = 0; i < length; i++ {
+		if new_index[symbols[i]] == histogramReindexLiteral_kInvalidIndex {
+			new_index[symbols[i]] = next_index
+			next_index++
+		}
+	}
+
+	/* TODO: by using idea of "cycle-sort" we can avoid allocation of
+	   tmp and reduce the number of copying by the factor of 2. */
+	tmp = make([]histogramLiteral, next_index)
+
+	next_index = 0
+	for i = 0; i < length; i++ {
+		if new_index[symbols[i]] == next_index {
+			tmp[next_index] = out[symbols[i]]
+			next_index++
+		}
+
+		symbols[i] = new_index[symbols[i]]
+	}
+
+	new_index = nil
+	for i = 0; uint32(i) < next_index; i++ {
+		out[i] = tmp[i]
+	}
+
+	tmp = nil
+	return uint(next_index)
+}
+
+func clusterHistogramsLiteral(in []histogramLiteral, in_size uint, max_histograms uint, out []histogramLiteral, out_size *uint, histogram_symbols []uint32) {
+	var cluster_size []uint32 = make([]uint32, in_size)
+	var clusters []uint32 = make([]uint32, in_size)
+	var num_clusters uint = 0
+	var max_input_histograms uint = 64
+	var pairs_capacity uint = max_input_histograms * max_input_histograms / 2
+	var pairs []histogramPair = make([]histogramPair, (pairs_capacity + 1))
+	var i uint
+
+	/* For the first pass of clustering, we allow all pairs. */
+	for i = 0; i < in_size; i++ {
+		cluster_size[i] = 1
+	}
+
+	for i = 0; i < in_size; i++ {
+		out[i] = in[i]
+		out[i].bit_cost_ = populationCostLiteral(&in[i])
+		histogram_symbols[i] = uint32(i)
+	}
+
+	for i = 0; i < in_size; i += max_input_histograms {
+		var num_to_combine uint = brotli_min_size_t(in_size-i, max_input_histograms)
+		var num_new_clusters uint
+		var j uint
+		for j = 0; j < num_to_combine; j++ {
+			clusters[num_clusters+j] = uint32(i + j)
+		}
+
+		num_new_clusters = histogramCombineLiteral(out, cluster_size, histogram_symbols[i:], clusters[num_clusters:], pairs, num_to_combine, num_to_combine, max_histograms, pairs_capacity)
+		num_clusters += num_new_clusters
+	}
+	{
+		/* For the second pass, we limit the total number of histogram pairs.
+		   After this limit is reached, we only keep searching for the best pair. */
+		var max_num_pairs uint = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
+		if pairs_capacity < (max_num_pairs + 1) {
+			var _new_size uint
+			if pairs_capacity == 0 {
+				_new_size = max_num_pairs + 1
+			} else {
+				_new_size = pairs_capacity
+			}
+			var new_array []histogramPair
+			for _new_size < (max_num_pairs + 1) {
+				_new_size *= 2
+			}
+			new_array = make([]histogramPair, _new_size)
+			if pairs_capacity != 0 {
+				copy(new_array, pairs[:pairs_capacity])
+			}
+
+			pairs = new_array
+			pairs_capacity = _new_size
+		}
+
+		/* Collapse similar histograms. */
+		num_clusters = histogramCombineLiteral(out, cluster_size, histogram_symbols, clusters, pairs, num_clusters, in_size, max_histograms, max_num_pairs)
+	}
+
+	pairs = nil
+	cluster_size = nil
+
+	/* Find the optimal map from original histograms to the final ones. */
+	histogramRemapLiteral(in, in_size, clusters, num_clusters, out, histogram_symbols)
+
+	clusters = nil
+
+	/* Convert the context map to a canonical form. */
+	*out_size = histogramReindexLiteral(out, histogram_symbols, in_size)
+}

vendor/github.com/andybalholm/brotli/command.go

@@ -0,0 +1,252 @@
+package brotli
+
+var kInsBase = []uint32{
+	0,
+	1,
+	2,
+	3,
+	4,
+	5,
+	6,
+	8,
+	10,
+	14,
+	18,
+	26,
+	34,
+	50,
+	66,
+	98,
+	130,
+	194,
+	322,
+	578,
+	1090,
+	2114,
+	6210,
+	22594,
+}
+
+var kInsExtra = []uint32{
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	1,
+	1,
+	2,
+	2,
+	3,
+	3,
+	4,
+	4,
+	5,
+	5,
+	6,
+	7,
+	8,
+	9,
+	10,
+	12,
+	14,
+	24,
+}
+
+var kCopyBase = []uint32{
+	2,
+	3,
+	4,
+	5,
+	6,
+	7,
+	8,
+	9,
+	10,
+	12,
+	14,
+	18,
+	22,
+	30,
+	38,
+	54,
+	70,
+	102,
+	134,
+	198,
+	326,
+	582,
+	1094,
+	2118,
+}
+
+var kCopyExtra = []uint32{
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	1,
+	1,
+	2,
+	2,
+	3,
+	3,
+	4,
+	4,
+	5,
+	5,
+	6,
+	7,
+	8,
+	9,
+	10,
+	24,
+}
+
+func getInsertLengthCode(insertlen uint) uint16 {
+	if insertlen < 6 {
+		return uint16(insertlen)
+	} else if insertlen < 130 {
+		var nbits uint32 = log2FloorNonZero(insertlen-2) - 1
+		return uint16((nbits << 1) + uint32((insertlen-2)>>nbits) + 2)
+	} else if insertlen < 2114 {
+		return uint16(log2FloorNonZero(insertlen-66) + 10)
+	} else if insertlen < 6210 {
+		return 21
+	} else if insertlen < 22594 {
+		return 22
+	} else {
+		return 23
+	}
+}
+
+func getCopyLengthCode(copylen uint) uint16 {
+	if copylen < 10 {
+		return uint16(copylen - 2)
+	} else if copylen < 134 {
+		var nbits uint32 = log2FloorNonZero(copylen-6) - 1
+		return uint16((nbits << 1) + uint32((copylen-6)>>nbits) + 4)
+	} else if copylen < 2118 {
+		return uint16(log2FloorNonZero(copylen-70) + 12)
+	} else {
+		return 23
+	}
+}
+
+func combineLengthCodes(inscode uint16, copycode uint16, use_last_distance bool) uint16 {
+	var bits64 uint16 = uint16(copycode&0x7 | (inscode&0x7)<<3)
+	if use_last_distance && inscode < 8 && copycode < 16 {
+		if copycode < 8 {
+			return bits64
+		} else {
+			return bits64 | 64
+		}
+	} else {
+		/* Specification: 5 Encoding of ... (last table) */
+		/* offset = 2 * index, where index is in range [0..8] */
+		var offset uint32 = 2 * ((uint32(copycode) >> 3) + 3*(uint32(inscode)>>3))
+
+		/* All values in specification are K * 64,
+		   where   K = [2, 3, 6, 4, 5, 8, 7, 9, 10],
+		       i + 1 = [1, 2, 3, 4, 5, 6, 7, 8,  9],
+		   K - i - 1 = [1, 1, 3, 0, 0, 2, 0, 1,  2] = D.
+		   All values in D require only 2 bits to encode.
+		   Magic constant is shifted 6 bits left, to avoid final multiplication. */
+		offset = (offset << 5) + 0x40 + ((0x520D40 >> offset) & 0xC0)
+
+		return uint16(offset | uint32(bits64))
+	}
+}
+
+func getLengthCode(insertlen uint, copylen uint, use_last_distance bool, code *uint16) {
+	var inscode uint16 = getInsertLengthCode(insertlen)
+	var copycode uint16 = getCopyLengthCode(copylen)
+	*code = combineLengthCodes(inscode, copycode, use_last_distance)
+}
+
+func getInsertBase(inscode uint16) uint32 {
+	return kInsBase[inscode]
+}
+
+func getInsertExtra(inscode uint16) uint32 {
+	return kInsExtra[inscode]
+}
+
+func getCopyBase(copycode uint16) uint32 {
+	return kCopyBase[copycode]
+}
+
+func getCopyExtra(copycode uint16) uint32 {
+	return kCopyExtra[copycode]
+}
+
+type command struct {
+	insert_len_  uint32
+	copy_len_    uint32
+	dist_extra_  uint32
+	cmd_prefix_  uint16
+	dist_prefix_ uint16
+}
+
+/* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */
+func initCommand(self *command, dist *distanceParams, insertlen uint, copylen uint, copylen_code_delta int, distance_code uint) {
+	/* Don't rely on signed int representation, use honest casts. */
+	var delta uint32 = uint32(byte(int8(copylen_code_delta)))
+	self.insert_len_ = uint32(insertlen)
+	self.copy_len_ = uint32(uint32(copylen) | delta<<25)
+
+	/* The distance prefix and extra bits are stored in this Command as if
+	   npostfix and ndirect were 0, they are only recomputed later after the
+	   clustering if needed. */
+	prefixEncodeCopyDistance(distance_code, uint(dist.num_direct_distance_codes), uint(dist.distance_postfix_bits), &self.dist_prefix_, &self.dist_extra_)
+
+	getLengthCode(insertlen, uint(int(copylen)+copylen_code_delta), (self.dist_prefix_&0x3FF == 0), &self.cmd_prefix_)
+}
+
+func initInsertCommand(self *command, insertlen uint) {
+	self.insert_len_ = uint32(insertlen)
+	self.copy_len_ = 4 << 25
+	self.dist_extra_ = 0
+	self.dist_prefix_ = numDistanceShortCodes
+	getLengthCode(insertlen, 4, false, &self.cmd_prefix_)
+}
+
+func commandRestoreDistanceCode(self *command, dist *distanceParams) uint32 {
+	if uint32(self.dist_prefix_&0x3FF) < numDistanceShortCodes+dist.num_direct_distance_codes {
+		return uint32(self.dist_prefix_) & 0x3FF
+	} else {
+		var dcode uint32 = uint32(self.dist_prefix_) & 0x3FF
+		var nbits uint32 = uint32(self.dist_prefix_) >> 10
+		var extra uint32 = self.dist_extra_
+		var postfix_mask uint32 = (1 << dist.distance_postfix_bits) - 1
+		var hcode uint32 = (dcode - dist.num_direct_distance_codes - numDistanceShortCodes) >> dist.distance_postfix_bits
+		var lcode uint32 = (dcode - dist.num_direct_distance_codes - numDistanceShortCodes) & postfix_mask
+		var offset uint32 = ((2 + (hcode & 1)) << nbits) - 4
+		return ((offset + extra) << dist.distance_postfix_bits) + lcode + dist.num_direct_distance_codes + numDistanceShortCodes
+	}
+}
+
+func commandDistanceContext(self *command) uint32 {
+	var r uint32 = uint32(self.cmd_prefix_) >> 6
+	var c uint32 = uint32(self.cmd_prefix_) & 7
+	if (r == 0 || r == 2 || r == 4 || r == 7) && (c <= 2) {
+		return c
+	}
+
+	return 3
+}
+
+func commandCopyLen(self *command) uint32 {
+	return self.copy_len_ & 0x1FFFFFF
+}
+
+func commandCopyLenCode(self *command) uint32 {
+	var modifier uint32 = self.copy_len_ >> 25
+	var delta int32 = int32(int8(byte(modifier | (modifier&0x40)<<1)))
+	return uint32(int32(self.copy_len_&0x1FFFFFF) + delta)
+}

vendor/github.com/andybalholm/brotli/compress_fragment.go

@@ -0,0 +1,840 @@
+package brotli
+
+import "encoding/binary"
+
+/* Copyright 2015 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Function for fast encoding of an input fragment, independently from the input
+   history. This function uses one-pass processing: when we find a backward
+   match, we immediately emit the corresponding command and literal codes to
+   the bit stream.
+
+   Adapted from the CompressFragment() function in
+   https://github.com/google/snappy/blob/master/snappy.cc */
+
+const maxDistance_compress_fragment = 262128
+
+func hash5(p []byte, shift uint) uint32 {
+	var h uint64 = (binary.LittleEndian.Uint64(p) << 24) * uint64(kHashMul32)
+	return uint32(h >> shift)
+}
+
+func hashBytesAtOffset5(v uint64, offset int, shift uint) uint32 {
+	assert(offset >= 0)
+	assert(offset <= 3)
+	{
+		var h uint64 = ((v >> uint(8*offset)) << 24) * uint64(kHashMul32)
+		return uint32(h >> shift)
+	}
+}
+
+func isMatch5(p1 []byte, p2 []byte) bool {
+	var i int
+	for i = 0; i < 5; i++ {
+		if p1[i] != p2[i] {
+			return false
+		}
+	}
+
+	return true
+}
+
+/* Builds a literal prefix code into "depths" and "bits" based on the statistics
+   of the "input" string and stores it into the bit stream.
+   Note that the prefix code here is built from the pre-LZ77 input, therefore
+   we can only approximate the statistics of the actual literal stream.
+   Moreover, for long inputs we build a histogram from a sample of the input
+   and thus have to assign a non-zero depth for each literal.
+   Returns estimated compression ratio millibytes/char for encoding given input
+   with generated code. */
+func buildAndStoreLiteralPrefixCode(input []byte, input_size uint, depths []byte, bits []uint16, storage_ix *uint, storage []byte) uint {
+	var histogram = [256]uint32{0}
+	var histogram_total uint
+	var i uint
+	if input_size < 1<<15 {
+		for i = 0; i < input_size; i++ {
+			histogram[input[i]]++
+		}
+
+		histogram_total = input_size
+		for i = 0; i < 256; i++ {
+			/* We weigh the first 11 samples with weight 3 to account for the
+			   balancing effect of the LZ77 phase on the histogram. */
+			var adjust uint32 = 2 * brotli_min_uint32_t(histogram[i], 11)
+			histogram[i] += adjust
+			histogram_total += uint(adjust)
+		}
+	} else {
+		const kSampleRate uint = 29
+		for i = 0; i < input_size; i += kSampleRate {
+			histogram[input[i]]++
+		}
+
+		histogram_total = (input_size + kSampleRate - 1) / kSampleRate
+		for i = 0; i < 256; i++ {
+			/* We add 1 to each population count to avoid 0 bit depths (since this is
+			   only a sample and we don't know if the symbol appears or not), and we
+			   weigh the first 11 samples with weight 3 to account for the balancing
+			   effect of the LZ77 phase on the histogram (more frequent symbols are
+			   more likely to be in backward references instead as literals). */
+			var adjust uint32 = 1 + 2*brotli_min_uint32_t(histogram[i], 11)
+			histogram[i] += adjust
+			histogram_total += uint(adjust)
+		}
+	}
+
+	buildAndStoreHuffmanTreeFast(histogram[:], histogram_total, /* max_bits = */
+		8, depths, bits, storage_ix, storage)
+	{
+		var literal_ratio uint = 0
+		for i = 0; i < 256; i++ {
+			if histogram[i] != 0 {
+				literal_ratio += uint(histogram[i] * uint32(depths[i]))
+			}
+		}
+
+		/* Estimated encoding ratio, millibytes per symbol. */
+		return (literal_ratio * 125) / histogram_total
+	}
+}
+
+/* Builds a command and distance prefix code (each 64 symbols) into "depth" and
+   "bits" based on "histogram" and stores it into the bit stream. */
+func buildAndStoreCommandPrefixCode1(histogram []uint32, depth []byte, bits []uint16, storage_ix *uint, storage []byte) {
+	var tree [129]huffmanTree
+	var cmd_depth = [numCommandSymbols]byte{0}
+	/* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */
+
+	var cmd_bits [64]uint16
+
+	createHuffmanTree(histogram, 64, 15, tree[:], depth)
+	createHuffmanTree(histogram[64:], 64, 14, tree[:], depth[64:])
+
+	/* We have to jump through a few hoops here in order to compute
+	   the command bits because the symbols are in a different order than in
+	   the full alphabet. This looks complicated, but having the symbols
+	   in this order in the command bits saves a few branches in the Emit*
+	   functions. */
+	copy(cmd_depth[:], depth[:24])
+
+	copy(cmd_depth[24:][:], depth[40:][:8])
+	copy(cmd_depth[32:][:], depth[24:][:8])
+	copy(cmd_depth[40:][:], depth[48:][:8])
+	copy(cmd_depth[48:][:], depth[32:][:8])
+	copy(cmd_depth[56:][:], depth[56:][:8])
+	convertBitDepthsToSymbols(cmd_depth[:], 64, cmd_bits[:])
+	copy(bits, cmd_bits[:24])
+	copy(bits[24:], cmd_bits[32:][:8])
+	copy(bits[32:], cmd_bits[48:][:8])
+	copy(bits[40:], cmd_bits[24:][:8])
+	copy(bits[48:], cmd_bits[40:][:8])
+	copy(bits[56:], cmd_bits[56:][:8])
+	convertBitDepthsToSymbols(depth[64:], 64, bits[64:])
+	{
+		/* Create the bit length array for the full command alphabet. */
+		var i uint
+		for i := 0; i < int(64); i++ {
+			cmd_depth[i] = 0
+		} /* only 64 first values were used */
+		copy(cmd_depth[:], depth[:8])
+		copy(cmd_depth[64:][:], depth[8:][:8])
+		copy(cmd_depth[128:][:], depth[16:][:8])
+		copy(cmd_depth[192:][:], depth[24:][:8])
+		copy(cmd_depth[384:][:], depth[32:][:8])
+		for i = 0; i < 8; i++ {
+			cmd_depth[128+8*i] = depth[40+i]
+			cmd_depth[256+8*i] = depth[48+i]
+			cmd_depth[448+8*i] = depth[56+i]
+		}
+
+		storeHuffmanTree(cmd_depth[:], numCommandSymbols, tree[:], storage_ix, storage)
+	}
+
+	storeHuffmanTree(depth[64:], 64, tree[:], storage_ix, storage)
+}
+
+/* REQUIRES: insertlen < 6210 */
+func emitInsertLen1(insertlen uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) {
+	if insertlen < 6 {
+		var code uint = insertlen + 40
+		writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
+		histo[code]++
+	} else if insertlen < 130 {
+		var tail uint = insertlen - 2
+		var nbits uint32 = log2FloorNonZero(tail) - 1
+		var prefix uint = tail >> nbits
+		var inscode uint = uint((nbits << 1) + uint32(prefix) + 42)
+		writeBits(uint(depth[inscode]), uint64(bits[inscode]), storage_ix, storage)
+		writeBits(uint(nbits), uint64(tail)-(uint64(prefix)<<nbits), storage_ix, storage)
+		histo[inscode]++
+	} else if insertlen < 2114 {
+		var tail uint = insertlen - 66
+		var nbits uint32 = log2FloorNonZero(tail)
+		var code uint = uint(nbits + 50)
+		writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
+		writeBits(uint(nbits), uint64(tail)-(uint64(uint(1))<<nbits), storage_ix, storage)
+		histo[code]++
+	} else {
+		writeBits(uint(depth[61]), uint64(bits[61]), storage_ix, storage)
+		writeBits(12, uint64(insertlen)-2114, storage_ix, storage)
+		histo[61]++
+	}
+}
+
+func emitLongInsertLen(insertlen uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) {
+	if insertlen < 22594 {
+		writeBits(uint(depth[62]), uint64(bits[62]), storage_ix, storage)
+		writeBits(14, uint64(insertlen)-6210, storage_ix, storage)
+		histo[62]++
+	} else {
+		writeBits(uint(depth[63]), uint64(bits[63]), storage_ix, storage)
+		writeBits(24, uint64(insertlen)-22594, storage_ix, storage)
+		histo[63]++
+	}
+}
+
+func emitCopyLen1(copylen uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) {
+	if copylen < 10 {
+		writeBits(uint(depth[copylen+14]), uint64(bits[copylen+14]), storage_ix, storage)
+		histo[copylen+14]++
+	} else if copylen < 134 {
+		var tail uint = copylen - 6
+		var nbits uint32 = log2FloorNonZero(tail) - 1
+		var prefix uint = tail >> nbits
+		var code uint = uint((nbits << 1) + uint32(prefix) + 20)
+		writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
+		writeBits(uint(nbits), uint64(tail)-(uint64(prefix)<<nbits), storage_ix, storage)
+		histo[code]++
+	} else if copylen < 2118 {
+		var tail uint = copylen - 70
+		var nbits uint32 = log2FloorNonZero(tail)
+		var code uint = uint(nbits + 28)
+		writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
+		writeBits(uint(nbits), uint64(tail)-(uint64(uint(1))<<nbits), storage_ix, storage)
+		histo[code]++
+	} else {
+		writeBits(uint(depth[39]), uint64(bits[39]), storage_ix, storage)
+		writeBits(24, uint64(copylen)-2118, storage_ix, storage)
+		histo[39]++
+	}
+}
+
+func emitCopyLenLastDistance1(copylen uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) {
+	if copylen < 12 {
+		writeBits(uint(depth[copylen-4]), uint64(bits[copylen-4]), storage_ix, storage)
+		histo[copylen-4]++
+	} else if copylen < 72 {
+		var tail uint = copylen - 8
+		var nbits uint32 = log2FloorNonZero(tail) - 1
+		var prefix uint = tail >> nbits
+		var code uint = uint((nbits << 1) + uint32(prefix) + 4)
+		writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
+		writeBits(uint(nbits), uint64(tail)-(uint64(prefix)<<nbits), storage_ix, storage)
+		histo[code]++
+	} else if copylen < 136 {
+		var tail uint = copylen - 8
+		var code uint = (tail >> 5) + 30
+		writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
+		writeBits(5, uint64(tail)&31, storage_ix, storage)
+		writeBits(uint(depth[64]), uint64(bits[64]), storage_ix, storage)
+		histo[code]++
+		histo[64]++
+	} else if copylen < 2120 {
+		var tail uint = copylen - 72
+		var nbits uint32 = log2FloorNonZero(tail)
+		var code uint = uint(nbits + 28)
+		writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
+		writeBits(uint(nbits), uint64(tail)-(uint64(uint(1))<<nbits), storage_ix, storage)
+		writeBits(uint(depth[64]), uint64(bits[64]), storage_ix, storage)
+		histo[code]++
+		histo[64]++
+	} else {
+		writeBits(uint(depth[39]), uint64(bits[39]), storage_ix, storage)
+		writeBits(24, uint64(copylen)-2120, storage_ix, storage)
+		writeBits(uint(depth[64]), uint64(bits[64]), storage_ix, storage)
+		histo[39]++
+		histo[64]++
+	}
+}
+
+func emitDistance1(distance uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) {
+	var d uint = distance + 3
+	var nbits uint32 = log2FloorNonZero(d) - 1
+	var prefix uint = (d >> nbits) & 1
+	var offset uint = (2 + prefix) << nbits
+	var distcode uint = uint(2*(nbits-1) + uint32(prefix) + 80)
+	writeBits(uint(depth[distcode]), uint64(bits[distcode]), storage_ix, storage)
+	writeBits(uint(nbits), uint64(d)-uint64(offset), storage_ix, storage)
+	histo[distcode]++
+}
+
+func emitLiterals(input []byte, len uint, depth []byte, bits []uint16, storage_ix *uint, storage []byte) {
+	var j uint
+	for j = 0; j < len; j++ {
+		var lit byte = input[j]
+		writeBits(uint(depth[lit]), uint64(bits[lit]), storage_ix, storage)
+	}
+}
+
+/* REQUIRES: len <= 1 << 24. */
+func storeMetaBlockHeader1(len uint, is_uncompressed bool, storage_ix *uint, storage []byte) {
+	var nibbles uint = 6
+
+	/* ISLAST */
+	writeBits(1, 0, storage_ix, storage)
+
+	if len <= 1<<16 {
+		nibbles = 4
+	} else if len <= 1<<20 {
+		nibbles = 5
+	}
+
+	writeBits(2, uint64(nibbles)-4, storage_ix, storage)
+	writeBits(nibbles*4, uint64(len)-1, storage_ix, storage)
+
+	/* ISUNCOMPRESSED */
+	writeSingleBit(is_uncompressed, storage_ix, storage)
+}
+
+func updateBits(n_bits uint, bits uint32, pos uint, array []byte) {
+	for n_bits > 0 {
+		var byte_pos uint = pos >> 3
+		var n_unchanged_bits uint = pos & 7
+		var n_changed_bits uint = brotli_min_size_t(n_bits, 8-n_unchanged_bits)
+		var total_bits uint = n_unchanged_bits + n_changed_bits
+		var mask uint32 = (^((1 << total_bits) - 1)) | ((1 << n_unchanged_bits) - 1)
+		var unchanged_bits uint32 = uint32(array[byte_pos]) & mask
+		var changed_bits uint32 = bits & ((1 << n_changed_bits) - 1)
+		array[byte_pos] = byte(changed_bits<<n_unchanged_bits | unchanged_bits)
+		n_bits -= n_changed_bits
+		bits >>= n_changed_bits
+		pos += n_changed_bits
+	}
+}
+
+func rewindBitPosition1(new_storage_ix uint, storage_ix *uint, storage []byte) {
+	var bitpos uint = new_storage_ix & 7
+	var mask uint = (1 << bitpos) - 1
+	storage[new_storage_ix>>3] &= byte(mask)
+	*storage_ix = new_storage_ix
+}
+
+var shouldMergeBlock_kSampleRate uint = 43
+
+func shouldMergeBlock(data []byte, len uint, depths []byte) bool {
+	var histo = [256]uint{0}
+	var i uint
+	for i = 0; i < len; i += shouldMergeBlock_kSampleRate {
+		histo[data[i]]++
+	}
+	{
+		var total uint = (len + shouldMergeBlock_kSampleRate - 1) / shouldMergeBlock_kSampleRate
+		var r float64 = (fastLog2(total)+0.5)*float64(total) + 200
+		for i = 0; i < 256; i++ {
+			r -= float64(histo[i]) * (float64(depths[i]) + fastLog2(histo[i]))
+		}
+
+		return r >= 0.0
+	}
+}
+
+func shouldUseUncompressedMode(metablock_start []byte, next_emit []byte, insertlen uint, literal_ratio uint) bool {
+	var compressed uint = uint(-cap(next_emit) + cap(metablock_start))
+	if compressed*50 > insertlen {
+		return false
+	} else {
+		return literal_ratio > 980
+	}
+}
+
+func emitUncompressedMetaBlock1(begin []byte, end []byte, storage_ix_start uint, storage_ix *uint, storage []byte) {
+	var len uint = uint(-cap(end) + cap(begin))
+	rewindBitPosition1(storage_ix_start, storage_ix, storage)
+	storeMetaBlockHeader1(uint(len), true, storage_ix, storage)
+	*storage_ix = (*storage_ix + 7) &^ 7
+	copy(storage[*storage_ix>>3:], begin[:len])
+	*storage_ix += uint(len << 3)
+	storage[*storage_ix>>3] = 0
+}
+
+var kCmdHistoSeed = [128]uint32{
+	0,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	0,
+	0,
+	0,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	0,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	1,
+	0,
+	0,
+	0,
+	0,
+}
+
+var compressFragmentFastImpl_kFirstBlockSize uint = 3 << 15
+var compressFragmentFastImpl_kMergeBlockSize uint = 1 << 16
+
+func compressFragmentFastImpl(in []byte, input_size uint, is_last bool, table []int, table_bits uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, storage_ix *uint, storage []byte) {
+	var cmd_histo [128]uint32
+	var ip_end int
+	var next_emit int = 0
+	var base_ip int = 0
+	var input int = 0
+	const kInputMarginBytes uint = windowGap
+	const kMinMatchLen uint = 5
+	var metablock_start int = input
+	var block_size uint = brotli_min_size_t(input_size, compressFragmentFastImpl_kFirstBlockSize)
+	var total_block_size uint = block_size
+	var mlen_storage_ix uint = *storage_ix + 3
+	var lit_depth [256]byte
+	var lit_bits [256]uint16
+	var literal_ratio uint
+	var ip int
+	var last_distance int
+	var shift uint = 64 - table_bits
+
+	/* "next_emit" is a pointer to the first byte that is not covered by a
+	   previous copy. Bytes between "next_emit" and the start of the next copy or
+	   the end of the input will be emitted as literal bytes. */
+
+	/* Save the start of the first block for position and distance computations.
+	 */
+
+	/* Save the bit position of the MLEN field of the meta-block header, so that
+	   we can update it later if we decide to extend this meta-block. */
+	storeMetaBlockHeader1(block_size, false, storage_ix, storage)
+
+	/* No block splits, no contexts. */
+	writeBits(13, 0, storage_ix, storage)
+
+	literal_ratio = buildAndStoreLiteralPrefixCode(in[input:], block_size, lit_depth[:], lit_bits[:], storage_ix, storage)
+	{
+		/* Store the pre-compressed command and distance prefix codes. */
+		var i uint
+		for i = 0; i+7 < *cmd_code_numbits; i += 8 {
+			writeBits(8, uint64(cmd_code[i>>3]), storage_ix, storage)
+		}
+	}
+
+	writeBits(*cmd_code_numbits&7, uint64(cmd_code[*cmd_code_numbits>>3]), storage_ix, storage)
+
+	/* Initialize the command and distance histograms. We will gather
+	   statistics of command and distance codes during the processing
+	   of this block and use it to update the command and distance
+	   prefix codes for the next block. */
+emit_commands:
+	copy(cmd_histo[:], kCmdHistoSeed[:])
+
+	/* "ip" is the input pointer. */
+	ip = input
+
+	last_distance = -1
+	ip_end = int(uint(input) + block_size)
+
+	if block_size >= kInputMarginBytes {
+		var len_limit uint = brotli_min_size_t(block_size-kMinMatchLen, input_size-kInputMarginBytes)
+		var ip_limit int = int(uint(input) + len_limit)
+		/* For the last block, we need to keep a 16 bytes margin so that we can be
+		   sure that all distances are at most window size - 16.
+		   For all other blocks, we only need to keep a margin of 5 bytes so that
+		   we don't go over the block size with a copy. */
+
+		var next_hash uint32
+		ip++
+		for next_hash = hash5(in[ip:], shift); ; {
+			var skip uint32 = 32
+			var next_ip int = ip
+			/* Step 1: Scan forward in the input looking for a 5-byte-long match.
+			   If we get close to exhausting the input then goto emit_remainder.
+
+			   Heuristic match skipping: If 32 bytes are scanned with no matches
+			   found, start looking only at every other byte. If 32 more bytes are
+			   scanned, look at every third byte, etc.. When a match is found,
+			   immediately go back to looking at every byte. This is a small loss
+			   (~5% performance, ~0.1% density) for compressible data due to more
+			   bookkeeping, but for non-compressible data (such as JPEG) it's a huge
+			   win since the compressor quickly "realizes" the data is incompressible
+			   and doesn't bother looking for matches everywhere.
+
+			   The "skip" variable keeps track of how many bytes there are since the
+			   last match; dividing it by 32 (i.e. right-shifting by five) gives the
+			   number of bytes to move ahead for each iteration. */
+
+			var candidate int
+			assert(next_emit < ip)
+
+		trawl:
+			for {
+				var hash uint32 = next_hash
+				var bytes_between_hash_lookups uint32 = skip >> 5
+				skip++
+				assert(hash == hash5(in[next_ip:], shift))
+				ip = next_ip
+				next_ip = int(uint32(ip) + bytes_between_hash_lookups)
+				if next_ip > ip_limit {
+					goto emit_remainder
+				}
+
+				next_hash = hash5(in[next_ip:], shift)
+				candidate = ip - last_distance
+				if isMatch5(in[ip:], in[candidate:]) {
+					if candidate < ip {
+						table[hash] = int(ip - base_ip)
+						break
+					}
+				}
+
+				candidate = base_ip + table[hash]
+				assert(candidate >= base_ip)
+				assert(candidate < ip)
+
+				table[hash] = int(ip - base_ip)
+				if !(!isMatch5(in[ip:], in[candidate:])) {
+					break
+				}
+			}
+
+			/* Check copy distance. If candidate is not feasible, continue search.
+			   Checking is done outside of hot loop to reduce overhead. */
+			if ip-candidate > maxDistance_compress_fragment {
+				goto trawl
+			}
+
+			/* Step 2: Emit the found match together with the literal bytes from
+			   "next_emit" to the bit stream, and then see if we can find a next match
+			   immediately afterwards. Repeat until we find no match for the input
+			   without emitting some literal bytes. */
+			{
+				var base int = ip
+				/* > 0 */
+				var matched uint = 5 + findMatchLengthWithLimit(in[candidate+5:], in[ip+5:], uint(ip_end-ip)-5)
+				var distance int = int(base - candidate)
+				/* We have a 5-byte match at ip, and we need to emit bytes in
+				   [next_emit, ip). */
+
+				var insert uint = uint(base - next_emit)
+				ip += int(matched)
+				if insert < 6210 {
+					emitInsertLen1(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
+				} else if shouldUseUncompressedMode(in[metablock_start:], in[next_emit:], insert, literal_ratio) {
+					emitUncompressedMetaBlock1(in[metablock_start:], in[base:], mlen_storage_ix-3, storage_ix, storage)
+					input_size -= uint(base - input)
+					input = base
+					next_emit = input
+					goto next_block
+				} else {
+					emitLongInsertLen(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
+				}
+
+				emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], storage_ix, storage)
+				if distance == last_distance {
+					writeBits(uint(cmd_depth[64]), uint64(cmd_bits[64]), storage_ix, storage)
+					cmd_histo[64]++
+				} else {
+					emitDistance1(uint(distance), cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
+					last_distance = distance
+				}
+
+				emitCopyLenLastDistance1(matched, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
+
+				next_emit = ip
+				if ip >= ip_limit {
+					goto emit_remainder
+				}
+
+				/* We could immediately start working at ip now, but to improve
+				   compression we first update "table" with the hashes of some positions
+				   within the last copy. */
+				{
+					var input_bytes uint64 = binary.LittleEndian.Uint64(in[ip-3:])
+					var prev_hash uint32 = hashBytesAtOffset5(input_bytes, 0, shift)
+					var cur_hash uint32 = hashBytesAtOffset5(input_bytes, 3, shift)
+					table[prev_hash] = int(ip - base_ip - 3)
+					prev_hash = hashBytesAtOffset5(input_bytes, 1, shift)
+					table[prev_hash] = int(ip - base_ip - 2)
+					prev_hash = hashBytesAtOffset5(input_bytes, 2, shift)
+					table[prev_hash] = int(ip - base_ip - 1)
+
+					candidate = base_ip + table[cur_hash]
+					table[cur_hash] = int(ip - base_ip)
+				}
+			}
+
+			for isMatch5(in[ip:], in[candidate:]) {
+				var base int = ip
+				/* We have a 5-byte match at ip, and no need to emit any literal bytes
+				   prior to ip. */
+
+				var matched uint = 5 + findMatchLengthWithLimit(in[candidate+5:], in[ip+5:], uint(ip_end-ip)-5)
+				if ip-candidate > maxDistance_compress_fragment {
+					break
+				}
+				ip += int(matched)
+				last_distance = int(base - candidate) /* > 0 */
+				emitCopyLen1(matched, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
+				emitDistance1(uint(last_distance), cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
+
+				next_emit = ip
+				if ip >= ip_limit {
+					goto emit_remainder
+				}
+
+				/* We could immediately start working at ip now, but to improve
+				   compression we first update "table" with the hashes of some positions
+				   within the last copy. */
+				{
+					var input_bytes uint64 = binary.LittleEndian.Uint64(in[ip-3:])
+					var prev_hash uint32 = hashBytesAtOffset5(input_bytes, 0, shift)
+					var cur_hash uint32 = hashBytesAtOffset5(input_bytes, 3, shift)
+					table[prev_hash] = int(ip - base_ip - 3)
+					prev_hash = hashBytesAtOffset5(input_bytes, 1, shift)
+					table[prev_hash] = int(ip - base_ip - 2)
+					prev_hash = hashBytesAtOffset5(input_bytes, 2, shift)
+					table[prev_hash] = int(ip - base_ip - 1)
+
+					candidate = base_ip + table[cur_hash]
+					table[cur_hash] = int(ip - base_ip)
+				}
+			}
+
+			ip++
+			next_hash = hash5(in[ip:], shift)
+		}
+	}
+
+emit_remainder:
+	assert(next_emit <= ip_end)
+	input += int(block_size)
+	input_size -= block_size
+	block_size = brotli_min_size_t(input_size, compressFragmentFastImpl_kMergeBlockSize)
+
+	/* Decide if we want to continue this meta-block instead of emitting the
+	   last insert-only command. */
+	if input_size > 0 && total_block_size+block_size <= 1<<20 && shouldMergeBlock(in[input:], block_size, lit_depth[:]) {
+		assert(total_block_size > 1<<16)
+
+		/* Update the size of the current meta-block and continue emitting commands.
+		   We can do this because the current size and the new size both have 5
+		   nibbles. */
+		total_block_size += block_size
+
+		updateBits(20, uint32(total_block_size-1), mlen_storage_ix, storage)
+		goto emit_commands
+	}
+
+	/* Emit the remaining bytes as literals. */
+	if next_emit < ip_end {
+		var insert uint = uint(ip_end - next_emit)
+		if insert < 6210 {
+			emitInsertLen1(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
+			emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], storage_ix, storage)
+		} else if shouldUseUncompressedMode(in[metablock_start:], in[next_emit:], insert, literal_ratio) {
+			emitUncompressedMetaBlock1(in[metablock_start:], in[ip_end:], mlen_storage_ix-3, storage_ix, storage)
+		} else {
+			emitLongInsertLen(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage)
+			emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], storage_ix, storage)
+		}
+	}
+
+	next_emit = ip_end
+
+	/* If we have more data, write a new meta-block header and prefix codes and
+	   then continue emitting commands. */
+next_block:
+	if input_size > 0 {
+		metablock_start = input
+		block_size = brotli_min_size_t(input_size, compressFragmentFastImpl_kFirstBlockSize)
+		total_block_size = block_size
+
+		/* Save the bit position of the MLEN field of the meta-block header, so that
+		   we can update it later if we decide to extend this meta-block. */
+		mlen_storage_ix = *storage_ix + 3
+
+		storeMetaBlockHeader1(block_size, false, storage_ix, storage)
+
+		/* No block splits, no contexts. */
+		writeBits(13, 0, storage_ix, storage)
+
+		literal_ratio = buildAndStoreLiteralPrefixCode(in[input:], block_size, lit_depth[:], lit_bits[:], storage_ix, storage)
+		buildAndStoreCommandPrefixCode1(cmd_histo[:], cmd_depth, cmd_bits, storage_ix, storage)
+		goto emit_commands
+	}
+
+	if !is_last {
+		/* If this is not the last block, update the command and distance prefix
+		   codes for the next block and store the compressed forms. */
+		cmd_code[0] = 0
+
+		*cmd_code_numbits = 0
+		buildAndStoreCommandPrefixCode1(cmd_histo[:], cmd_depth, cmd_bits, cmd_code_numbits, cmd_code)
+	}
+}
+
+/* Compresses "input" string to the "*storage" buffer as one or more complete
+   meta-blocks, and updates the "*storage_ix" bit position.
+
+   If "is_last" is 1, emits an additional empty last meta-block.
+
+   "cmd_depth" and "cmd_bits" contain the command and distance prefix codes
+   (see comment in encode.h) used for the encoding of this input fragment.
+   If "is_last" is 0, they are updated to reflect the statistics
+   of this input fragment, to be used for the encoding of the next fragment.
+
+   "*cmd_code_numbits" is the number of bits of the compressed representation
+   of the command and distance prefix codes, and "cmd_code" is an array of
+   at least "(*cmd_code_numbits + 7) >> 3" size that contains the compressed
+   command and distance prefix codes. If "is_last" is 0, these are also
+   updated to represent the updated "cmd_depth" and "cmd_bits".
+
+   REQUIRES: "input_size" is greater than zero, or "is_last" is 1.
+   REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24).
+   REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero.
+   REQUIRES: "table_size" is an odd (9, 11, 13, 15) power of two
+   OUTPUT: maximal copy distance <= |input_size|
+   OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */
+func compressFragmentFast(input []byte, input_size uint, is_last bool, table []int, table_size uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, storage_ix *uint, storage []byte) {
+	var initial_storage_ix uint = *storage_ix
+	var table_bits uint = uint(log2FloorNonZero(table_size))
+
+	if input_size == 0 {
+		assert(is_last)
+		writeBits(1, 1, storage_ix, storage) /* islast */
+		writeBits(1, 1, storage_ix, storage) /* isempty */
+		*storage_ix = (*storage_ix + 7) &^ 7
+		return
+	}
+
+	compressFragmentFastImpl(input, input_size, is_last, table, table_bits, cmd_depth, cmd_bits, cmd_code_numbits, cmd_code, storage_ix, storage)
+
+	/* If output is larger than single uncompressed block, rewrite it. */
+	if *storage_ix-initial_storage_ix > 31+(input_size<<3) {
+		emitUncompressedMetaBlock1(input, input[input_size:], initial_storage_ix, storage_ix, storage)
+	}
+
+	if is_last {
+		writeBits(1, 1, storage_ix, storage) /* islast */
+		writeBits(1, 1, storage_ix, storage) /* isempty */
+		*storage_ix = (*storage_ix + 7) &^ 7
+	}
+}

vendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go

@@ -0,0 +1,749 @@
+package brotli
+
+import "encoding/binary"
+
+/* Copyright 2015 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Function for fast encoding of an input fragment, independently from the input
+   history. This function uses two-pass processing: in the first pass we save
+   the found backward matches and literal bytes into a buffer, and in the
+   second pass we emit them into the bit stream using prefix codes built based
+   on the actual command and literal byte histograms. */
+
+const kCompressFragmentTwoPassBlockSize uint = 1 << 17
+
+func hash1(p []byte, shift uint, length uint) uint32 {
+	var h uint64 = (binary.LittleEndian.Uint64(p) << ((8 - length) * 8)) * uint64(kHashMul32)
+	return uint32(h >> shift)
+}
+
+func hashBytesAtOffset(v uint64, offset uint, shift uint, length uint) uint32 {
+	assert(offset <= 8-length)
+	{
+		var h uint64 = ((v >> (8 * offset)) << ((8 - length) * 8)) * uint64(kHashMul32)
+		return uint32(h >> shift)
+	}
+}
+
+func isMatch1(p1 []byte, p2 []byte, length uint) bool {
+	var i uint
+	for i = 0; i < length && i < 6; i++ {
+		if p1[i] != p2[i] {
+			return false
+		}
+	}
+
+	return true
+}
+
+/* Builds a command and distance prefix code (each 64 symbols) into "depth" and
+   "bits" based on "histogram" and stores it into the bit stream. */
+func buildAndStoreCommandPrefixCode(histogram []uint32, depth []byte, bits []uint16, storage_ix *uint, storage []byte) {
+	var tree [129]huffmanTree
+	var cmd_depth = [numCommandSymbols]byte{0}
+	/* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */
+
+	var cmd_bits [64]uint16
+	createHuffmanTree(histogram, 64, 15, tree[:], depth)
+	createHuffmanTree(histogram[64:], 64, 14, tree[:], depth[64:])
+
+	/* We have to jump through a few hoops here in order to compute
+	   the command bits because the symbols are in a different order than in
+	   the full alphabet. This looks complicated, but having the symbols
+	   in this order in the command bits saves a few branches in the Emit*
+	   functions. */
+	copy(cmd_depth[:], depth[24:][:24])
+
+	copy(cmd_depth[24:][:], depth[:8])
+	copy(cmd_depth[32:][:], depth[48:][:8])
+	copy(cmd_depth[40:][:], depth[8:][:8])
+	copy(cmd_depth[48:][:], depth[56:][:8])
+	copy(cmd_depth[56:][:], depth[16:][:8])
+	convertBitDepthsToSymbols(cmd_depth[:], 64, cmd_bits[:])
+	copy(bits, cmd_bits[24:][:8])
+	copy(bits[8:], cmd_bits[40:][:8])
+	copy(bits[16:], cmd_bits[56:][:8])
+	copy(bits[24:], cmd_bits[:24])
+	copy(bits[48:], cmd_bits[32:][:8])
+	copy(bits[56:], cmd_bits[48:][:8])
+	convertBitDepthsToSymbols(depth[64:], 64, bits[64:])
+	{
+		/* Create the bit length array for the full command alphabet. */
+		var i uint
+		for i := 0; i < int(64); i++ {
+			cmd_depth[i] = 0
+		} /* only 64 first values were used */
+		copy(cmd_depth[:], depth[24:][:8])
+		copy(cmd_depth[64:][:], depth[32:][:8])
+		copy(cmd_depth[128:][:], depth[40:][:8])
+		copy(cmd_depth[192:][:], depth[48:][:8])
+		copy(cmd_depth[384:][:], depth[56:][:8])
+		for i = 0; i < 8; i++ {
+			cmd_depth[128+8*i] = depth[i]
+			cmd_depth[256+8*i] = depth[8+i]
+			cmd_depth[448+8*i] = depth[16+i]
+		}
+
+		storeHuffmanTree(cmd_depth[:], numCommandSymbols, tree[:], storage_ix, storage)
+	}
+
+	storeHuffmanTree(depth[64:], 64, tree[:], storage_ix, storage)
+}
+
+func emitInsertLen(insertlen uint32, commands *[]uint32) {
+	if insertlen < 6 {
+		(*commands)[0] = insertlen
+	} else if insertlen < 130 {
+		var tail uint32 = insertlen - 2
+		var nbits uint32 = log2FloorNonZero(uint(tail)) - 1
+		var prefix uint32 = tail >> nbits
+		var inscode uint32 = (nbits << 1) + prefix + 2
+		var extra uint32 = tail - (prefix << nbits)
+		(*commands)[0] = inscode | extra<<8
+	} else if insertlen < 2114 {
+		var tail uint32 = insertlen - 66
+		var nbits uint32 = log2FloorNonZero(uint(tail))
+		var code uint32 = nbits + 10
+		var extra uint32 = tail - (1 << nbits)
+		(*commands)[0] = code | extra<<8
+	} else if insertlen < 6210 {
+		var extra uint32 = insertlen - 2114
+		(*commands)[0] = 21 | extra<<8
+	} else if insertlen < 22594 {
+		var extra uint32 = insertlen - 6210
+		(*commands)[0] = 22 | extra<<8
+	} else {
+		var extra uint32 = insertlen - 22594
+		(*commands)[0] = 23 | extra<<8
+	}
+
+	*commands = (*commands)[1:]
+}
+
+func emitCopyLen(copylen uint, commands *[]uint32) {
+	if copylen < 10 {
+		(*commands)[0] = uint32(copylen + 38)
+	} else if copylen < 134 {
+		var tail uint = copylen - 6
+		var nbits uint = uint(log2FloorNonZero(tail) - 1)
+		var prefix uint = tail >> nbits
+		var code uint = (nbits << 1) + prefix + 44
+		var extra uint = tail - (prefix << nbits)
+		(*commands)[0] = uint32(code | extra<<8)
+	} else if copylen < 2118 {
+		var tail uint = copylen - 70
+		var nbits uint = uint(log2FloorNonZero(tail))
+		var code uint = nbits + 52
+		var extra uint = tail - (uint(1) << nbits)
+		(*commands)[0] = uint32(code | extra<<8)
+	} else {
+		var extra uint = copylen - 2118
+		(*commands)[0] = uint32(63 | extra<<8)
+	}
+
+	*commands = (*commands)[1:]
+}
+
+func emitCopyLenLastDistance(copylen uint, commands *[]uint32) {
+	if copylen < 12 {
+		(*commands)[0] = uint32(copylen + 20)
+		*commands = (*commands)[1:]
+	} else if copylen < 72 {
+		var tail uint = copylen - 8
+		var nbits uint = uint(log2FloorNonZero(tail) - 1)
+		var prefix uint = tail >> nbits
+		var code uint = (nbits << 1) + prefix + 28
+		var extra uint = tail - (prefix << nbits)
+		(*commands)[0] = uint32(code | extra<<8)
+		*commands = (*commands)[1:]
+	} else if copylen < 136 {
+		var tail uint = copylen - 8
+		var code uint = (tail >> 5) + 54
+		var extra uint = tail & 31
+		(*commands)[0] = uint32(code | extra<<8)
+		*commands = (*commands)[1:]
+		(*commands)[0] = 64
+		*commands = (*commands)[1:]
+	} else if copylen < 2120 {
+		var tail uint = copylen - 72
+		var nbits uint = uint(log2FloorNonZero(tail))
+		var code uint = nbits + 52
+		var extra uint = tail - (uint(1) << nbits)
+		(*commands)[0] = uint32(code | extra<<8)
+		*commands = (*commands)[1:]
+		(*commands)[0] = 64
+		*commands = (*commands)[1:]
+	} else {
+		var extra uint = copylen - 2120
+		(*commands)[0] = uint32(63 | extra<<8)
+		*commands = (*commands)[1:]
+		(*commands)[0] = 64
+		*commands = (*commands)[1:]
+	}
+}
+
+func emitDistance(distance uint32, commands *[]uint32) {
+	var d uint32 = distance + 3
+	var nbits uint32 = log2FloorNonZero(uint(d)) - 1
+	var prefix uint32 = (d >> nbits) & 1
+	var offset uint32 = (2 + prefix) << nbits
+	var distcode uint32 = 2*(nbits-1) + prefix + 80
+	var extra uint32 = d - offset
+	(*commands)[0] = distcode | extra<<8
+	*commands = (*commands)[1:]
+}
+
+/* REQUIRES: len <= 1 << 24. */
+func storeMetaBlockHeader(len uint, is_uncompressed bool, storage_ix *uint, storage []byte) {
+	var nibbles uint = 6
+
+	/* ISLAST */
+	writeBits(1, 0, storage_ix, storage)
+
+	if len <= 1<<16 {
+		nibbles = 4
+	} else if len <= 1<<20 {
+		nibbles = 5
+	}
+
+	writeBits(2, uint64(nibbles)-4, storage_ix, storage)
+	writeBits(nibbles*4, uint64(len)-1, storage_ix, storage)
+
+	/* ISUNCOMPRESSED */
+	writeSingleBit(is_uncompressed, storage_ix, storage)
+}
+
+func createCommands(input []byte, block_size uint, input_size uint, base_ip_ptr []byte, table []int, table_bits uint, min_match uint, literals *[]byte, commands *[]uint32) {
+	var ip int = 0
+	var shift uint = 64 - table_bits
+	var ip_end int = int(block_size)
+	var base_ip int = -cap(base_ip_ptr) + cap(input)
+	var next_emit int = 0
+	var last_distance int = -1
+	/* "ip" is the input pointer. */
+
+	const kInputMarginBytes uint = windowGap
+
+	/* "next_emit" is a pointer to the first byte that is not covered by a
+	   previous copy. Bytes between "next_emit" and the start of the next copy or
+	   the end of the input will be emitted as literal bytes. */
+	if block_size >= kInputMarginBytes {
+		var len_limit uint = brotli_min_size_t(block_size-min_match, input_size-kInputMarginBytes)
+		var ip_limit int = int(len_limit)
+		/* For the last block, we need to keep a 16 bytes margin so that we can be
+		   sure that all distances are at most window size - 16.
+		   For all other blocks, we only need to keep a margin of 5 bytes so that
+		   we don't go over the block size with a copy. */
+
+		var next_hash uint32
+		ip++
+		for next_hash = hash1(input[ip:], shift, min_match); ; {
+			var skip uint32 = 32
+			var next_ip int = ip
+			/* Step 1: Scan forward in the input looking for a 6-byte-long match.
+			   If we get close to exhausting the input then goto emit_remainder.
+
+			   Heuristic match skipping: If 32 bytes are scanned with no matches
+			   found, start looking only at every other byte. If 32 more bytes are
+			   scanned, look at every third byte, etc.. When a match is found,
+			   immediately go back to looking at every byte. This is a small loss
+			   (~5% performance, ~0.1% density) for compressible data due to more
+			   bookkeeping, but for non-compressible data (such as JPEG) it's a huge
+			   win since the compressor quickly "realizes" the data is incompressible
+			   and doesn't bother looking for matches everywhere.
+
+			   The "skip" variable keeps track of how many bytes there are since the
+			   last match; dividing it by 32 (ie. right-shifting by five) gives the
+			   number of bytes to move ahead for each iteration. */
+
+			var candidate int
+
+			assert(next_emit < ip)
+
+		trawl:
+			for {
+				var hash uint32 = next_hash
+				var bytes_between_hash_lookups uint32 = skip >> 5
+				skip++
+				ip = next_ip
+				assert(hash == hash1(input[ip:], shift, min_match))
+				next_ip = int(uint32(ip) + bytes_between_hash_lookups)
+				if next_ip > ip_limit {
+					goto emit_remainder
+				}
+
+				next_hash = hash1(input[next_ip:], shift, min_match)
+				candidate = ip - last_distance
+				if isMatch1(input[ip:], base_ip_ptr[candidate-base_ip:], min_match) {
+					if candidate < ip {
+						table[hash] = int(ip - base_ip)
+						break
+					}
+				}
+
+				candidate = base_ip + table[hash]
+				assert(candidate >= base_ip)
+				assert(candidate < ip)
+
+				table[hash] = int(ip - base_ip)
+				if isMatch1(input[ip:], base_ip_ptr[candidate-base_ip:], min_match) {
+					break
+				}
+			}
+
+			/* Check copy distance. If candidate is not feasible, continue search.
+			   Checking is done outside of hot loop to reduce overhead. */
+			if ip-candidate > maxDistance_compress_fragment {
+				goto trawl
+			}
+
+			/* Step 2: Emit the found match together with the literal bytes from
+			   "next_emit", and then see if we can find a next match immediately
+			   afterwards. Repeat until we find no match for the input
+			   without emitting some literal bytes. */
+			{
+				var base int = ip
+				/* > 0 */
+				var matched uint = min_match + findMatchLengthWithLimit(base_ip_ptr[uint(candidate-base_ip)+min_match:], input[uint(ip)+min_match:], uint(ip_end-ip)-min_match)
+				var distance int = int(base - candidate)
+				/* We have a 6-byte match at ip, and we need to emit bytes in
+				   [next_emit, ip). */
+
+				var insert int = int(base - next_emit)
+				ip += int(matched)
+				emitInsertLen(uint32(insert), commands)
+				copy(*literals, input[next_emit:][:uint(insert)])
+				*literals = (*literals)[insert:]
+				if distance == last_distance {
+					(*commands)[0] = 64
+					*commands = (*commands)[1:]
+				} else {
+					emitDistance(uint32(distance), commands)
+					last_distance = distance
+				}
+
+				emitCopyLenLastDistance(matched, commands)
+
+				next_emit = ip
+				if ip >= ip_limit {
+					goto emit_remainder
+				}
+				{
+					var input_bytes uint64
+					var cur_hash uint32
+					/* We could immediately start working at ip now, but to improve
+					   compression we first update "table" with the hashes of some
+					   positions within the last copy. */
+
+					var prev_hash uint32
+					if min_match == 4 {
+						input_bytes = binary.LittleEndian.Uint64(input[ip-3:])
+						cur_hash = hashBytesAtOffset(input_bytes, 3, shift, min_match)
+						prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 3)
+						prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 2)
+						prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 1)
+					} else {
+						input_bytes = binary.LittleEndian.Uint64(input[ip-5:])
+						prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 5)
+						prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 4)
+						prev_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 3)
+						input_bytes = binary.LittleEndian.Uint64(input[ip-2:])
+						cur_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match)
+						prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 2)
+						prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 1)
+					}
+
+					candidate = base_ip + table[cur_hash]
+					table[cur_hash] = int(ip - base_ip)
+				}
+			}
+
+			for ip-candidate <= maxDistance_compress_fragment && isMatch1(input[ip:], base_ip_ptr[candidate-base_ip:], min_match) {
+				var base int = ip
+				/* We have a 6-byte match at ip, and no need to emit any
+				   literal bytes prior to ip. */
+
+				var matched uint = min_match + findMatchLengthWithLimit(base_ip_ptr[uint(candidate-base_ip)+min_match:], input[uint(ip)+min_match:], uint(ip_end-ip)-min_match)
+				ip += int(matched)
+				last_distance = int(base - candidate) /* > 0 */
+				emitCopyLen(matched, commands)
+				emitDistance(uint32(last_distance), commands)
+
+				next_emit = ip
+				if ip >= ip_limit {
+					goto emit_remainder
+				}
+				{
+					var input_bytes uint64
+					var cur_hash uint32
+					/* We could immediately start working at ip now, but to improve
+					   compression we first update "table" with the hashes of some
+					   positions within the last copy. */
+
+					var prev_hash uint32
+					if min_match == 4 {
+						input_bytes = binary.LittleEndian.Uint64(input[ip-3:])
+						cur_hash = hashBytesAtOffset(input_bytes, 3, shift, min_match)
+						prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 3)
+						prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 2)
+						prev_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 1)
+					} else {
+						input_bytes = binary.LittleEndian.Uint64(input[ip-5:])
+						prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 5)
+						prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 4)
+						prev_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 3)
+						input_bytes = binary.LittleEndian.Uint64(input[ip-2:])
+						cur_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match)
+						prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 2)
+						prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match)
+						table[prev_hash] = int(ip - base_ip - 1)
+					}
+
+					candidate = base_ip + table[cur_hash]
+					table[cur_hash] = int(ip - base_ip)
+				}
+			}
+
+			ip++
+			next_hash = hash1(input[ip:], shift, min_match)
+		}
+	}
+
+emit_remainder:
+	assert(next_emit <= ip_end)
+
+	/* Emit the remaining bytes as literals. */
+	if next_emit < ip_end {
+		var insert uint32 = uint32(ip_end - next_emit)
+		emitInsertLen(insert, commands)
+		copy(*literals, input[next_emit:][:insert])
+		*literals = (*literals)[insert:]
+	}
+}
+
+var storeCommands_kNumExtraBits = [128]uint32{
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	1,
+	1,
+	2,
+	2,
+	3,
+	3,
+	4,
+	4,
+	5,
+	5,
+	6,
+	7,
+	8,
+	9,
+	10,
+	12,
+	14,
+	24,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	1,
+	1,
+	2,
+	2,
+	3,
+	3,
+	4,
+	4,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	1,
+	1,
+	2,
+	2,
+	3,
+	3,
+	4,
+	4,
+	5,
+	5,
+	6,
+	7,
+	8,
+	9,
+	10,
+	24,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	0,
+	1,
+	1,
+	2,
+	2,
+	3,
+	3,
+	4,
+	4,
+	5,
+	5,
+	6,
+	6,
+	7,
+	7,
+	8,
+	8,
+	9,
+	9,
+	10,
+	10,
+	11,
+	11,
+	12,
+	12,
+	13,
+	13,
+	14,
+	14,
+	15,
+	15,
+	16,
+	16,
+	17,
+	17,
+	18,
+	18,
+	19,
+	19,
+	20,
+	20,
+	21,
+	21,
+	22,
+	22,
+	23,
+	23,
+	24,
+	24,
+}
+var storeCommands_kInsertOffset = [24]uint32{
+	0,
+	1,
+	2,
+	3,
+	4,
+	5,
+	6,
+	8,
+	10,
+	14,
+	18,
+	26,
+	34,
+	50,
+	66,
+	98,
+	130,
+	194,
+	322,
+	578,
+	1090,
+	2114,
+	6210,
+	22594,
+}
+
+func storeCommands(literals []byte, num_literals uint, commands []uint32, num_commands uint, storage_ix *uint, storage []byte) {
+	var lit_depths [256]byte
+	var lit_bits [256]uint16
+	var lit_histo = [256]uint32{0}
+	var cmd_depths = [128]byte{0}
+	var cmd_bits = [128]uint16{0}
+	var cmd_histo = [128]uint32{0}
+	var i uint
+	for i = 0; i < num_literals; i++ {
+		lit_histo[literals[i]]++
+	}
+
+	buildAndStoreHuffmanTreeFast(lit_histo[:], num_literals, /* max_bits = */
+		8, lit_depths[:], lit_bits[:], storage_ix, storage)
+
+	for i = 0; i < num_commands; i++ {
+		var code uint32 = commands[i] & 0xFF
+		assert(code < 128)
+		cmd_histo[code]++
+	}
+
+	cmd_histo[1] += 1
+	cmd_histo[2] += 1
+	cmd_histo[64] += 1
+	cmd_histo[84] += 1
+	buildAndStoreCommandPrefixCode(cmd_histo[:], cmd_depths[:], cmd_bits[:], storage_ix, storage)
+
+	for i = 0; i < num_commands; i++ {
+		var cmd uint32 = commands[i]
+		var code uint32 = cmd & 0xFF
+		var extra uint32 = cmd >> 8
+		assert(code < 128)
+		writeBits(uint(cmd_depths[code]), uint64(cmd_bits[code]), storage_ix, storage)
+		writeBits(uint(storeCommands_kNumExtraBits[code]), uint64(extra), storage_ix, storage)
+		if code < 24 {
+			var insert uint32 = storeCommands_kInsertOffset[code] + extra
+			var j uint32
+			for j = 0; j < insert; j++ {
+				var lit byte = literals[0]
+				writeBits(uint(lit_depths[lit]), uint64(lit_bits[lit]), storage_ix, storage)
+				literals = literals[1:]
+			}
+		}
+	}
+}
+
+/* Acceptable loss for uncompressible speedup is 2% */
+const minRatio = 0.98
+
+const sampleRate = 43
+
+func shouldCompress(input []byte, input_size uint, num_literals uint) bool {
+	var corpus_size float64 = float64(input_size)
+	if float64(num_literals) < minRatio*corpus_size {
+		return true
+	} else {
+		var literal_histo = [256]uint32{0}
+		var max_total_bit_cost float64 = corpus_size * 8 * minRatio / sampleRate
+		var i uint
+		for i = 0; i < input_size; i += sampleRate {
+			literal_histo[input[i]]++
+		}
+
+		return bitsEntropy(literal_histo[:], 256) < max_total_bit_cost
+	}
+}
+
+func rewindBitPosition(new_storage_ix uint, storage_ix *uint, storage []byte) {
+	var bitpos uint = new_storage_ix & 7
+	var mask uint = (1 << bitpos) - 1
+	storage[new_storage_ix>>3] &= byte(mask)
+	*storage_ix = new_storage_ix
+}
+
+func emitUncompressedMetaBlock(input []byte, input_size uint, storage_ix *uint, storage []byte) {
+	storeMetaBlockHeader(input_size, true, storage_ix, storage)
+	*storage_ix = (*storage_ix + 7) &^ 7
+	copy(storage[*storage_ix>>3:], input[:input_size])
+	*storage_ix += input_size << 3
+	storage[*storage_ix>>3] = 0
+}
+
+func compressFragmentTwoPassImpl(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_bits uint, min_match uint, storage_ix *uint, storage []byte) {
+	/* Save the start of the first block for position and distance computations.
+	 */
+	var base_ip []byte = input
+
+	for input_size > 0 {
+		var block_size uint = brotli_min_size_t(input_size, kCompressFragmentTwoPassBlockSize)
+		var commands []uint32 = command_buf
+		var literals []byte = literal_buf
+		var num_literals uint
+		createCommands(input, block_size, input_size, base_ip, table, table_bits, min_match, &literals, &commands)
+		num_literals = uint(-cap(literals) + cap(literal_buf))
+		if shouldCompress(input, block_size, num_literals) {
+			var num_commands uint = uint(-cap(commands) + cap(command_buf))
+			storeMetaBlockHeader(block_size, false, storage_ix, storage)
+
+			/* No block splits, no contexts. */
+			writeBits(13, 0, storage_ix, storage)
+
+			storeCommands(literal_buf, num_literals, command_buf, num_commands, storage_ix, storage)
+		} else {
+			/* Since we did not find many backward references and the entropy of
+			   the data is close to 8 bits, we can simply emit an uncompressed block.
+			   This makes compression speed of uncompressible data about 3x faster. */
+			emitUncompressedMetaBlock(input, block_size, storage_ix, storage)
+		}
+
+		input = input[block_size:]
+		input_size -= block_size
+	}
+}
+
+/* Compresses "input" string to the "*storage" buffer as one or more complete
+   meta-blocks, and updates the "*storage_ix" bit position.
+
+   If "is_last" is 1, emits an additional empty last meta-block.
+
+   REQUIRES: "input_size" is greater than zero, or "is_last" is 1.
+   REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24).
+   REQUIRES: "command_buf" and "literal_buf" point to at least
+              kCompressFragmentTwoPassBlockSize long arrays.
+   REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero.
+   REQUIRES: "table_size" is a power of two
+   OUTPUT: maximal copy distance <= |input_size|
+   OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */
+func compressFragmentTwoPass(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_size uint, storage_ix *uint, storage []byte) {
+	var initial_storage_ix uint = *storage_ix
+	var table_bits uint = uint(log2FloorNonZero(table_size))
+	var min_match uint
+	if table_bits <= 15 {
+		min_match = 4
+	} else {
+		min_match = 6
+	}
+	compressFragmentTwoPassImpl(input, input_size, is_last, command_buf, literal_buf, table, table_bits, min_match, storage_ix, storage)
+
+	/* If output is larger than single uncompressed block, rewrite it. */
+	if *storage_ix-initial_storage_ix > 31+(input_size<<3) {
+		rewindBitPosition(initial_storage_ix, storage_ix, storage)
+		emitUncompressedMetaBlock(input, input_size, storage_ix, storage)
+	}
+
+	if is_last {
+		writeBits(1, 1, storage_ix, storage) /* islast */
+		writeBits(1, 1, storage_ix, storage) /* isempty */
+		*storage_ix = (*storage_ix + 7) &^ 7
+	}
+}

vendor/github.com/andybalholm/brotli/constants.go

@@ -0,0 +1,77 @@
+package brotli
+
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Specification: 7.3. Encoding of the context map */
+const contextMapMaxRle = 16
+
+/* Specification: 2. Compressed representation overview */
+const maxNumberOfBlockTypes = 256
+
+/* Specification: 3.3. Alphabet sizes: insert-and-copy length */
+const numLiteralSymbols = 256
+
+const numCommandSymbols = 704
+
+const numBlockLenSymbols = 26
+
+const maxContextMapSymbols = (maxNumberOfBlockTypes + contextMapMaxRle)
+
+const maxBlockTypeSymbols = (maxNumberOfBlockTypes + 2)
+
+/* Specification: 3.5. Complex prefix codes */
+const repeatPreviousCodeLength = 16
+
+const repeatZeroCodeLength = 17
+
+const codeLengthCodes = (repeatZeroCodeLength + 1)
+
+/* "code length of 8 is repeated" */
+const initialRepeatedCodeLength = 8
+
+/* "Large Window Brotli" */
+const largeMaxDistanceBits = 62
+
+const largeMinWbits = 10
+
+const largeMaxWbits = 30
+
+/* Specification: 4. Encoding of distances */
+const numDistanceShortCodes = 16
+
+const maxNpostfix = 3
+
+const maxNdirect = 120
+
+const maxDistanceBits = 24
+
+func distanceAlphabetSize(NPOSTFIX uint, NDIRECT uint, MAXNBITS uint) uint {
+	return numDistanceShortCodes + NDIRECT + uint(MAXNBITS<<(NPOSTFIX+1))
+}
+
+/* numDistanceSymbols == 1128 */
+const numDistanceSymbols = 1128
+
+const maxDistance = 0x3FFFFFC
+
+const maxAllowedDistance = 0x7FFFFFFC
+
+/* 7.1. Context modes and context ID lookup for literals */
+/* "context IDs for literals are in the range of 0..63" */
+const literalContextBits = 6
+
+/* 7.2. Context ID for distances */
+const distanceContextBits = 2
+
+/* 9.1. Format of the Stream Header */
+/* Number of slack bytes for window size. Don't confuse
+   with BROTLI_NUM_DISTANCE_SHORT_CODES. */
+const windowGap = 16
+
+func maxBackwardLimit(W uint) uint {
+	return (uint(1) << W) - windowGap
+}

vendor/github.com/andybalholm/brotli/encoder_dict.go

@@ -0,0 +1,22 @@
+package brotli
+
+/* Dictionary data (words and transforms) for 1 possible context */
+type encoderDictionary struct {
+	words                 *dictionary
+	cutoffTransformsCount uint32
+	cutoffTransforms      uint64
+	hash_table            []uint16
+	buckets               []uint16
+	dict_words            []dictWord
+}
+
+func initEncoderDictionary(dict *encoderDictionary) {
+	dict.words = getDictionary()
+
+	dict.hash_table = kStaticDictionaryHash[:]
+	dict.buckets = kStaticDictionaryBuckets[:]
+	dict.dict_words = kStaticDictionaryWords[:]
+
+	dict.cutoffTransformsCount = kCutoffTransformsCount
+	dict.cutoffTransforms = kCutoffTransforms
+}

vendor/github.com/andybalholm/brotli/entropy_encode.go

@@ -0,0 +1,593 @@
+package brotli
+
+import "math"
+
+/* Copyright 2010 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Entropy encoding (Huffman) utilities. */
+
+/* A node of a Huffman tree. */
+type huffmanTree struct {
+	total_count_          uint32
+	index_left_           int16
+	index_right_or_value_ int16
+}
+
+func initHuffmanTree(self *huffmanTree, count uint32, left int16, right int16) {
+	self.total_count_ = count
+	self.index_left_ = left
+	self.index_right_or_value_ = right
+}
+
+/* Input size optimized Shell sort. */
+type huffmanTreeComparator func(*huffmanTree, *huffmanTree) bool
+
+var sortHuffmanTreeItems_gaps = []uint{132, 57, 23, 10, 4, 1}
+
+func sortHuffmanTreeItems(items []huffmanTree, n uint, comparator huffmanTreeComparator) {
+	if n < 13 {
+		/* Insertion sort. */
+		var i uint
+		for i = 1; i < n; i++ {
+			var tmp huffmanTree = items[i]
+			var k uint = i
+			var j uint = i - 1
+			for comparator(&tmp, &items[j]) {
+				items[k] = items[j]
+				k = j
+				tmp10 := j
+				j--
+				if tmp10 == 0 {
+					break
+				}
+			}
+
+			items[k] = tmp
+		}
+
+		return
+	} else {
+		var g int
+		if n < 57 {
+			g = 2
+		} else {
+			g = 0
+		}
+		for ; g < 6; g++ {
+			var gap uint = sortHuffmanTreeItems_gaps[g]
+			var i uint
+			for i = gap; i < n; i++ {
+				var j uint = i
+				var tmp huffmanTree = items[i]
+				for ; j >= gap && comparator(&tmp, &items[j-gap]); j -= gap {
+					items[j] = items[j-gap]
+				}
+
+				items[j] = tmp
+			}
+		}
+	}
+}
+
+/* Returns 1 if assignment of depths succeeded, otherwise 0. */
+func setDepth(p0 int, pool []huffmanTree, depth []byte, max_depth int) bool {
+	var stack [16]int
+	var level int = 0
+	var p int = p0
+	assert(max_depth <= 15)
+	stack[0] = -1
+	for {
+		if pool[p].index_left_ >= 0 {
+			level++
+			if level > max_depth {
+				return false
+			}
+			stack[level] = int(pool[p].index_right_or_value_)
+			p = int(pool[p].index_left_)
+			continue
+		} else {
+			depth[pool[p].index_right_or_value_] = byte(level)
+		}
+
+		for level >= 0 && stack[level] == -1 {
+			level--
+		}
+		if level < 0 {
+			return true
+		}
+		p = stack[level]
+		stack[level] = -1
+	}
+}
+
+/* Sort the root nodes, least popular first. */
+func sortHuffmanTree(v0 *huffmanTree, v1 *huffmanTree) bool {
+	if v0.total_count_ != v1.total_count_ {
+		return v0.total_count_ < v1.total_count_
+	}
+
+	return v0.index_right_or_value_ > v1.index_right_or_value_
+}
+
+/* This function will create a Huffman tree.
+
+   The catch here is that the tree cannot be arbitrarily deep.
+   Brotli specifies a maximum depth of 15 bits for "code trees"
+   and 7 bits for "code length code trees."
+
+   count_limit is the value that is to be faked as the minimum value
+   and this minimum value is raised until the tree matches the
+   maximum length requirement.
+
+   This algorithm is not of excellent performance for very long data blocks,
+   especially when population counts are longer than 2**tree_limit, but
+   we are not planning to use this with extremely long blocks.
+
+   See http://en.wikipedia.org/wiki/Huffman_coding */
+func createHuffmanTree(data []uint32, length uint, tree_limit int, tree []huffmanTree, depth []byte) {
+	var count_limit uint32
+	var sentinel huffmanTree
+	initHuffmanTree(&sentinel, math.MaxUint32, -1, -1)
+
+	/* For block sizes below 64 kB, we never need to do a second iteration
+	   of this loop. Probably all of our block sizes will be smaller than
+	   that, so this loop is mostly of academic interest. If we actually
+	   would need this, we would be better off with the Katajainen algorithm. */
+	for count_limit = 1; ; count_limit *= 2 {
+		var n uint = 0
+		var i uint
+		var j uint
+		var k uint
+		for i = length; i != 0; {
+			i--
+			if data[i] != 0 {
+				var count uint32 = brotli_max_uint32_t(data[i], count_limit)
+				initHuffmanTree(&tree[n], count, -1, int16(i))
+				n++
+			}
+		}
+
+		if n == 1 {
+			depth[tree[0].index_right_or_value_] = 1 /* Only one element. */
+			break
+		}
+
+		sortHuffmanTreeItems(tree, n, huffmanTreeComparator(sortHuffmanTree))
+
+		/* The nodes are:
+		   [0, n): the sorted leaf nodes that we start with.
+		   [n]: we add a sentinel here.
+		   [n + 1, 2n): new parent nodes are added here, starting from
+		                (n+1). These are naturally in ascending order.
+		   [2n]: we add a sentinel at the end as well.
+		   There will be (2n+1) elements at the end. */
+		tree[n] = sentinel
+
+		tree[n+1] = sentinel
+
+		i = 0     /* Points to the next leaf node. */
+		j = n + 1 /* Points to the next non-leaf node. */
+		for k = n - 1; k != 0; k-- {
+			var left uint
+			var right uint
+			if tree[i].total_count_ <= tree[j].total_count_ {
+				left = i
+				i++
+			} else {
+				left = j
+				j++
+			}
+
+			if tree[i].total_count_ <= tree[j].total_count_ {
+				right = i
+				i++
+			} else {
+				right = j
+				j++
+			}
+			{
+				/* The sentinel node becomes the parent node. */
+				var j_end uint = 2*n - k
+				tree[j_end].total_count_ = tree[left].total_count_ + tree[right].total_count_
+				tree[j_end].index_left_ = int16(left)
+				tree[j_end].index_right_or_value_ = int16(right)
+
+				/* Add back the last sentinel node. */
+				tree[j_end+1] = sentinel
+			}
+		}
+
+		if setDepth(int(2*n-1), tree[0:], depth, tree_limit) {
+			/* We need to pack the Huffman tree in tree_limit bits. If this was not
+			   successful, add fake entities to the lowest values and retry. */
+			break
+		}
+	}
+}
+
+func reverse(v []byte, start uint, end uint) {
+	end--
+	for start < end {
+		var tmp byte = v[start]
+		v[start] = v[end]
+		v[end] = tmp
+		start++
+		end--
+	}
+}
+
+func writeHuffmanTreeRepetitions(previous_value byte, value byte, repetitions uint, tree_size *uint, tree []byte, extra_bits_data []byte) {
+	assert(repetitions > 0)
+	if previous_value != value {
+		tree[*tree_size] = value
+		extra_bits_data[*tree_size] = 0
+		(*tree_size)++
+		repetitions--
+	}
+
+	if repetitions == 7 {
+		tree[*tree_size] = value
+		extra_bits_data[*tree_size] = 0
+		(*tree_size)++
+		repetitions--
+	}
+
+	if repetitions < 3 {
+		var i uint
+		for i = 0; i < repetitions; i++ {
+			tree[*tree_size] = value
+			extra_bits_data[*tree_size] = 0
+			(*tree_size)++
+		}
+	} else {
+		var start uint = *tree_size
+		repetitions -= 3
+		for {
+			tree[*tree_size] = repeatPreviousCodeLength
+			extra_bits_data[*tree_size] = byte(repetitions & 0x3)
+			(*tree_size)++
+			repetitions >>= 2
+			if repetitions == 0 {
+				break
+			}
+
+			repetitions--
+		}
+
+		reverse(tree, start, *tree_size)
+		reverse(extra_bits_data, start, *tree_size)
+	}
+}
+
+func writeHuffmanTreeRepetitionsZeros(repetitions uint, tree_size *uint, tree []byte, extra_bits_data []byte) {
+	if repetitions == 11 {
+		tree[*tree_size] = 0
+		extra_bits_data[*tree_size] = 0
+		(*tree_size)++
+		repetitions--
+	}
+
+	if repetitions < 3 {
+		var i uint
+		for i = 0; i < repetitions; i++ {
+			tree[*tree_size] = 0
+			extra_bits_data[*tree_size] = 0
+			(*tree_size)++
+		}
+	} else {
+		var start uint = *tree_size
+		repetitions -= 3
+		for {
+			tree[*tree_size] = repeatZeroCodeLength
+			extra_bits_data[*tree_size] = byte(repetitions & 0x7)
+			(*tree_size)++
+			repetitions >>= 3
+			if repetitions == 0 {
+				break
+			}
+
+			repetitions--
+		}
+
+		reverse(tree, start, *tree_size)
+		reverse(extra_bits_data, start, *tree_size)
+	}
+}
+
+/* Change the population counts in a way that the consequent
+   Huffman tree compression, especially its RLE-part will be more
+   likely to compress this data more efficiently.
+
+   length contains the size of the histogram.
+   counts contains the population counts.
+   good_for_rle is a buffer of at least length size */
+func optimizeHuffmanCountsForRLE(length uint, counts []uint32, good_for_rle []byte) {
+	var nonzero_count uint = 0
+	var stride uint
+	var limit uint
+	var sum uint
+	var streak_limit uint = 1240
+	var i uint
+	/* Let's make the Huffman code more compatible with RLE encoding. */
+	for i = 0; i < length; i++ {
+		if counts[i] != 0 {
+			nonzero_count++
+		}
+	}
+
+	if nonzero_count < 16 {
+		return
+	}
+
+	for length != 0 && counts[length-1] == 0 {
+		length--
+	}
+
+	if length == 0 {
+		return /* All zeros. */
+	}
+
+	/* Now counts[0..length - 1] does not have trailing zeros. */
+	{
+		var nonzeros uint = 0
+		var smallest_nonzero uint32 = 1 << 30
+		for i = 0; i < length; i++ {
+			if counts[i] != 0 {
+				nonzeros++
+				if smallest_nonzero > counts[i] {
+					smallest_nonzero = counts[i]
+				}
+			}
+		}
+
+		if nonzeros < 5 {
+			/* Small histogram will model it well. */
+			return
+		}
+
+		if smallest_nonzero < 4 {
+			var zeros uint = length - nonzeros
+			if zeros < 6 {
+				for i = 1; i < length-1; i++ {
+					if counts[i-1] != 0 && counts[i] == 0 && counts[i+1] != 0 {
+						counts[i] = 1
+					}
+				}
+			}
+		}
+
+		if nonzeros < 28 {
+			return
+		}
+	}
+
+	/* 2) Let's mark all population counts that already can be encoded
+	   with an RLE code. */
+	for i := 0; i < int(length); i++ {
+		good_for_rle[i] = 0
+	}
+	{
+		var symbol uint32 = counts[0]
+		/* Let's not spoil any of the existing good RLE codes.
+		   Mark any seq of 0's that is longer as 5 as a good_for_rle.
+		   Mark any seq of non-0's that is longer as 7 as a good_for_rle. */
+
+		var step uint = 0
+		for i = 0; i <= length; i++ {
+			if i == length || counts[i] != symbol {
+				if (symbol == 0 && step >= 5) || (symbol != 0 && step >= 7) {
+					var k uint
+					for k = 0; k < step; k++ {
+						good_for_rle[i-k-1] = 1
+					}
+				}
+
+				step = 1
+				if i != length {
+					symbol = counts[i]
+				}
+			} else {
+				step++
+			}
+		}
+	}
+
+	/* 3) Let's replace those population counts that lead to more RLE codes.
+	   Math here is in 24.8 fixed point representation. */
+	stride = 0
+
+	limit = uint(256*(counts[0]+counts[1]+counts[2])/3 + 420)
+	sum = 0
+	for i = 0; i <= length; i++ {
+		if i == length || good_for_rle[i] != 0 || (i != 0 && good_for_rle[i-1] != 0) || (256*counts[i]-uint32(limit)+uint32(streak_limit)) >= uint32(2*streak_limit) {
+			if stride >= 4 || (stride >= 3 && sum == 0) {
+				var k uint
+				var count uint = (sum + stride/2) / stride
+				/* The stride must end, collapse what we have, if we have enough (4). */
+				if count == 0 {
+					count = 1
+				}
+
+				if sum == 0 {
+					/* Don't make an all zeros stride to be upgraded to ones. */
+					count = 0
+				}
+
+				for k = 0; k < stride; k++ {
+					/* We don't want to change value at counts[i],
+					   that is already belonging to the next stride. Thus - 1. */
+					counts[i-k-1] = uint32(count)
+				}
+			}
+
+			stride = 0
+			sum = 0
+			if i < length-2 {
+				/* All interesting strides have a count of at least 4, */
+				/* at least when non-zeros. */
+				limit = uint(256*(counts[i]+counts[i+1]+counts[i+2])/3 + 420)
+			} else if i < length {
+				limit = uint(256 * counts[i])
+			} else {
+				limit = 0
+			}
+		}
+
+		stride++
+		if i != length {
+			sum += uint(counts[i])
+			if stride >= 4 {
+				limit = (256*sum + stride/2) / stride
+			}
+
+			if stride == 4 {
+				limit += 120
+			}
+		}
+	}
+}
+
+func decideOverRLEUse(depth []byte, length uint, use_rle_for_non_zero *bool, use_rle_for_zero *bool) {
+	var total_reps_zero uint = 0
+	var total_reps_non_zero uint = 0
+	var count_reps_zero uint = 1
+	var count_reps_non_zero uint = 1
+	var i uint
+	for i = 0; i < length; {
+		var value byte = depth[i]
+		var reps uint = 1
+		var k uint
+		for k = i + 1; k < length && depth[k] == value; k++ {
+			reps++
+		}
+
+		if reps >= 3 && value == 0 {
+			total_reps_zero += reps
+			count_reps_zero++
+		}
+
+		if reps >= 4 && value != 0 {
+			total_reps_non_zero += reps
+			count_reps_non_zero++
+		}
+
+		i += reps
+	}
+
+	*use_rle_for_non_zero = total_reps_non_zero > count_reps_non_zero*2
+	*use_rle_for_zero = total_reps_zero > count_reps_zero*2
+}
+
+/* Write a Huffman tree from bit depths into the bit-stream representation
+   of a Huffman tree. The generated Huffman tree is to be compressed once
+   more using a Huffman tree */
+func writeHuffmanTree(depth []byte, length uint, tree_size *uint, tree []byte, extra_bits_data []byte) {
+	var previous_value byte = initialRepeatedCodeLength
+	var i uint
+	var use_rle_for_non_zero bool = false
+	var use_rle_for_zero bool = false
+	var new_length uint = length
+	/* Throw away trailing zeros. */
+	for i = 0; i < length; i++ {
+		if depth[length-i-1] == 0 {
+			new_length--
+		} else {
+			break
+		}
+	}
+
+	/* First gather statistics on if it is a good idea to do RLE. */
+	if length > 50 {
+		/* Find RLE coding for longer codes.
+		   Shorter codes seem not to benefit from RLE. */
+		decideOverRLEUse(depth, new_length, &use_rle_for_non_zero, &use_rle_for_zero)
+	}
+
+	/* Actual RLE coding. */
+	for i = 0; i < new_length; {
+		var value byte = depth[i]
+		var reps uint = 1
+		if (value != 0 && use_rle_for_non_zero) || (value == 0 && use_rle_for_zero) {
+			var k uint
+			for k = i + 1; k < new_length && depth[k] == value; k++ {
+				reps++
+			}
+		}
+
+		if value == 0 {
+			writeHuffmanTreeRepetitionsZeros(reps, tree_size, tree, extra_bits_data)
+		} else {
+			writeHuffmanTreeRepetitions(previous_value, value, reps, tree_size, tree, extra_bits_data)
+			previous_value = value
+		}
+
+		i += reps
+	}
+}
+
+var reverseBits_kLut = [16]uint{
+	0x00,
+	0x08,
+	0x04,
+	0x0C,
+	0x02,
+	0x0A,
+	0x06,
+	0x0E,
+	0x01,
+	0x09,
+	0x05,
+	0x0D,
+	0x03,
+	0x0B,
+	0x07,
+	0x0F,
+}
+
+func reverseBits(num_bits uint, bits uint16) uint16 {
+	var retval uint = reverseBits_kLut[bits&0x0F]
+	var i uint
+	for i = 4; i < num_bits; i += 4 {
+		retval <<= 4
+		bits = uint16(bits >> 4)
+		retval |= reverseBits_kLut[bits&0x0F]
+	}
+
+	retval >>= ((0 - num_bits) & 0x03)
+	return uint16(retval)
+}
+
+/* 0..15 are values for bits */
+const maxHuffmanBits = 16
+
+/* Get the actual bit values for a tree of bit depths. */
+func convertBitDepthsToSymbols(depth []byte, len uint, bits []uint16) {
+	var bl_count = [maxHuffmanBits]uint16{0}
+	var next_code [maxHuffmanBits]uint16
+	var i uint
+	/* In Brotli, all bit depths are [1..15]
+	   0 bit depth means that the symbol does not exist. */
+
+	var code int = 0
+	for i = 0; i < len; i++ {
+		bl_count[depth[i]]++
+	}
+
+	bl_count[0] = 0
+	next_code[0] = 0
+	for i = 1; i < maxHuffmanBits; i++ {
+		code = (code + int(bl_count[i-1])) << 1
+		next_code[i] = uint16(code)
+	}
+
+	for i = 0; i < len; i++ {
+		if depth[i] != 0 {
+			bits[i] = reverseBits(uint(depth[i]), next_code[depth[i]])
+			next_code[depth[i]]++
+		}
+	}
+}

vendor/github.com/andybalholm/brotli/fast_log.go

@@ -0,0 +1,296 @@
+package brotli
+
+import "math"
+
+/* Copyright 2013 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Utilities for fast computation of logarithms. */
+
+func log2FloorNonZero(n uint) uint32 {
+	/* TODO: generalize and move to platform.h */
+	var result uint32 = 0
+	for {
+		n >>= 1
+		if n == 0 {
+			break
+		}
+		result++
+	}
+	return result
+}
+
+/* A lookup table for small values of log2(int) to be used in entropy
+   computation.
+
+   ", ".join(["%.16ff" % x for x in [0.0]+[log2(x) for x in range(1, 256)]]) */
+var kLog2Table = []float32{
+	0.0000000000000000,
+	0.0000000000000000,
+	1.0000000000000000,
+	1.5849625007211563,
+	2.0000000000000000,
+	2.3219280948873622,
+	2.5849625007211561,
+	2.8073549220576042,
+	3.0000000000000000,
+	3.1699250014423126,
+	3.3219280948873626,
+	3.4594316186372978,
+	3.5849625007211565,
+	3.7004397181410922,
+	3.8073549220576037,
+	3.9068905956085187,
+	4.0000000000000000,
+	4.0874628412503400,
+	4.1699250014423122,
+	4.2479275134435852,
+	4.3219280948873626,
+	4.3923174227787607,
+	4.4594316186372973,
+	4.5235619560570131,
+	4.5849625007211570,
+	4.6438561897747244,
+	4.7004397181410926,
+	4.7548875021634691,
+	4.8073549220576037,
+	4.8579809951275728,
+	4.9068905956085187,
+	4.9541963103868758,
+	5.0000000000000000,
+	5.0443941193584534,
+	5.0874628412503400,
+	5.1292830169449664,
+	5.1699250014423122,
+	5.2094533656289501,
+	5.2479275134435852,
+	5.2854022188622487,
+	5.3219280948873626,
+	5.3575520046180838,
+	5.3923174227787607,
+	5.4262647547020979,
+	5.4594316186372973,
+	5.4918530963296748,
+	5.5235619560570131,
+	5.5545888516776376,
+	5.5849625007211570,
+	5.6147098441152083,
+	5.6438561897747244,
+	5.6724253419714961,
+	5.7004397181410926,
+	5.7279204545631996,
+	5.7548875021634691,
+	5.7813597135246599,
+	5.8073549220576046,
+	5.8328900141647422,
+	5.8579809951275719,
+	5.8826430493618416,
+	5.9068905956085187,
+	5.9307373375628867,
+	5.9541963103868758,
+	5.9772799234999168,
+	6.0000000000000000,
+	6.0223678130284544,
+	6.0443941193584534,
+	6.0660891904577721,
+	6.0874628412503400,
+	6.1085244567781700,
+	6.1292830169449672,
+	6.1497471195046822,
+	6.1699250014423122,
+	6.1898245588800176,
+	6.2094533656289510,
+	6.2288186904958804,
+	6.2479275134435861,
+	6.2667865406949019,
+	6.2854022188622487,
+	6.3037807481771031,
+	6.3219280948873617,
+	6.3398500028846252,
+	6.3575520046180847,
+	6.3750394313469254,
+	6.3923174227787598,
+	6.4093909361377026,
+	6.4262647547020979,
+	6.4429434958487288,
+	6.4594316186372982,
+	6.4757334309663976,
+	6.4918530963296748,
+	6.5077946401986964,
+	6.5235619560570131,
+	6.5391588111080319,
+	6.5545888516776376,
+	6.5698556083309478,
+	6.5849625007211561,
+	6.5999128421871278,
+	6.6147098441152092,
+	6.6293566200796095,
+	6.6438561897747253,
+	6.6582114827517955,
+	6.6724253419714952,
+	6.6865005271832185,
+	6.7004397181410917,
+	6.7142455176661224,
+	6.7279204545631988,
+	6.7414669864011465,
+	6.7548875021634691,
+	6.7681843247769260,
+	6.7813597135246599,
+	6.7944158663501062,
+	6.8073549220576037,
+	6.8201789624151887,
+	6.8328900141647422,
+	6.8454900509443757,
+	6.8579809951275719,
+	6.8703647195834048,
+	6.8826430493618416,
+	6.8948177633079437,
+	6.9068905956085187,
+	6.9188632372745955,
+	6.9307373375628867,
+	6.9425145053392399,
+	6.9541963103868758,
+	6.9657842846620879,
+	6.9772799234999168,
+	6.9886846867721664,
+	7.0000000000000000,
+	7.0112272554232540,
+	7.0223678130284544,
+	7.0334230015374501,
+	7.0443941193584534,
+	7.0552824355011898,
+	7.0660891904577721,
+	7.0768155970508317,
+	7.0874628412503400,
+	7.0980320829605272,
+	7.1085244567781700,
+	7.1189410727235076,
+	7.1292830169449664,
+	7.1395513523987937,
+	7.1497471195046822,
+	7.1598713367783891,
+	7.1699250014423130,
+	7.1799090900149345,
+	7.1898245588800176,
+	7.1996723448363644,
+	7.2094533656289492,
+	7.2191685204621621,
+	7.2288186904958804,
+	7.2384047393250794,
+	7.2479275134435861,
+	7.2573878426926521,
+	7.2667865406949019,
+	7.2761244052742384,
+	7.2854022188622487,
+	7.2946207488916270,
+	7.3037807481771031,
+	7.3128829552843557,
+	7.3219280948873617,
+	7.3309168781146177,
+	7.3398500028846243,
+	7.3487281542310781,
+	7.3575520046180847,
+	7.3663222142458151,
+	7.3750394313469254,
+	7.3837042924740528,
+	7.3923174227787607,
+	7.4008794362821844,
+	7.4093909361377026,
+	7.4178525148858991,
+	7.4262647547020979,
+	7.4346282276367255,
+	7.4429434958487288,
+	7.4512111118323299,
+	7.4594316186372973,
+	7.4676055500829976,
+	7.4757334309663976,
+	7.4838157772642564,
+	7.4918530963296748,
+	7.4998458870832057,
+	7.5077946401986964,
+	7.5156998382840436,
+	7.5235619560570131,
+	7.5313814605163119,
+	7.5391588111080319,
+	7.5468944598876373,
+	7.5545888516776376,
+	7.5622424242210728,
+	7.5698556083309478,
+	7.5774288280357487,
+	7.5849625007211561,
+	7.5924570372680806,
+	7.5999128421871278,
+	7.6073303137496113,
+	7.6147098441152075,
+	7.6220518194563764,
+	7.6293566200796095,
+	7.6366246205436488,
+	7.6438561897747244,
+	7.6510516911789290,
+	7.6582114827517955,
+	7.6653359171851765,
+	7.6724253419714952,
+	7.6794800995054464,
+	7.6865005271832185,
+	7.6934869574993252,
+	7.7004397181410926,
+	7.7073591320808825,
+	7.7142455176661224,
+	7.7210991887071856,
+	7.7279204545631996,
+	7.7347096202258392,
+	7.7414669864011465,
+	7.7481928495894596,
+	7.7548875021634691,
+	7.7615512324444795,
+	7.7681843247769260,
+	7.7747870596011737,
+	7.7813597135246608,
+	7.7879025593914317,
+	7.7944158663501062,
+	7.8008998999203047,
+	7.8073549220576037,
+	7.8137811912170374,
+	7.8201789624151887,
+	7.8265484872909159,
+	7.8328900141647422,
+	7.8392037880969445,
+	7.8454900509443757,
+	7.8517490414160571,
+	7.8579809951275719,
+	7.8641861446542798,
+	7.8703647195834048,
+	7.8765169465650002,
+	7.8826430493618425,
+	7.8887432488982601,
+	7.8948177633079446,
+	7.9008668079807496,
+	7.9068905956085187,
+	7.9128893362299619,
+	7.9188632372745955,
+	7.9248125036057813,
+	7.9307373375628867,
+	7.9366379390025719,
+	7.9425145053392399,
+	7.9483672315846778,
+	7.9541963103868758,
+	7.9600019320680806,
+	7.9657842846620870,
+	7.9715435539507720,
+	7.9772799234999168,
+	7.9829935746943104,
+	7.9886846867721664,
+	7.9943534368588578,
+}
+
+/* Faster logarithm for small integers, with the property of log2(0) == 0. */
+func fastLog2(v uint) float64 {
+	if v < uint(len(kLog2Table)) {
+		return float64(kLog2Table[v])
+	}
+
+	return math.Log2(float64(v))
+}

vendor/github.com/andybalholm/brotli/find_match_length.go

@@ -0,0 +1,16 @@
+package brotli
+
+/* Copyright 2010 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* Function to find maximal matching prefixes of strings. */
+func findMatchLengthWithLimit(s1 []byte, s2 []byte, limit uint) uint {
+	var matched uint = 0
+	for matched < limit && s1[matched] == s2[matched] {
+		matched++
+	}
+	return matched
+}

vendor/github.com/andybalholm/brotli/go.mod

@@ -0,0 +1,3 @@
+module github.com/andybalholm/brotli
+
+go 1.12

vendor/github.com/andybalholm/brotli/h10.go

@@ -0,0 +1,287 @@
+package brotli
+
+import "encoding/binary"
+
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+func (*h10) HashTypeLength() uint {
+	return 4
+}
+
+func (*h10) StoreLookahead() uint {
+	return 128
+}
+
+func hashBytesH10(data []byte) uint32 {
+	var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32
+
+	/* The higher bits contain more mixture from the multiplication,
+	   so we take our results from there. */
+	return h >> (32 - 17)
+}
+
+/* A (forgetful) hash table where each hash bucket contains a binary tree of
+   sequences whose first 4 bytes share the same hash code.
+   Each sequence is 128 long and is identified by its starting
+   position in the input data. The binary tree is sorted by the lexicographic
+   order of the sequences, and it is also a max-heap with respect to the
+   starting positions. */
+type h10 struct {
+	hasherCommon
+	window_mask_ uint
+	buckets_     [1 << 17]uint32
+	invalid_pos_ uint32
+	forest       []uint32
+}
+
+func (h *h10) Initialize(params *encoderParams) {
+	h.window_mask_ = (1 << params.lgwin) - 1
+	h.invalid_pos_ = uint32(0 - h.window_mask_)
+	var num_nodes uint = uint(1) << params.lgwin
+	h.forest = make([]uint32, 2*num_nodes)
+}
+
+func (h *h10) Prepare(one_shot bool, input_size uint, data []byte) {
+	var invalid_pos uint32 = h.invalid_pos_
+	var i uint32
+	for i = 0; i < 1<<17; i++ {
+		h.buckets_[i] = invalid_pos
+	}
+}
+
+func leftChildIndexH10(self *h10, pos uint) uint {
+	return 2 * (pos & self.window_mask_)
+}
+
+func rightChildIndexH10(self *h10, pos uint) uint {
+	return 2*(pos&self.window_mask_) + 1
+}
+
+/* Stores the hash of the next 4 bytes and in a single tree-traversal, the
+   hash bucket's binary tree is searched for matches and is re-rooted at the
+   current position.
+
+   If less than 128 data is available, the hash bucket of the
+   current position is searched for matches, but the state of the hash table
+   is not changed, since we can not know the final sorting order of the
+   current (incomplete) sequence.
+
+   This function must be called with increasing cur_ix positions. */
+func storeAndFindMatchesH10(self *h10, data []byte, cur_ix uint, ring_buffer_mask uint, max_length uint, max_backward uint, best_len *uint, matches []backwardMatch) []backwardMatch {
+	var cur_ix_masked uint = cur_ix & ring_buffer_mask
+	var max_comp_len uint = brotli_min_size_t(max_length, 128)
+	var should_reroot_tree bool = (max_length >= 128)
+	var key uint32 = hashBytesH10(data[cur_ix_masked:])
+	var forest []uint32 = self.forest
+	var prev_ix uint = uint(self.buckets_[key])
+	var node_left uint = leftChildIndexH10(self, cur_ix)
+	var node_right uint = rightChildIndexH10(self, cur_ix)
+	var best_len_left uint = 0
+	var best_len_right uint = 0
+	var depth_remaining uint
+	/* The forest index of the rightmost node of the left subtree of the new
+	   root, updated as we traverse and re-root the tree of the hash bucket. */
+
+	/* The forest index of the leftmost node of the right subtree of the new
+	   root, updated as we traverse and re-root the tree of the hash bucket. */
+
+	/* The match length of the rightmost node of the left subtree of the new
+	   root, updated as we traverse and re-root the tree of the hash bucket. */
+
+	/* The match length of the leftmost node of the right subtree of the new
+	   root, updated as we traverse and re-root the tree of the hash bucket. */
+	if should_reroot_tree {
+		self.buckets_[key] = uint32(cur_ix)
+	}
+
+	for depth_remaining = 64; ; depth_remaining-- {
+		var backward uint = cur_ix - prev_ix
+		var prev_ix_masked uint = prev_ix & ring_buffer_mask
+		if backward == 0 || backward > max_backward || depth_remaining == 0 {
+			if should_reroot_tree {
+				forest[node_left] = self.invalid_pos_
+				forest[node_right] = self.invalid_pos_
+			}
+
+			break
+		}
+		{
+			var cur_len uint = brotli_min_size_t(best_len_left, best_len_right)
+			var len uint
+			assert(cur_len <= 128)
+			len = cur_len + findMatchLengthWithLimit(data[cur_ix_masked+cur_len:], data[prev_ix_masked+cur_len:], max_length-cur_len)
+			if matches != nil && len > *best_len {
+				*best_len = uint(len)
+				initBackwardMatch(&matches[0], backward, uint(len))
+				matches = matches[1:]
+			}
+
+			if len >= max_comp_len {
+				if should_reroot_tree {
+					forest[node_left] = forest[leftChildIndexH10(self, prev_ix)]
+					forest[node_right] = forest[rightChildIndexH10(self, prev_ix)]
+				}
+
+				break
+			}
+
+			if data[cur_ix_masked+len] > data[prev_ix_masked+len] {
+				best_len_left = uint(len)
+				if should_reroot_tree {
+					forest[node_left] = uint32(prev_ix)
+				}
+
+				node_left = rightChildIndexH10(self, prev_ix)
+				prev_ix = uint(forest[node_left])
+			} else {
+				best_len_right = uint(len)
+				if should_reroot_tree {
+					forest[node_right] = uint32(prev_ix)
+				}
+
+				node_right = leftChildIndexH10(self, prev_ix)
+				prev_ix = uint(forest[node_right])
+			}
+		}
+	}
+
+	return matches
+}
+
+/* Finds all backward matches of &data[cur_ix & ring_buffer_mask] up to the
+   length of max_length and stores the position cur_ix in the hash table.
+
+   Sets *num_matches to the number of matches found, and stores the found
+   matches in matches[0] to matches[*num_matches - 1]. The matches will be
+   sorted by strictly increasing length and (non-strictly) increasing
+   distance. */
+func findAllMatchesH10(handle *h10, dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, cur_ix uint, max_length uint, max_backward uint, gap uint, params *encoderParams, matches []backwardMatch) uint {
+	var orig_matches []backwardMatch = matches
+	var cur_ix_masked uint = cur_ix & ring_buffer_mask
+	var best_len uint = 1
+	var short_match_max_backward uint
+	if params.quality != hqZopflificationQuality {
+		short_match_max_backward = 16
+	} else {
+		short_match_max_backward = 64
+	}
+	var stop uint = cur_ix - short_match_max_backward
+	var dict_matches [maxStaticDictionaryMatchLen + 1]uint32
+	var i uint
+	if cur_ix < short_match_max_backward {
+		stop = 0
+	}
+	for i = cur_ix - 1; i > stop && best_len <= 2; i-- {
+		var prev_ix uint = i
+		var backward uint = cur_ix - prev_ix
+		if backward > max_backward {
+			break
+		}
+
+		prev_ix &= ring_buffer_mask
+		if data[cur_ix_masked] != data[prev_ix] || data[cur_ix_masked+1] != data[prev_ix+1] {
+			continue
+		}
+		{
+			var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
+			if len > best_len {
+				best_len = uint(len)
+				initBackwardMatch(&matches[0], backward, uint(len))
+				matches = matches[1:]
+			}
+		}
+	}
+
+	if best_len < max_length {
+		matches = storeAndFindMatchesH10(handle, data, cur_ix, ring_buffer_mask, max_length, max_backward, &best_len, matches)
+	}
+
+	for i = 0; i <= maxStaticDictionaryMatchLen; i++ {
+		dict_matches[i] = kInvalidMatch
+	}
+	{
+		var minlen uint = brotli_max_size_t(4, best_len+1)
+		if findAllStaticDictionaryMatches(dictionary, data[cur_ix_masked:], minlen, max_length, dict_matches[0:]) {
+			var maxlen uint = brotli_min_size_t(maxStaticDictionaryMatchLen, max_length)
+			var l uint
+			for l = minlen; l <= maxlen; l++ {
+				var dict_id uint32 = dict_matches[l]
+				if dict_id < kInvalidMatch {
+					var distance uint = max_backward + gap + uint(dict_id>>5) + 1
+					if distance <= params.dist.max_distance {
+						initDictionaryBackwardMatch(&matches[0], distance, l, uint(dict_id&31))
+						matches = matches[1:]
+					}
+				}
+			}
+		}
+	}
+
+	return uint(-cap(matches) + cap(orig_matches))
+}
+
+/* Stores the hash of the next 4 bytes and re-roots the binary tree at the
+   current sequence, without returning any matches.
+   REQUIRES: ix + 128 <= end-of-current-block */
+func (h *h10) Store(data []byte, mask uint, ix uint) {
+	var max_backward uint = h.window_mask_ - windowGap + 1
+	/* Maximum distance is window size - 16, see section 9.1. of the spec. */
+	storeAndFindMatchesH10(h, data, ix, mask, 128, max_backward, nil, nil)
+}
+
+func (h *h10) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
+	var i uint = ix_start
+	var j uint = ix_start
+	if ix_start+63 <= ix_end {
+		i = ix_end - 63
+	}
+
+	if ix_start+512 <= i {
+		for ; j < i; j += 8 {
+			h.Store(data, mask, j)
+		}
+	}
+
+	for ; i < ix_end; i++ {
+		h.Store(data, mask, i)
+	}
+}
+
+func (h *h10) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) {
+	if num_bytes >= h.HashTypeLength()-1 && position >= 128 {
+		var i_start uint = position - 128 + 1
+		var i_end uint = brotli_min_size_t(position, i_start+num_bytes)
+		/* Store the last `128 - 1` positions in the hasher.
+		   These could not be calculated before, since they require knowledge
+		   of both the previous and the current block. */
+
+		var i uint
+		for i = i_start; i < i_end; i++ {
+			/* Maximum distance is window size - 16, see section 9.1. of the spec.
+			   Furthermore, we have to make sure that we don't look further back
+			   from the start of the next block than the window size, otherwise we
+			   could access already overwritten areas of the ring-buffer. */
+			var max_backward uint = h.window_mask_ - brotli_max_size_t(windowGap-1, position-i)
+
+			/* We know that i + 128 <= position + num_bytes, i.e. the
+			   end of the current block and that we have at least
+			   128 tail in the ring-buffer. */
+			storeAndFindMatchesH10(h, ringbuffer, i, ringbuffer_mask, 128, max_backward, nil, nil)
+		}
+	}
+}
+
+/* MAX_NUM_MATCHES == 64 + MAX_TREE_SEARCH_DEPTH */
+const maxNumMatchesH10 = 128
+
+func (*h10) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
+	panic("unimplemented")
+}
+
+func (*h10) PrepareDistanceCache(distance_cache []int) {
+	panic("unimplemented")
+}

vendor/github.com/andybalholm/brotli/h5.go

@@ -0,0 +1,214 @@
+package brotli
+
+import "encoding/binary"
+
+/* Copyright 2010 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* A (forgetful) hash table to the data seen by the compressor, to
+   help create backward references to previous data.
+
+   This is a hash map of fixed size (bucket_size_) to a ring buffer of
+   fixed size (block_size_). The ring buffer contains the last block_size_
+   index positions of the given hash key in the compressed data. */
+func (*h5) HashTypeLength() uint {
+	return 4
+}
+
+func (*h5) StoreLookahead() uint {
+	return 4
+}
+
+/* HashBytes is the function that chooses the bucket to place the address in. */
+func hashBytesH5(data []byte, shift int) uint32 {
+	var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32
+
+	/* The higher bits contain more mixture from the multiplication,
+	   so we take our results from there. */
+	return uint32(h >> uint(shift))
+}
+
+type h5 struct {
+	hasherCommon
+	bucket_size_ uint
+	block_size_  uint
+	hash_shift_  int
+	block_mask_  uint32
+	num          []uint16
+	buckets      []uint32
+}
+
+func (h *h5) Initialize(params *encoderParams) {
+	h.hash_shift_ = 32 - h.params.bucket_bits
+	h.bucket_size_ = uint(1) << uint(h.params.bucket_bits)
+	h.block_size_ = uint(1) << uint(h.params.block_bits)
+	h.block_mask_ = uint32(h.block_size_ - 1)
+	h.num = make([]uint16, h.bucket_size_)
+	h.buckets = make([]uint32, h.block_size_*h.bucket_size_)
+}
+
+func (h *h5) Prepare(one_shot bool, input_size uint, data []byte) {
+	var num []uint16 = h.num
+	var partial_prepare_threshold uint = h.bucket_size_ >> 6
+	/* Partial preparation is 100 times slower (per socket). */
+	if one_shot && input_size <= partial_prepare_threshold {
+		var i uint
+		for i = 0; i < input_size; i++ {
+			var key uint32 = hashBytesH5(data[i:], h.hash_shift_)
+			num[key] = 0
+		}
+	} else {
+		for i := 0; i < int(h.bucket_size_); i++ {
+			num[i] = 0
+		}
+	}
+}
+
+/* Look at 4 bytes at &data[ix & mask].
+   Compute a hash from these, and store the value of ix at that position. */
+func (h *h5) Store(data []byte, mask uint, ix uint) {
+	var num []uint16 = h.num
+	var key uint32 = hashBytesH5(data[ix&mask:], h.hash_shift_)
+	var minor_ix uint = uint(num[key]) & uint(h.block_mask_)
+	var offset uint = minor_ix + uint(key<<uint(h.params.block_bits))
+	h.buckets[offset] = uint32(ix)
+	num[key]++
+}
+
+func (h *h5) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
+	var i uint
+	for i = ix_start; i < ix_end; i++ {
+		h.Store(data, mask, i)
+	}
+}
+
+func (h *h5) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) {
+	if num_bytes >= h.HashTypeLength()-1 && position >= 3 {
+		/* Prepare the hashes for three last bytes of the last write.
+		   These could not be calculated before, since they require knowledge
+		   of both the previous and the current block. */
+		h.Store(ringbuffer, ringbuffer_mask, position-3)
+		h.Store(ringbuffer, ringbuffer_mask, position-2)
+		h.Store(ringbuffer, ringbuffer_mask, position-1)
+	}
+}
+
+func (h *h5) PrepareDistanceCache(distance_cache []int) {
+	prepareDistanceCache(distance_cache, h.params.num_last_distances_to_check)
+}
+
+/* Find a longest backward match of &data[cur_ix] up to the length of
+   max_length and stores the position cur_ix in the hash table.
+
+   REQUIRES: PrepareDistanceCacheH5 must be invoked for current distance cache
+             values; if this method is invoked repeatedly with the same distance
+             cache values, it is enough to invoke PrepareDistanceCacheH5 once.
+
+   Does not look for matches longer than max_length.
+   Does not look for matches further away than max_backward.
+   Writes the best match into |out|.
+   |out|->score is updated only if a better match is found. */
+func (h *h5) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
+	var num []uint16 = h.num
+	var buckets []uint32 = h.buckets
+	var cur_ix_masked uint = cur_ix & ring_buffer_mask
+	var min_score uint = out.score
+	var best_score uint = out.score
+	var best_len uint = out.len
+	var i uint
+	var bucket []uint32
+	/* Don't accept a short copy from far away. */
+	out.len = 0
+
+	out.len_code_delta = 0
+
+	/* Try last distance first. */
+	for i = 0; i < uint(h.params.num_last_distances_to_check); i++ {
+		var backward uint = uint(distance_cache[i])
+		var prev_ix uint = uint(cur_ix - backward)
+		if prev_ix >= cur_ix {
+			continue
+		}
+
+		if backward > max_backward {
+			continue
+		}
+
+		prev_ix &= ring_buffer_mask
+
+		if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] {
+			continue
+		}
+		{
+			var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
+			if len >= 3 || (len == 2 && i < 2) {
+				/* Comparing for >= 2 does not change the semantics, but just saves for
+				   a few unnecessary binary logarithms in backward reference score,
+				   since we are not interested in such short matches. */
+				var score uint = backwardReferenceScoreUsingLastDistance(uint(len))
+				if best_score < score {
+					if i != 0 {
+						score -= backwardReferencePenaltyUsingLastDistance(i)
+					}
+					if best_score < score {
+						best_score = score
+						best_len = uint(len)
+						out.len = best_len
+						out.distance = backward
+						out.score = best_score
+					}
+				}
+			}
+		}
+	}
+	{
+		var key uint32 = hashBytesH5(data[cur_ix_masked:], h.hash_shift_)
+		bucket = buckets[key<<uint(h.params.block_bits):]
+		var down uint
+		if uint(num[key]) > h.block_size_ {
+			down = uint(num[key]) - h.block_size_
+		} else {
+			down = 0
+		}
+		for i = uint(num[key]); i > down; {
+			var prev_ix uint
+			i--
+			prev_ix = uint(bucket[uint32(i)&h.block_mask_])
+			var backward uint = cur_ix - prev_ix
+			if backward > max_backward {
+				break
+			}
+
+			prev_ix &= ring_buffer_mask
+			if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] {
+				continue
+			}
+			{
+				var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
+				if len >= 4 {
+					/* Comparing for >= 3 does not change the semantics, but just saves
+					   for a few unnecessary binary logarithms in backward reference
+					   score, since we are not interested in such short matches. */
+					var score uint = backwardReferenceScore(uint(len), backward)
+					if best_score < score {
+						best_score = score
+						best_len = uint(len)
+						out.len = best_len
+						out.distance = backward
+						out.score = best_score
+					}
+				}
+			}
+		}
+
+		bucket[uint32(num[key])&h.block_mask_] = uint32(cur_ix)
+		num[key]++
+	}
+
+	if min_score == out.score {
+		searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false)
+	}
+}

vendor/github.com/andybalholm/brotli/h6.go

@@ -0,0 +1,216 @@
+package brotli
+
+import "encoding/binary"
+
+/* Copyright 2010 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* A (forgetful) hash table to the data seen by the compressor, to
+   help create backward references to previous data.
+
+   This is a hash map of fixed size (bucket_size_) to a ring buffer of
+   fixed size (block_size_). The ring buffer contains the last block_size_
+   index positions of the given hash key in the compressed data. */
+func (*h6) HashTypeLength() uint {
+	return 8
+}
+
+func (*h6) StoreLookahead() uint {
+	return 8
+}
+
+/* HashBytes is the function that chooses the bucket to place the address in. */
+func hashBytesH6(data []byte, mask uint64, shift int) uint32 {
+	var h uint64 = (binary.LittleEndian.Uint64(data) & mask) * kHashMul64Long
+
+	/* The higher bits contain more mixture from the multiplication,
+	   so we take our results from there. */
+	return uint32(h >> uint(shift))
+}
+
+type h6 struct {
+	hasherCommon
+	bucket_size_ uint
+	block_size_  uint
+	hash_shift_  int
+	hash_mask_   uint64
+	block_mask_  uint32
+	num          []uint16
+	buckets      []uint32
+}
+
+func (h *h6) Initialize(params *encoderParams) {
+	h.hash_shift_ = 64 - h.params.bucket_bits
+	h.hash_mask_ = (^(uint64(0))) >> uint(64-8*h.params.hash_len)
+	h.bucket_size_ = uint(1) << uint(h.params.bucket_bits)
+	h.block_size_ = uint(1) << uint(h.params.block_bits)
+	h.block_mask_ = uint32(h.block_size_ - 1)
+	h.num = make([]uint16, h.bucket_size_)
+	h.buckets = make([]uint32, h.block_size_*h.bucket_size_)
+}
+
+func (h *h6) Prepare(one_shot bool, input_size uint, data []byte) {
+	var num []uint16 = h.num
+	var partial_prepare_threshold uint = h.bucket_size_ >> 6
+	/* Partial preparation is 100 times slower (per socket). */
+	if one_shot && input_size <= partial_prepare_threshold {
+		var i uint
+		for i = 0; i < input_size; i++ {
+			var key uint32 = hashBytesH6(data[i:], h.hash_mask_, h.hash_shift_)
+			num[key] = 0
+		}
+	} else {
+		for i := 0; i < int(h.bucket_size_); i++ {
+			num[i] = 0
+		}
+	}
+}
+
+/* Look at 4 bytes at &data[ix & mask].
+   Compute a hash from these, and store the value of ix at that position. */
+func (h *h6) Store(data []byte, mask uint, ix uint) {
+	var num []uint16 = h.num
+	var key uint32 = hashBytesH6(data[ix&mask:], h.hash_mask_, h.hash_shift_)
+	var minor_ix uint = uint(num[key]) & uint(h.block_mask_)
+	var offset uint = minor_ix + uint(key<<uint(h.params.block_bits))
+	h.buckets[offset] = uint32(ix)
+	num[key]++
+}
+
+func (h *h6) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
+	var i uint
+	for i = ix_start; i < ix_end; i++ {
+		h.Store(data, mask, i)
+	}
+}
+
+func (h *h6) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) {
+	if num_bytes >= h.HashTypeLength()-1 && position >= 3 {
+		/* Prepare the hashes for three last bytes of the last write.
+		   These could not be calculated before, since they require knowledge
+		   of both the previous and the current block. */
+		h.Store(ringbuffer, ringbuffer_mask, position-3)
+		h.Store(ringbuffer, ringbuffer_mask, position-2)
+		h.Store(ringbuffer, ringbuffer_mask, position-1)
+	}
+}
+
+func (h *h6) PrepareDistanceCache(distance_cache []int) {
+	prepareDistanceCache(distance_cache, h.params.num_last_distances_to_check)
+}
+
+/* Find a longest backward match of &data[cur_ix] up to the length of
+   max_length and stores the position cur_ix in the hash table.
+
+   REQUIRES: PrepareDistanceCacheH6 must be invoked for current distance cache
+             values; if this method is invoked repeatedly with the same distance
+             cache values, it is enough to invoke PrepareDistanceCacheH6 once.
+
+   Does not look for matches longer than max_length.
+   Does not look for matches further away than max_backward.
+   Writes the best match into |out|.
+   |out|->score is updated only if a better match is found. */
+func (h *h6) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
+	var num []uint16 = h.num
+	var buckets []uint32 = h.buckets
+	var cur_ix_masked uint = cur_ix & ring_buffer_mask
+	var min_score uint = out.score
+	var best_score uint = out.score
+	var best_len uint = out.len
+	var i uint
+	var bucket []uint32
+	/* Don't accept a short copy from far away. */
+	out.len = 0
+
+	out.len_code_delta = 0
+
+	/* Try last distance first. */
+	for i = 0; i < uint(h.params.num_last_distances_to_check); i++ {
+		var backward uint = uint(distance_cache[i])
+		var prev_ix uint = uint(cur_ix - backward)
+		if prev_ix >= cur_ix {
+			continue
+		}
+
+		if backward > max_backward {
+			continue
+		}
+
+		prev_ix &= ring_buffer_mask
+
+		if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] {
+			continue
+		}
+		{
+			var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
+			if len >= 3 || (len == 2 && i < 2) {
+				/* Comparing for >= 2 does not change the semantics, but just saves for
+				   a few unnecessary binary logarithms in backward reference score,
+				   since we are not interested in such short matches. */
+				var score uint = backwardReferenceScoreUsingLastDistance(uint(len))
+				if best_score < score {
+					if i != 0 {
+						score -= backwardReferencePenaltyUsingLastDistance(i)
+					}
+					if best_score < score {
+						best_score = score
+						best_len = uint(len)
+						out.len = best_len
+						out.distance = backward
+						out.score = best_score
+					}
+				}
+			}
+		}
+	}
+	{
+		var key uint32 = hashBytesH6(data[cur_ix_masked:], h.hash_mask_, h.hash_shift_)
+		bucket = buckets[key<<uint(h.params.block_bits):]
+		var down uint
+		if uint(num[key]) > h.block_size_ {
+			down = uint(num[key]) - h.block_size_
+		} else {
+			down = 0
+		}
+		for i = uint(num[key]); i > down; {
+			var prev_ix uint
+			i--
+			prev_ix = uint(bucket[uint32(i)&h.block_mask_])
+			var backward uint = cur_ix - prev_ix
+			if backward > max_backward {
+				break
+			}
+
+			prev_ix &= ring_buffer_mask
+			if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] {
+				continue
+			}
+			{
+				var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
+				if len >= 4 {
+					/* Comparing for >= 3 does not change the semantics, but just saves
+					   for a few unnecessary binary logarithms in backward reference
+					   score, since we are not interested in such short matches. */
+					var score uint = backwardReferenceScore(uint(len), backward)
+					if best_score < score {
+						best_score = score
+						best_len = uint(len)
+						out.len = best_len
+						out.distance = backward
+						out.score = best_score
+					}
+				}
+			}
+		}
+
+		bucket[uint32(num[key])&h.block_mask_] = uint32(cur_ix)
+		num[key]++
+	}
+
+	if min_score == out.score {
+		searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false)
+	}
+}

vendor/github.com/andybalholm/brotli/hash.go

@@ -0,0 +1,344 @@
+package brotli
+
+import (
+	"encoding/binary"
+	"fmt"
+)
+
+type hasherCommon struct {
+	params           hasherParams
+	is_prepared_     bool
+	dict_num_lookups uint
+	dict_num_matches uint
+}
+
+func (h *hasherCommon) Common() *hasherCommon {
+	return h
+}
+
+type hasherHandle interface {
+	Common() *hasherCommon
+	Initialize(params *encoderParams)
+	Prepare(one_shot bool, input_size uint, data []byte)
+	StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint)
+	HashTypeLength() uint
+	StoreLookahead() uint
+	PrepareDistanceCache(distance_cache []int)
+	FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult)
+	StoreRange(data []byte, mask uint, ix_start uint, ix_end uint)
+	Store(data []byte, mask uint, ix uint)
+}
+
+type score_t uint
+
+const kCutoffTransformsCount uint32 = 10
+
+/*   0,  12,   27,    23,    42,    63,    56,    48,    59,    64 */
+/* 0+0, 4+8, 8+19, 12+11, 16+26, 20+43, 24+32, 28+20, 32+27, 36+28 */
+const kCutoffTransforms uint64 = 0x071B520ADA2D3200
+
+type hasherSearchResult struct {
+	len            uint
+	distance       uint
+	score          uint
+	len_code_delta int
+}
+
+/* kHashMul32 multiplier has these properties:
+   * The multiplier must be odd. Otherwise we may lose the highest bit.
+   * No long streaks of ones or zeros.
+   * There is no effort to ensure that it is a prime, the oddity is enough
+     for this use.
+   * The number has been tuned heuristically against compression benchmarks. */
+const kHashMul32 uint32 = 0x1E35A7BD
+
+const kHashMul64 uint64 = 0x1E35A7BD1E35A7BD
+
+const kHashMul64Long uint64 = 0x1FE35A7BD3579BD3
+
+func hash14(data []byte) uint32 {
+	var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32
+
+	/* The higher bits contain more mixture from the multiplication,
+	   so we take our results from there. */
+	return h >> (32 - 14)
+}
+
+func prepareDistanceCache(distance_cache []int, num_distances int) {
+	if num_distances > 4 {
+		var last_distance int = distance_cache[0]
+		distance_cache[4] = last_distance - 1
+		distance_cache[5] = last_distance + 1
+		distance_cache[6] = last_distance - 2
+		distance_cache[7] = last_distance + 2
+		distance_cache[8] = last_distance - 3
+		distance_cache[9] = last_distance + 3
+		if num_distances > 10 {
+			var next_last_distance int = distance_cache[1]
+			distance_cache[10] = next_last_distance - 1
+			distance_cache[11] = next_last_distance + 1
+			distance_cache[12] = next_last_distance - 2
+			distance_cache[13] = next_last_distance + 2
+			distance_cache[14] = next_last_distance - 3
+			distance_cache[15] = next_last_distance + 3
+		}
+	}
+}
+
+const literalByteScore = 135
+
+const distanceBitPenalty = 30
+
+/* Score must be positive after applying maximal penalty. */
+const scoreBase = (distanceBitPenalty * 8 * 8)
+
+/* Usually, we always choose the longest backward reference. This function
+   allows for the exception of that rule.
+
+   If we choose a backward reference that is further away, it will
+   usually be coded with more bits. We approximate this by assuming
+   log2(distance). If the distance can be expressed in terms of the
+   last four distances, we use some heuristic constants to estimate
+   the bits cost. For the first up to four literals we use the bit
+   cost of the literals from the literal cost model, after that we
+   use the average bit cost of the cost model.
+
+   This function is used to sometimes discard a longer backward reference
+   when it is not much longer and the bit cost for encoding it is more
+   than the saved literals.
+
+   backward_reference_offset MUST be positive. */
+func backwardReferenceScore(copy_length uint, backward_reference_offset uint) uint {
+	return scoreBase + literalByteScore*uint(copy_length) - distanceBitPenalty*uint(log2FloorNonZero(backward_reference_offset))
+}
+
+func backwardReferenceScoreUsingLastDistance(copy_length uint) uint {
+	return literalByteScore*uint(copy_length) + scoreBase + 15
+}
+
+func backwardReferencePenaltyUsingLastDistance(distance_short_code uint) uint {
+	return uint(39) + ((0x1CA10 >> (distance_short_code & 0xE)) & 0xE)
+}
+
+func testStaticDictionaryItem(dictionary *encoderDictionary, item uint, data []byte, max_length uint, max_backward uint, max_distance uint, out *hasherSearchResult) bool {
+	var len uint
+	var word_idx uint
+	var offset uint
+	var matchlen uint
+	var backward uint
+	var score uint
+	len = item & 0x1F
+	word_idx = item >> 5
+	offset = uint(dictionary.words.offsets_by_length[len]) + len*word_idx
+	if len > max_length {
+		return false
+	}
+
+	matchlen = findMatchLengthWithLimit(data, dictionary.words.data[offset:], uint(len))
+	if matchlen+uint(dictionary.cutoffTransformsCount) <= len || matchlen == 0 {
+		return false
+	}
+	{
+		var cut uint = len - matchlen
+		var transform_id uint = (cut << 2) + uint((dictionary.cutoffTransforms>>(cut*6))&0x3F)
+		backward = max_backward + 1 + word_idx + (transform_id << dictionary.words.size_bits_by_length[len])
+	}
+
+	if backward > max_distance {
+		return false
+	}
+
+	score = backwardReferenceScore(matchlen, backward)
+	if score < out.score {
+		return false
+	}
+
+	out.len = matchlen
+	out.len_code_delta = int(len) - int(matchlen)
+	out.distance = backward
+	out.score = score
+	return true
+}
+
+func searchInStaticDictionary(dictionary *encoderDictionary, handle hasherHandle, data []byte, max_length uint, max_backward uint, max_distance uint, out *hasherSearchResult, shallow bool) {
+	var key uint
+	var i uint
+	var self *hasherCommon = handle.Common()
+	if self.dict_num_matches < self.dict_num_lookups>>7 {
+		return
+	}
+
+	key = uint(hash14(data) << 1)
+	for i = 0; ; (func() { i++; key++ })() {
+		var tmp uint
+		if shallow {
+			tmp = 1
+		} else {
+			tmp = 2
+		}
+		if i >= tmp {
+			break
+		}
+		var item uint = uint(dictionary.hash_table[key])
+		self.dict_num_lookups++
+		if item != 0 {
+			var item_matches bool = testStaticDictionaryItem(dictionary, item, data, max_length, max_backward, max_distance, out)
+			if item_matches {
+				self.dict_num_matches++
+			}
+		}
+	}
+}
+
+type backwardMatch struct {
+	distance        uint32
+	length_and_code uint32
+}
+
+func initBackwardMatch(self *backwardMatch, dist uint, len uint) {
+	self.distance = uint32(dist)
+	self.length_and_code = uint32(len << 5)
+}
+
+func initDictionaryBackwardMatch(self *backwardMatch, dist uint, len uint, len_code uint) {
+	self.distance = uint32(dist)
+	var tmp uint
+	if len == len_code {
+		tmp = 0
+	} else {
+		tmp = len_code
+	}
+	self.length_and_code = uint32(len<<5 | tmp)
+}
+
+func backwardMatchLength(self *backwardMatch) uint {
+	return uint(self.length_and_code >> 5)
+}
+
+func backwardMatchLengthCode(self *backwardMatch) uint {
+	var code uint = uint(self.length_and_code) & 31
+	if code != 0 {
+		return code
+	} else {
+		return backwardMatchLength(self)
+	}
+}
+
+func hasherReset(handle hasherHandle) {
+	if handle == nil {
+		return
+	}
+	handle.Common().is_prepared_ = false
+}
+
+func newHasher(typ int) hasherHandle {
+	switch typ {
+	case 2:
+		return &hashLongestMatchQuickly{
+			bucketBits:    16,
+			bucketSweep:   1,
+			hashLen:       5,
+			useDictionary: true,
+		}
+	case 3:
+		return &hashLongestMatchQuickly{
+			bucketBits:    16,
+			bucketSweep:   2,
+			hashLen:       5,
+			useDictionary: false,
+		}
+	case 4:
+		return &hashLongestMatchQuickly{
+			bucketBits:    17,
+			bucketSweep:   4,
+			hashLen:       5,
+			useDictionary: true,
+		}
+	case 5:
+		return new(h5)
+	case 6:
+		return new(h6)
+	case 10:
+		return new(h10)
+	case 35:
+		return &hashComposite{
+			ha: newHasher(3),
+			hb: &hashRolling{jump: 4},
+		}
+	case 40:
+		return &hashForgetfulChain{
+			bucketBits:              15,
+			numBanks:                1,
+			bankBits:                16,
+			numLastDistancesToCheck: 4,
+		}
+	case 41:
+		return &hashForgetfulChain{
+			bucketBits:              15,
+			numBanks:                1,
+			bankBits:                16,
+			numLastDistancesToCheck: 10,
+		}
+	case 42:
+		return &hashForgetfulChain{
+			bucketBits:              15,
+			numBanks:                512,
+			bankBits:                9,
+			numLastDistancesToCheck: 16,
+		}
+	case 54:
+		return &hashLongestMatchQuickly{
+			bucketBits:    20,
+			bucketSweep:   4,
+			hashLen:       7,
+			useDictionary: false,
+		}
+	case 55:
+		return &hashComposite{
+			ha: newHasher(54),
+			hb: &hashRolling{jump: 4},
+		}
+	case 65:
+		return &hashComposite{
+			ha: newHasher(6),
+			hb: &hashRolling{jump: 1},
+		}
+	}
+
+	panic(fmt.Sprintf("unknown hasher type: %d", typ))
+}
+
+func hasherSetup(handle *hasherHandle, params *encoderParams, data []byte, position uint, input_size uint, is_last bool) {
+	var self hasherHandle = nil
+	var common *hasherCommon = nil
+	var one_shot bool = (position == 0 && is_last)
+	if *handle == nil {
+		chooseHasher(params, &params.hasher)
+		self = newHasher(params.hasher.type_)
+
+		*handle = self
+		common = self.Common()
+		common.params = params.hasher
+		self.Initialize(params)
+	}
+
+	self = *handle
+	common = self.Common()
+	if !common.is_prepared_ {
+		self.Prepare(one_shot, input_size, data)
+
+		if position == 0 {
+			common.dict_num_lookups = 0
+			common.dict_num_matches = 0
+		}
+
+		common.is_prepared_ = true
+	}
+}
+
+func initOrStitchToPreviousBlock(handle *hasherHandle, data []byte, mask uint, params *encoderParams, position uint, input_size uint, is_last bool) {
+	var self hasherHandle
+	hasherSetup(handle, params, data, position, input_size, is_last)
+	self = *handle
+	self.StitchToPreviousBlock(input_size, position, data, mask)
+}

vendor/github.com/andybalholm/brotli/hash_composite.go

@@ -0,0 +1,93 @@
+package brotli
+
+/* Copyright 2018 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+func (h *hashComposite) HashTypeLength() uint {
+	var a uint = h.ha.HashTypeLength()
+	var b uint = h.hb.HashTypeLength()
+	if a > b {
+		return a
+	} else {
+		return b
+	}
+}
+
+func (h *hashComposite) StoreLookahead() uint {
+	var a uint = h.ha.StoreLookahead()
+	var b uint = h.hb.StoreLookahead()
+	if a > b {
+		return a
+	} else {
+		return b
+	}
+}
+
+/* Composite hasher: This hasher allows to combine two other hashers, HASHER_A
+   and HASHER_B. */
+type hashComposite struct {
+	hasherCommon
+	ha     hasherHandle
+	hb     hasherHandle
+	params *encoderParams
+}
+
+func (h *hashComposite) Initialize(params *encoderParams) {
+	h.params = params
+}
+
+/* TODO: Initialize of the hashers is defered to Prepare (and params
+   remembered here) because we don't get the one_shot and input_size params
+   here that are needed to know the memory size of them. Instead provide
+   those params to all hashers InitializehashComposite */
+func (h *hashComposite) Prepare(one_shot bool, input_size uint, data []byte) {
+	if h.ha == nil {
+		var common_a *hasherCommon
+		var common_b *hasherCommon
+
+		common_a = h.ha.Common()
+		common_a.params = h.params.hasher
+		common_a.is_prepared_ = false
+		common_a.dict_num_lookups = 0
+		common_a.dict_num_matches = 0
+		h.ha.Initialize(h.params)
+
+		common_b = h.hb.Common()
+		common_b.params = h.params.hasher
+		common_b.is_prepared_ = false
+		common_b.dict_num_lookups = 0
+		common_b.dict_num_matches = 0
+		h.hb.Initialize(h.params)
+	}
+
+	h.ha.Prepare(one_shot, input_size, data)
+	h.hb.Prepare(one_shot, input_size, data)
+}
+
+func (h *hashComposite) Store(data []byte, mask uint, ix uint) {
+	h.ha.Store(data, mask, ix)
+	h.hb.Store(data, mask, ix)
+}
+
+func (h *hashComposite) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
+	h.ha.StoreRange(data, mask, ix_start, ix_end)
+	h.hb.StoreRange(data, mask, ix_start, ix_end)
+}
+
+func (h *hashComposite) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ring_buffer_mask uint) {
+	h.ha.StitchToPreviousBlock(num_bytes, position, ringbuffer, ring_buffer_mask)
+	h.hb.StitchToPreviousBlock(num_bytes, position, ringbuffer, ring_buffer_mask)
+}
+
+func (h *hashComposite) PrepareDistanceCache(distance_cache []int) {
+	h.ha.PrepareDistanceCache(distance_cache)
+	h.hb.PrepareDistanceCache(distance_cache)
+}
+
+func (h *hashComposite) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
+	h.ha.FindLongestMatch(dictionary, data, ring_buffer_mask, distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out)
+	h.hb.FindLongestMatch(dictionary, data, ring_buffer_mask, distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out)
+}

vendor/github.com/andybalholm/brotli/hash_forgetful_chain.go

@@ -0,0 +1,253 @@
+package brotli
+
+import "encoding/binary"
+
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+func (*hashForgetfulChain) HashTypeLength() uint {
+	return 4
+}
+
+func (*hashForgetfulChain) StoreLookahead() uint {
+	return 4
+}
+
+/* HashBytes is the function that chooses the bucket to place the address in.*/
+func (h *hashForgetfulChain) HashBytes(data []byte) uint {
+	var hash uint32 = binary.LittleEndian.Uint32(data) * kHashMul32
+
+	/* The higher bits contain more mixture from the multiplication,
+	   so we take our results from there. */
+	return uint(hash >> (32 - h.bucketBits))
+}
+
+type slot struct {
+	delta uint16
+	next  uint16
+}
+
+/* A (forgetful) hash table to the data seen by the compressor, to
+   help create backward references to previous data.
+
+   Hashes are stored in chains which are bucketed to groups. Group of chains
+   share a storage "bank". When more than "bank size" chain nodes are added,
+   oldest nodes are replaced; this way several chains may share a tail. */
+type hashForgetfulChain struct {
+	hasherCommon
+
+	bucketBits              uint
+	numBanks                uint
+	bankBits                uint
+	numLastDistancesToCheck int
+
+	addr          []uint32
+	head          []uint16
+	tiny_hash     [65536]byte
+	banks         [][]slot
+	free_slot_idx []uint16
+	max_hops      uint
+}
+
+func (h *hashForgetfulChain) Initialize(params *encoderParams) {
+	var q uint
+	if params.quality > 6 {
+		q = 7
+	} else {
+		q = 8
+	}
+	h.max_hops = q << uint(params.quality-4)
+
+	bankSize := 1 << h.bankBits
+	bucketSize := 1 << h.bucketBits
+
+	h.addr = make([]uint32, bucketSize)
+	h.head = make([]uint16, bucketSize)
+	h.banks = make([][]slot, h.numBanks)
+	for i := range h.banks {
+		h.banks[i] = make([]slot, bankSize)
+	}
+	h.free_slot_idx = make([]uint16, h.numBanks)
+}
+
+func (h *hashForgetfulChain) Prepare(one_shot bool, input_size uint, data []byte) {
+	var partial_prepare_threshold uint = (1 << h.bucketBits) >> 6
+	/* Partial preparation is 100 times slower (per socket). */
+	if one_shot && input_size <= partial_prepare_threshold {
+		var i uint
+		for i = 0; i < input_size; i++ {
+			var bucket uint = h.HashBytes(data[i:])
+
+			/* See InitEmpty comment. */
+			h.addr[bucket] = 0xCCCCCCCC
+
+			h.head[bucket] = 0xCCCC
+		}
+	} else {
+		/* Fill |addr| array with 0xCCCCCCCC value. Because of wrapping, position
+		   processed by hasher never reaches 3GB + 64M; this makes all new chains
+		   to be terminated after the first node. */
+		for i := range h.addr {
+			h.addr[i] = 0xCCCCCCCC
+		}
+
+		for i := range h.head {
+			h.head[i] = 0
+		}
+	}
+
+	h.tiny_hash = [65536]byte{}
+	for i := range h.free_slot_idx {
+		h.free_slot_idx[i] = 0
+	}
+}
+
+/* Look at 4 bytes at &data[ix & mask]. Compute a hash from these, and prepend
+   node to corresponding chain; also update tiny_hash for current position. */
+func (h *hashForgetfulChain) Store(data []byte, mask uint, ix uint) {
+	var key uint = h.HashBytes(data[ix&mask:])
+	var bank uint = key & (h.numBanks - 1)
+	var idx uint
+	idx = uint(h.free_slot_idx[bank]) & ((1 << h.bankBits) - 1)
+	h.free_slot_idx[bank]++
+	var delta uint = ix - uint(h.addr[key])
+	h.tiny_hash[uint16(ix)] = byte(key)
+	if delta > 0xFFFF {
+		delta = 0xFFFF
+	}
+	h.banks[bank][idx].delta = uint16(delta)
+	h.banks[bank][idx].next = h.head[key]
+	h.addr[key] = uint32(ix)
+	h.head[key] = uint16(idx)
+}
+
+func (h *hashForgetfulChain) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
+	var i uint
+	for i = ix_start; i < ix_end; i++ {
+		h.Store(data, mask, i)
+	}
+}
+
+func (h *hashForgetfulChain) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ring_buffer_mask uint) {
+	if num_bytes >= h.HashTypeLength()-1 && position >= 3 {
+		/* Prepare the hashes for three last bytes of the last write.
+		   These could not be calculated before, since they require knowledge
+		   of both the previous and the current block. */
+		h.Store(ringbuffer, ring_buffer_mask, position-3)
+		h.Store(ringbuffer, ring_buffer_mask, position-2)
+		h.Store(ringbuffer, ring_buffer_mask, position-1)
+	}
+}
+
+func (h *hashForgetfulChain) PrepareDistanceCache(distance_cache []int) {
+	prepareDistanceCache(distance_cache, h.numLastDistancesToCheck)
+}
+
+/* Find a longest backward match of &data[cur_ix] up to the length of
+   max_length and stores the position cur_ix in the hash table.
+
+   REQUIRES: PrepareDistanceCachehashForgetfulChain must be invoked for current distance cache
+             values; if this method is invoked repeatedly with the same distance
+             cache values, it is enough to invoke PrepareDistanceCachehashForgetfulChain once.
+
+   Does not look for matches longer than max_length.
+   Does not look for matches further away than max_backward.
+   Writes the best match into |out|.
+   |out|->score is updated only if a better match is found. */
+func (h *hashForgetfulChain) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
+	var cur_ix_masked uint = cur_ix & ring_buffer_mask
+	var min_score uint = out.score
+	var best_score uint = out.score
+	var best_len uint = out.len
+	var key uint = h.HashBytes(data[cur_ix_masked:])
+	var tiny_hash byte = byte(key)
+	/* Don't accept a short copy from far away. */
+	out.len = 0
+
+	out.len_code_delta = 0
+
+	/* Try last distance first. */
+	for i := 0; i < h.numLastDistancesToCheck; i++ {
+		var backward uint = uint(distance_cache[i])
+		var prev_ix uint = (cur_ix - backward)
+
+		/* For distance code 0 we want to consider 2-byte matches. */
+		if i > 0 && h.tiny_hash[uint16(prev_ix)] != tiny_hash {
+			continue
+		}
+		if prev_ix >= cur_ix || backward > max_backward {
+			continue
+		}
+
+		prev_ix &= ring_buffer_mask
+		{
+			var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
+			if len >= 2 {
+				var score uint = backwardReferenceScoreUsingLastDistance(uint(len))
+				if best_score < score {
+					if i != 0 {
+						score -= backwardReferencePenaltyUsingLastDistance(uint(i))
+					}
+					if best_score < score {
+						best_score = score
+						best_len = uint(len)
+						out.len = best_len
+						out.distance = backward
+						out.score = best_score
+					}
+				}
+			}
+		}
+	}
+	{
+		var bank uint = key & (h.numBanks - 1)
+		var backward uint = 0
+		var hops uint = h.max_hops
+		var delta uint = cur_ix - uint(h.addr[key])
+		var slot uint = uint(h.head[key])
+		for {
+			tmp6 := hops
+			hops--
+			if tmp6 == 0 {
+				break
+			}
+			var prev_ix uint
+			var last uint = slot
+			backward += delta
+			if backward > max_backward {
+				break
+			}
+			prev_ix = (cur_ix - backward) & ring_buffer_mask
+			slot = uint(h.banks[bank][last].next)
+			delta = uint(h.banks[bank][last].delta)
+			if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] {
+				continue
+			}
+			{
+				var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
+				if len >= 4 {
+					/* Comparing for >= 3 does not change the semantics, but just saves
+					   for a few unnecessary binary logarithms in backward reference
+					   score, since we are not interested in such short matches. */
+					var score uint = backwardReferenceScore(uint(len), backward)
+					if best_score < score {
+						best_score = score
+						best_len = uint(len)
+						out.len = best_len
+						out.distance = backward
+						out.score = best_score
+					}
+				}
+			}
+		}
+
+		h.Store(data, ring_buffer_mask, cur_ix)
+	}
+
+	if out.score == min_score {
+		searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false)
+	}
+}

vendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go

@@ -0,0 +1,214 @@
+package brotli
+
+import "encoding/binary"
+
+/* Copyright 2010 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* For BUCKET_SWEEP == 1, enabling the dictionary lookup makes compression
+   a little faster (0.5% - 1%) and it compresses 0.15% better on small text
+   and HTML inputs. */
+
+func (*hashLongestMatchQuickly) HashTypeLength() uint {
+	return 8
+}
+
+func (*hashLongestMatchQuickly) StoreLookahead() uint {
+	return 8
+}
+
+/* HashBytes is the function that chooses the bucket to place
+   the address in. The HashLongestMatch and hashLongestMatchQuickly
+   classes have separate, different implementations of hashing. */
+func (h *hashLongestMatchQuickly) HashBytes(data []byte) uint32 {
+	var hash uint64 = ((binary.LittleEndian.Uint64(data) << (64 - 8*h.hashLen)) * kHashMul64)
+
+	/* The higher bits contain more mixture from the multiplication,
+	   so we take our results from there. */
+	return uint32(hash >> (64 - h.bucketBits))
+}
+
+/* A (forgetful) hash table to the data seen by the compressor, to
+   help create backward references to previous data.
+
+   This is a hash map of fixed size (1 << 16). Starting from the
+   given index, 1 buckets are used to store values of a key. */
+type hashLongestMatchQuickly struct {
+	hasherCommon
+
+	bucketBits    uint
+	bucketSweep   int
+	hashLen       uint
+	useDictionary bool
+
+	buckets []uint32
+}
+
+func (h *hashLongestMatchQuickly) Initialize(params *encoderParams) {
+	h.buckets = make([]uint32, 1<<h.bucketBits+h.bucketSweep)
+}
+
+func (h *hashLongestMatchQuickly) Prepare(one_shot bool, input_size uint, data []byte) {
+	var partial_prepare_threshold uint = (4 << h.bucketBits) >> 7
+	/* Partial preparation is 100 times slower (per socket). */
+	if one_shot && input_size <= partial_prepare_threshold {
+		var i uint
+		for i = 0; i < input_size; i++ {
+			var key uint32 = h.HashBytes(data[i:])
+			for j := 0; j < h.bucketSweep; j++ {
+				h.buckets[key+uint32(j)] = 0
+			}
+		}
+	} else {
+		/* It is not strictly necessary to fill this buffer here, but
+		   not filling will make the results of the compression stochastic
+		   (but correct). This is because random data would cause the
+		   system to find accidentally good backward references here and there. */
+		for i := range h.buckets {
+			h.buckets[i] = 0
+		}
+	}
+}
+
+/* Look at 5 bytes at &data[ix & mask].
+   Compute a hash from these, and store the value somewhere within
+   [ix .. ix+3]. */
+func (h *hashLongestMatchQuickly) Store(data []byte, mask uint, ix uint) {
+	var key uint32 = h.HashBytes(data[ix&mask:])
+	var off uint32 = uint32(ix>>3) % uint32(h.bucketSweep)
+	/* Wiggle the value with the bucket sweep range. */
+	h.buckets[key+off] = uint32(ix)
+}
+
+func (h *hashLongestMatchQuickly) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
+	var i uint
+	for i = ix_start; i < ix_end; i++ {
+		h.Store(data, mask, i)
+	}
+}
+
+func (h *hashLongestMatchQuickly) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) {
+	if num_bytes >= h.HashTypeLength()-1 && position >= 3 {
+		/* Prepare the hashes for three last bytes of the last write.
+		   These could not be calculated before, since they require knowledge
+		   of both the previous and the current block. */
+		h.Store(ringbuffer, ringbuffer_mask, position-3)
+		h.Store(ringbuffer, ringbuffer_mask, position-2)
+		h.Store(ringbuffer, ringbuffer_mask, position-1)
+	}
+}
+
+func (*hashLongestMatchQuickly) PrepareDistanceCache(distance_cache []int) {
+}
+
+/* Find a longest backward match of &data[cur_ix & ring_buffer_mask]
+   up to the length of max_length and stores the position cur_ix in the
+   hash table.
+
+   Does not look for matches longer than max_length.
+   Does not look for matches further away than max_backward.
+   Writes the best match into |out|.
+   |out|->score is updated only if a better match is found. */
+func (h *hashLongestMatchQuickly) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
+	var best_len_in uint = out.len
+	var cur_ix_masked uint = cur_ix & ring_buffer_mask
+	var key uint32 = h.HashBytes(data[cur_ix_masked:])
+	var compare_char int = int(data[cur_ix_masked+best_len_in])
+	var min_score uint = out.score
+	var best_score uint = out.score
+	var best_len uint = best_len_in
+	var cached_backward uint = uint(distance_cache[0])
+	var prev_ix uint = cur_ix - cached_backward
+	var bucket []uint32
+	out.len_code_delta = 0
+	if prev_ix < cur_ix {
+		prev_ix &= uint(uint32(ring_buffer_mask))
+		if compare_char == int(data[prev_ix+best_len]) {
+			var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
+			if len >= 4 {
+				var score uint = backwardReferenceScoreUsingLastDistance(uint(len))
+				if best_score < score {
+					best_score = score
+					best_len = uint(len)
+					out.len = uint(len)
+					out.distance = cached_backward
+					out.score = best_score
+					compare_char = int(data[cur_ix_masked+best_len])
+					if h.bucketSweep == 1 {
+						h.buckets[key] = uint32(cur_ix)
+						return
+					}
+				}
+			}
+		}
+	}
+
+	if h.bucketSweep == 1 {
+		var backward uint
+		var len uint
+
+		/* Only one to look for, don't bother to prepare for a loop. */
+		prev_ix = uint(h.buckets[key])
+
+		h.buckets[key] = uint32(cur_ix)
+		backward = cur_ix - prev_ix
+		prev_ix &= uint(uint32(ring_buffer_mask))
+		if compare_char != int(data[prev_ix+best_len_in]) {
+			return
+		}
+
+		if backward == 0 || backward > max_backward {
+			return
+		}
+
+		len = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
+		if len >= 4 {
+			var score uint = backwardReferenceScore(uint(len), backward)
+			if best_score < score {
+				out.len = uint(len)
+				out.distance = backward
+				out.score = score
+				return
+			}
+		}
+	} else {
+		bucket = h.buckets[key:]
+		var i int
+		prev_ix = uint(bucket[0])
+		bucket = bucket[1:]
+		for i = 0; i < h.bucketSweep; (func() { i++; tmp3 := bucket; bucket = bucket[1:]; prev_ix = uint(tmp3[0]) })() {
+			var backward uint = cur_ix - prev_ix
+			var len uint
+			prev_ix &= uint(uint32(ring_buffer_mask))
+			if compare_char != int(data[prev_ix+best_len]) {
+				continue
+			}
+
+			if backward == 0 || backward > max_backward {
+				continue
+			}
+
+			len = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
+			if len >= 4 {
+				var score uint = backwardReferenceScore(uint(len), backward)
+				if best_score < score {
+					best_score = score
+					best_len = uint(len)
+					out.len = best_len
+					out.distance = backward
+					out.score = score
+					compare_char = int(data[cur_ix_masked+best_len])
+				}
+			}
+		}
+	}
+
+	if h.useDictionary && min_score == out.score {
+		searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, true)
+	}
+
+	h.buckets[key+uint32((cur_ix>>3)%uint(h.bucketSweep))] = uint32(cur_ix)
+}

vendor/github.com/andybalholm/brotli/hash_rolling.go

@@ -0,0 +1,169 @@
+package brotli
+
+/* Copyright 2018 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+/* NOTE: this hasher does not search in the dictionary. It is used as
+   backup-hasher, the main hasher already searches in it. */
+
+const kRollingHashMul32 uint32 = 69069
+
+const kInvalidPosHashRolling uint32 = 0xffffffff
+
+/* This hasher uses a longer forward length, but returning a higher value here
+   will hurt compression by the main hasher when combined with a composite
+   hasher. The hasher tests for forward itself instead. */
+func (*hashRolling) HashTypeLength() uint {
+	return 4
+}
+
+func (*hashRolling) StoreLookahead() uint {
+	return 4
+}
+
+/* Computes a code from a single byte. A lookup table of 256 values could be
+   used, but simply adding 1 works about as good. */
+func (*hashRolling) HashByte(b byte) uint32 {
+	return uint32(b) + 1
+}
+
+func (h *hashRolling) HashRollingFunctionInitial(state uint32, add byte, factor uint32) uint32 {
+	return uint32(factor*state + h.HashByte(add))
+}
+
+func (h *hashRolling) HashRollingFunction(state uint32, add byte, rem byte, factor uint32, factor_remove uint32) uint32 {
+	return uint32(factor*state + h.HashByte(add) - factor_remove*h.HashByte(rem))
+}
+
+/* Rolling hash for long distance long string matches. Stores one position
+   per bucket, bucket key is computed over a long region. */
+type hashRolling struct {
+	hasherCommon
+
+	jump int
+
+	state         uint32
+	table         []uint32
+	next_ix       uint
+	chunk_len     uint32
+	factor        uint32
+	factor_remove uint32
+}
+
+func (h *hashRolling) Initialize(params *encoderParams) {
+	h.state = 0
+	h.next_ix = 0
+
+	h.factor = kRollingHashMul32
+
+	/* Compute the factor of the oldest byte to remove: factor**steps modulo
+	   0xffffffff (the multiplications rely on 32-bit overflow) */
+	h.factor_remove = 1
+
+	for i := 0; i < 32; i += h.jump {
+		h.factor_remove *= h.factor
+	}
+
+	h.table = make([]uint32, 16777216)
+	for i := 0; i < 16777216; i++ {
+		h.table[i] = kInvalidPosHashRolling
+	}
+}
+
+func (h *hashRolling) Prepare(one_shot bool, input_size uint, data []byte) {
+	/* Too small size, cannot use this hasher. */
+	if input_size < 32 {
+		return
+	}
+	h.state = 0
+	for i := 0; i < 32; i += h.jump {
+		h.state = h.HashRollingFunctionInitial(h.state, data[i], h.factor)
+	}
+}
+
+func (*hashRolling) Store(data []byte, mask uint, ix uint) {
+}
+
+func (*hashRolling) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
+}
+
+func (h *hashRolling) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ring_buffer_mask uint) {
+	var position_masked uint
+	/* In this case we must re-initialize the hasher from scratch from the
+	   current position. */
+
+	var available uint = num_bytes
+	if position&uint(h.jump-1) != 0 {
+		var diff uint = uint(h.jump) - (position & uint(h.jump-1))
+		if diff > available {
+			available = 0
+		} else {
+			available = available - diff
+		}
+		position += diff
+	}
+
+	position_masked = position & ring_buffer_mask
+
+	/* wrapping around ringbuffer not handled. */
+	if available > ring_buffer_mask-position_masked {
+		available = ring_buffer_mask - position_masked
+	}
+
+	h.Prepare(false, available, ringbuffer[position&ring_buffer_mask:])
+	h.next_ix = position
+}
+
+func (*hashRolling) PrepareDistanceCache(distance_cache []int) {
+}
+
+func (h *hashRolling) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
+	var cur_ix_masked uint = cur_ix & ring_buffer_mask
+	var pos uint = h.next_ix
+
+	if cur_ix&uint(h.jump-1) != 0 {
+		return
+	}
+
+	/* Not enough lookahead */
+	if max_length < 32 {
+		return
+	}
+
+	for pos = h.next_ix; pos <= cur_ix; pos += uint(h.jump) {
+		var code uint32 = h.state & ((16777216 * 64) - 1)
+		var rem byte = data[pos&ring_buffer_mask]
+		var add byte = data[(pos+32)&ring_buffer_mask]
+		var found_ix uint = uint(kInvalidPosHashRolling)
+
+		h.state = h.HashRollingFunction(h.state, add, rem, h.factor, h.factor_remove)
+
+		if code < 16777216 {
+			found_ix = uint(h.table[code])
+			h.table[code] = uint32(pos)
+			if pos == cur_ix && uint32(found_ix) != kInvalidPosHashRolling {
+				/* The cast to 32-bit makes backward distances up to 4GB work even
+				   if cur_ix is above 4GB, despite using 32-bit values in the table. */
+				var backward uint = uint(uint32(cur_ix - found_ix))
+				if backward <= max_backward {
+					var found_ix_masked uint = found_ix & ring_buffer_mask
+					var len uint = findMatchLengthWithLimit(data[found_ix_masked:], data[cur_ix_masked:], max_length)
+					if len >= 4 && len > out.len {
+						var score uint = backwardReferenceScore(uint(len), backward)
+						if score > out.score {
+							out.len = uint(len)
+							out.distance = backward
+							out.score = score
+							out.len_code_delta = 0
+						}
+					}
+				}
+			}
+		}
+	}
+
+	h.next_ix = cur_ix + uint(h.jump)
+}

vendor/github.com/andybalholm/brotli/histogram.go

@@ -0,0 +1,227 @@
+package brotli
+
+import "math"
+
+/* The distance symbols effectively used by "Large Window Brotli" (32-bit). */
+const numHistogramDistanceSymbols = 544
+
+type histogramLiteral struct {
+	data_        [numLiteralSymbols]uint32
+	total_count_ uint
+	bit_cost_    float64
+}
+
+func histogramClearLiteral(self *histogramLiteral) {
+	self.data_ = [numLiteralSymbols]uint32{}
+	self.total_count_ = 0
+	self.bit_cost_ = math.MaxFloat64
+}
+
+func clearHistogramsLiteral(array []histogramLiteral, length uint) {
+	var i uint
+	for i = 0; i < length; i++ {
+		histogramClearLiteral(&array[i:][0])
+	}
+}
+
+func histogramAddLiteral(self *histogramLiteral, val uint) {
+	self.data_[val]++
+	self.total_count_++
+}
+
+func histogramAddVectorLiteral(self *histogramLiteral, p []byte, n uint) {
+	self.total_count_ += n
+	n += 1
+	for {
+		n--
+		if n == 0 {
+			break
+		}
+		self.data_[p[0]]++
+		p = p[1:]
+	}
+}
+
+func histogramAddHistogramLiteral(self *histogramLiteral, v *histogramLiteral) {
+	var i uint
+	self.total_count_ += v.total_count_
+	for i = 0; i < numLiteralSymbols; i++ {
+		self.data_[i] += v.data_[i]
+	}
+}
+
+func histogramDataSizeLiteral() uint {
+	return numLiteralSymbols
+}
+
+type histogramCommand struct {
+	data_        [numCommandSymbols]uint32
+	total_count_ uint
+	bit_cost_    float64
+}
+
+func histogramClearCommand(self *histogramCommand) {
+	self.data_ = [numCommandSymbols]uint32{}
+	self.total_count_ = 0
+	self.bit_cost_ = math.MaxFloat64
+}
+
+func clearHistogramsCommand(array []histogramCommand, length uint) {
+	var i uint
+	for i = 0; i < length; i++ {
+		histogramClearCommand(&array[i:][0])
+	}
+}
+
+func histogramAddCommand(self *histogramCommand, val uint) {
+	self.data_[val]++
+	self.total_count_++
+}
+
+func histogramAddVectorCommand(self *histogramCommand, p []uint16, n uint) {
+	self.total_count_ += n
+	n += 1
+	for {
+		n--
+		if n == 0 {
+			break
+		}
+		self.data_[p[0]]++
+		p = p[1:]
+	}
+}
+
+func histogramAddHistogramCommand(self *histogramCommand, v *histogramCommand) {
+	var i uint
+	self.total_count_ += v.total_count_
+	for i = 0; i < numCommandSymbols; i++ {
+		self.data_[i] += v.data_[i]
+	}
+}
+
+func histogramDataSizeCommand() uint {
+	return numCommandSymbols
+}
+
+type histogramDistance struct {
+	data_        [numDistanceSymbols]uint32
+	total_count_ uint
+	bit_cost_    float64
+}
+
+func histogramClearDistance(self *histogramDistance) {
+	self.data_ = [numDistanceSymbols]uint32{}
+	self.total_count_ = 0
+	self.bit_cost_ = math.MaxFloat64
+}
+
+func clearHistogramsDistance(array []histogramDistance, length uint) {
+	var i uint
+	for i = 0; i < length; i++ {
+		histogramClearDistance(&array[i:][0])
+	}
+}
+
+func histogramAddDistance(self *histogramDistance, val uint) {
+	self.data_[val]++
+	self.total_count_++
+}
+
+func histogramAddVectorDistance(self *histogramDistance, p []uint16, n uint) {
+	self.total_count_ += n
+	n += 1
+	for {
+		n--
+		if n == 0 {
+			break
+		}
+		self.data_[p[0]]++
+		p = p[1:]
+	}
+}
+
+func histogramAddHistogramDistance(self *histogramDistance, v *histogramDistance) {
+	var i uint
+	self.total_count_ += v.total_count_
+	for i = 0; i < numDistanceSymbols; i++ {
+		self.data_[i] += v.data_[i]
+	}
+}
+
+func histogramDataSizeDistance() uint {
+	return numDistanceSymbols
+}
+
+type blockSplitIterator struct {
+	split_  *blockSplit
+	idx_    uint
+	type_   uint
+	length_ uint
+}
+
+func initBlockSplitIterator(self *blockSplitIterator, split *blockSplit) {
+	self.split_ = split
+	self.idx_ = 0
+	self.type_ = 0
+	if split.lengths != nil {
+		self.length_ = uint(split.lengths[0])
+	} else {
+		self.length_ = 0
+	}
+}
+
+func blockSplitIteratorNext(self *blockSplitIterator) {
+	if self.length_ == 0 {
+		self.idx_++
+		self.type_ = uint(self.split_.types[self.idx_])
+		self.length_ = uint(self.split_.lengths[self.idx_])
+	}
+
+	self.length_--
+}
+
+func buildHistogramsWithContext(cmds []command, num_commands uint, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit, ringbuffer []byte, start_pos uint, mask uint, prev_byte byte, prev_byte2 byte, context_modes []int, literal_histograms []histogramLiteral, insert_and_copy_histograms []histogramCommand, copy_dist_histograms []histogramDistance) {
+	var pos uint = start_pos
+	var literal_it blockSplitIterator
+	var insert_and_copy_it blockSplitIterator
+	var dist_it blockSplitIterator
+	var i uint
+
+	initBlockSplitIterator(&literal_it, literal_split)
+	initBlockSplitIterator(&insert_and_copy_it, insert_and_copy_split)
+	initBlockSplitIterator(&dist_it, dist_split)
+	for i = 0; i < num_commands; i++ {
+		var cmd *command = &cmds[i]
+		var j uint
+		blockSplitIteratorNext(&insert_and_copy_it)
+		histogramAddCommand(&insert_and_copy_histograms[insert_and_copy_it.type_], uint(cmd.cmd_prefix_))
+
+		/* TODO: unwrap iterator blocks. */
+		for j = uint(cmd.insert_len_); j != 0; j-- {
+			var context uint
+			blockSplitIteratorNext(&literal_it)
+			context = literal_it.type_
+			if context_modes != nil {
+				var lut contextLUT = getContextLUT(context_modes[context])
+				context = (context << literalContextBits) + uint(getContext(prev_byte, prev_byte2, lut))
+			}
+
+			histogramAddLiteral(&literal_histograms[context], uint(ringbuffer[pos&mask]))
+			prev_byte2 = prev_byte
+			prev_byte = ringbuffer[pos&mask]
+			pos++
+		}
+
+		pos += uint(commandCopyLen(cmd))
+		if commandCopyLen(cmd) != 0 {
+			prev_byte2 = ringbuffer[(pos-2)&mask]
+			prev_byte = ringbuffer[(pos-1)&mask]
+			if cmd.cmd_prefix_ >= 128 {
+				var context uint
+				blockSplitIteratorNext(&dist_it)
+				context = uint(uint32(dist_it.type_<<distanceContextBits) + commandDistanceContext(cmd))
+				histogramAddDistance(&copy_dist_histograms[context], uint(cmd.dist_prefix_)&0x3FF)
+			}
+		}
+	}
+}

vendor/github.com/andybalholm/brotli/http.go

@@ -0,0 +1,192 @@
+package brotli
+
+import (
+	"compress/gzip"
+	"io"
+	"net/http"
+	"strings"
+)
+
+// HTTPCompressor chooses a compression method (brotli, gzip, or none) based on
+// the Accept-Encoding header, sets the Content-Encoding header, and returns a
+// WriteCloser that implements that compression. The Close method must be called
+// before the current HTTP handler returns.
+//
+// Due to https://github.com/golang/go/issues/31753, the response will not be
+// compressed unless you set a Content-Type header before you call
+// HTTPCompressor.
+func HTTPCompressor(w http.ResponseWriter, r *http.Request) io.WriteCloser {
+	if w.Header().Get("Content-Type") == "" {
+		return nopCloser{w}
+	}
+
+	if w.Header().Get("Vary") == "" {
+		w.Header().Set("Vary", "Accept-Encoding")
+	}
+
+	encoding := negotiateContentEncoding(r, []string{"br", "gzip"})
+	switch encoding {
+	case "br":
+		w.Header().Set("Content-Encoding", "br")
+		return NewWriter(w)
+	case "gzip":
+		w.Header().Set("Content-Encoding", "gzip")
+		return gzip.NewWriter(w)
+	}
+	return nopCloser{w}
+}
+
+// negotiateContentEncoding returns the best offered content encoding for the
+// request's Accept-Encoding header. If two offers match with equal weight and
+// then the offer earlier in the list is preferred. If no offers are
+// acceptable, then "" is returned.
+func negotiateContentEncoding(r *http.Request, offers []string) string {
+	bestOffer := "identity"
+	bestQ := -1.0
+	specs := parseAccept(r.Header, "Accept-Encoding")
+	for _, offer := range offers {
+		for _, spec := range specs {
+			if spec.Q > bestQ &&
+				(spec.Value == "*" || spec.Value == offer) {
+				bestQ = spec.Q
+				bestOffer = offer
+			}
+		}
+	}
+	if bestQ == 0 {
+		bestOffer = ""
+	}
+	return bestOffer
+}
+
+// acceptSpec describes an Accept* header.
+type acceptSpec struct {
+	Value string
+	Q     float64
+}
+
+// parseAccept parses Accept* headers.
+func parseAccept(header http.Header, key string) (specs []acceptSpec) {
+loop:
+	for _, s := range header[key] {
+		for {
+			var spec acceptSpec
+			spec.Value, s = expectTokenSlash(s)
+			if spec.Value == "" {
+				continue loop
+			}
+			spec.Q = 1.0
+			s = skipSpace(s)
+			if strings.HasPrefix(s, ";") {
+				s = skipSpace(s[1:])
+				if !strings.HasPrefix(s, "q=") {
+					continue loop
+				}
+				spec.Q, s = expectQuality(s[2:])
+				if spec.Q < 0.0 {
+					continue loop
+				}
+			}
+			specs = append(specs, spec)
+			s = skipSpace(s)
+			if !strings.HasPrefix(s, ",") {
+				continue loop
+			}
+			s = skipSpace(s[1:])
+		}
+	}
+	return
+}
+
+func skipSpace(s string) (rest string) {
+	i := 0
+	for ; i < len(s); i++ {
+		if octetTypes[s[i]]&isSpace == 0 {
+			break
+		}
+	}
+	return s[i:]
+}
+
+func expectTokenSlash(s string) (token, rest string) {
+	i := 0
+	for ; i < len(s); i++ {
+		b := s[i]
+		if (octetTypes[b]&isToken == 0) && b != '/' {
+			break
+		}
+	}
+	return s[:i], s[i:]
+}
+
+func expectQuality(s string) (q float64, rest string) {
+	switch {
+	case len(s) == 0:
+		return -1, ""
+	case s[0] == '0':
+		q = 0
+	case s[0] == '1':
+		q = 1
+	default:
+		return -1, ""
+	}
+	s = s[1:]
+	if !strings.HasPrefix(s, ".") {
+		return q, s
+	}
+	s = s[1:]
+	i := 0
+	n := 0
+	d := 1
+	for ; i < len(s); i++ {
+		b := s[i]
+		if b < '0' || b > '9' {
+			break
+		}
+		n = n*10 + int(b) - '0'
+		d *= 10
+	}
+	return q + float64(n)/float64(d), s[i:]
+}
+
+// Octet types from RFC 2616.
+var octetTypes [256]octetType
+
+type octetType byte
+
+const (
+	isToken octetType = 1 << iota
+	isSpace
+)
+
+func init() {
+	// OCTET      = <any 8-bit sequence of data>
+	// CHAR       = <any US-ASCII character (octets 0 - 127)>
+	// CTL        = <any US-ASCII control character (octets 0 - 31) and DEL (127)>
+	// CR         = <US-ASCII CR, carriage return (13)>
+	// LF         = <US-ASCII LF, linefeed (10)>
+	// SP         = <US-ASCII SP, space (32)>
+	// HT         = <US-ASCII HT, horizontal-tab (9)>
+	// <">        = <US-ASCII double-quote mark (34)>
+	// CRLF       = CR LF
+	// LWS        = [CRLF] 1*( SP | HT )
+	// TEXT       = <any OCTET except CTLs, but including LWS>
+	// separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <">
+	//              | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT
+	// token      = 1*<any CHAR except CTLs or separators>
+	// qdtext     = <any TEXT except <">>
+
+	for c := 0; c < 256; c++ {
+		var t octetType
+		isCtl := c <= 31 || c == 127
+		isChar := 0 <= c && c <= 127
+		isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0
+		if strings.IndexRune(" \t\r\n", rune(c)) >= 0 {
+			t |= isSpace
+		}
+		if isChar && !isCtl && !isSeparator {
+			t |= isToken
+		}
+		octetTypes[c] = t
+	}
+}