Note: We keep splitio/go-client untouched
because the dependency is broken.

See https://github.com/mattermost/mattermost-server/pull/18604

```release-note
NONE
```
Этот коммит содержится в:
Agniva De Sarker
2021-11-24 19:45:28 +05:30
коммит произвёл GitHub
родитель fd8fea804b
Коммит 189d447591
362 изменённых файлов: 66284 добавлений и 19348 удалений

65
go.mod
Просмотреть файл

@@ -6,18 +6,17 @@ require (
code.sajari.com/docconv v1.2.0
github.com/HdrHistogram/hdrhistogram-go v0.9.0 // indirect
github.com/Masterminds/semver/v3 v3.1.1
github.com/Masterminds/squirrel v1.5.0
github.com/PuerkitoBio/goquery v1.7.1 // indirect
github.com/Masterminds/squirrel v1.5.1
github.com/PuerkitoBio/goquery v1.8.0 // indirect
github.com/advancedlogic/GoOse v0.0.0-20210820140952-9d5822d4a625 // indirect
github.com/andybalholm/brotli v1.0.3 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect
github.com/armon/go-metrics v0.3.9 // indirect
github.com/armon/go-metrics v0.3.10 // indirect
github.com/avct/uasurfer v0.0.0-20191028135549-26b5daa857f1
github.com/aws/aws-sdk-go v1.40.57
github.com/aws/aws-sdk-go v1.42.11
github.com/bits-and-blooms/bitset v1.2.1 // indirect
github.com/blang/semver v3.5.1+incompatible
github.com/blevesearch/bleve/v2 v2.2.1
github.com/blevesearch/bleve/v2 v2.2.2
github.com/cespare/xxhash/v2 v2.1.2
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/dgryski/dgoogauth v0.0.0-20190221195224-5a805980a5f3
@@ -31,19 +30,16 @@ require (
github.com/getsentry/sentry-go v0.11.0
github.com/go-asn1-ber/asn1-ber v1.5.3 // indirect
github.com/go-redis/redis/v8 v8.11.4 // indirect
github.com/go-resty/resty/v2 v2.6.0 // indirect
github.com/go-resty/resty/v2 v2.7.0 // indirect
github.com/go-sql-driver/mysql v1.6.0
github.com/golang-migrate/migrate/v4 v4.15.0
github.com/golang-migrate/migrate/v4 v4.15.1
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gopherjs/gopherjs v0.0.0-20211004101933-6b77bd30416d // indirect
github.com/gopherjs/gopherjs v0.0.0-20211111143520-d0d5ecc1a356 // indirect
github.com/gorilla/handlers v1.5.1
github.com/gorilla/mux v1.8.0
github.com/gorilla/schema v1.2.0
github.com/gorilla/websocket v1.4.2
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c
github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b
github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -54,17 +50,15 @@ require (
github.com/hashicorp/go-plugin v1.4.3
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/memberlist v0.2.4
github.com/hashicorp/yamux v0.0.0-20210826001029-26ff87cf9493 // indirect
github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7
github.com/hashicorp/memberlist v0.3.0
github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 // indirect
github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba
github.com/jmoiron/sqlx v1.3.4
github.com/jonboulle/clockwork v0.2.2
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.13.6 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
github.com/ledongthuc/pdf v0.0.0-20210621053716-e28cb8259002
github.com/lib/pq v1.10.3
github.com/lib/pq v1.10.4
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattermost/go-i18n v1.11.1-0.20211013152124-5c415071e404
github.com/mattermost/gorp v1.6.2-0.20210714143452-8b50f5209a7f
@@ -73,15 +67,14 @@ require (
github.com/mattermost/ldap v0.0.0-20201202150706-ee0e6284187d
github.com/mattermost/logr/v2 v2.0.15
github.com/mattermost/rsc v0.0.0-20160330161541-bbaefb05eaa0
github.com/mattn/go-colorable v0.1.11 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/mholt/archiver/v3 v3.5.0
github.com/mholt/archiver/v3 v3.5.1
github.com/miekg/dns v1.1.43 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/minio-go/v7 v7.0.14
github.com/minio/minio-go/v7 v7.0.16
github.com/minio/sha256-simd v1.0.0 // indirect
github.com/mitchellh/go-testing-interface v1.14.1 // indirect
github.com/muesli/smartcrop v0.3.0 // indirect
github.com/nwaples/rardecode v1.1.2 // indirect
github.com/oklog/run v1.1.0 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
@@ -91,11 +84,11 @@ require (
github.com/otiai10/gosseract/v2 v2.3.1 // indirect
github.com/pborman/uuid v1.2.1
github.com/pelletier/go-toml v1.9.4 // indirect
github.com/pierrec/lz4/v4 v4.1.8 // indirect
github.com/pierrec/lz4/v4 v4.1.11 // indirect
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.11.0
github.com/prometheus/client_model v0.2.0
github.com/prometheus/common v0.31.1 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/reflog/dateconstraints v0.2.1
github.com/rs/cors v1.8.0
@@ -109,34 +102,34 @@ require (
github.com/stretchr/objx v0.3.0 // indirect
github.com/stretchr/testify v1.7.0
github.com/throttled/throttled v2.2.5+incompatible
github.com/tidwall/gjson v1.9.2 // indirect
github.com/tidwall/gjson v1.11.0 // indirect
github.com/tinylib/msgp v1.1.6
github.com/tylerb/graceful v1.2.15
github.com/uber/jaeger-client-go v2.29.1+incompatible
github.com/uber/jaeger-lib v2.4.1+incompatible
github.com/ulikunitz/xz v0.5.10 // indirect
github.com/vmihailenco/msgpack/v5 v5.3.4
github.com/vmihailenco/msgpack/v5 v5.3.5
github.com/wiggin77/merror v1.0.3
github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c
github.com/yuin/goldmark v1.4.1
github.com/yuin/goldmark v1.4.4
go.etcd.io/bbolt v1.3.6 // indirect
go.uber.org/atomic v1.9.0 // indirect
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d
golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871
golang.org/x/image v0.0.0-20211028202545-6944b10bf410
golang.org/x/mod v0.5.1 // indirect
golang.org/x/net v0.0.0-20211006190231-62292e806868
golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20211006225509-1a26e0398eed // indirect
golang.org/x/sys v0.0.0-20211123173158-ef496fb156ab // indirect
golang.org/x/text v0.3.7
golang.org/x/tools v0.1.7
google.golang.org/genproto v0.0.0-20211005153810-c76a74d43a8e // indirect
google.golang.org/grpc v1.41.0 // indirect
google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1 // indirect
google.golang.org/grpc v1.42.0 // indirect
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/ini.v1 v1.63.2 // indirect
gopkg.in/ini.v1 v1.64.0 // indirect
gopkg.in/mail.v2 v2.3.1
gopkg.in/olivere/elastic.v6 v6.2.37
gopkg.in/yaml.v2 v2.4.0
willnorris.com/go/imageproxy v0.10.0
willnorris.com/go/imageproxy v0.11.2
)
// Hack to prevent the willf/bitset module from being upgraded to 1.2.0.

774
go.sum

Разница между файлами не показана из-за своего большого размера Загрузить разницу

4
vendor/github.com/Masterminds/squirrel/README.md сгенерированный поставляемый
Просмотреть файл

@@ -1,4 +1,4 @@
[![Project Status: Inactive – The project has reached a stable, usable state but is no longer being actively developed; support/maintenance will be provided as time allows.](https://www.repostatus.org/badges/latest/inactive.svg)](https://www.repostatus.org/#inactive)
[![Stability: Maintenance](https://masterminds.github.io/stability/maintenance.svg)](https://masterminds.github.io/stability/maintenance.html)
### Squirrel is "complete".
Bug fixes will still be merged (slowly). Bug reports are welcome, but I will not necessarily respond to them. If another fork (or substantially similar project) actively improves on what Squirrel does, let me know and I may link to it here.
@@ -64,7 +64,7 @@ Squirrel wants to make your life easier:
```go
// StmtCache caches Prepared Stmts for you
dbCache := sq.NewStmtCacher(db)
dbCache := sq.NewStmtCache(db)
// StatementBuilder keeps your syntax neat
mydb := sq.StatementBuilder.RunWith(dbCache)

12
vendor/github.com/Masterminds/squirrel/case.go сгенерированный поставляемый
Просмотреть файл

@@ -27,7 +27,7 @@ func (b *sqlizerBuffer) WriteSql(item Sqlizer) {
var str string
var args []interface{}
str, args, b.err = item.ToSql()
str, args, b.err = nestedToSql(item)
if b.err != nil {
return
@@ -100,6 +100,16 @@ func (b CaseBuilder) ToSql() (string, []interface{}, error) {
return data.ToSql()
}
// MustSql builds the query into a SQL string and bound args.
// It panics if there are any errors.
func (b CaseBuilder) MustSql() (string, []interface{}) {
sql, args, err := b.ToSql()
if err != nil {
panic(err)
}
return sql, args
}
// what sets optional value for CASE construct "CASE [value] ..."
func (b CaseBuilder) what(expr interface{}) CaseBuilder {
return builder.Set(b, "What", newPart(expr)).(CaseBuilder)

10
vendor/github.com/Masterminds/squirrel/delete.go сгенерированный поставляемый
Просмотреть файл

@@ -121,6 +121,16 @@ func (b DeleteBuilder) ToSql() (string, []interface{}, error) {
return data.ToSql()
}
// MustSql builds the query into a SQL string and bound args.
// It panics if there are any errors.
func (b DeleteBuilder) MustSql() (string, []interface{}) {
sql, args, err := b.ToSql()
if err != nil {
panic(err)
}
return sql, args
}
// Prefix adds an expression to the beginning of the query
func (b DeleteBuilder) Prefix(sql string, args ...interface{}) DeleteBuilder {
return b.PrefixExpr(Expr(sql, args...))

10
vendor/github.com/Masterminds/squirrel/insert.go сгенерированный поставляемый
Просмотреть файл

@@ -216,6 +216,16 @@ func (b InsertBuilder) ToSql() (string, []interface{}, error) {
return data.ToSql()
}
// MustSql builds the query into a SQL string and bound args.
// It panics if there are any errors.
func (b InsertBuilder) MustSql() (string, []interface{}) {
sql, args, err := b.ToSql()
if err != nil {
panic(err)
}
return sql, args
}
// Prefix adds an expression to the beginning of the query
func (b InsertBuilder) Prefix(sql string, args ...interface{}) InsertBuilder {
return b.PrefixExpr(Expr(sql, args...))

10
vendor/github.com/Masterminds/squirrel/part.go сгенерированный поставляемый
Просмотреть файл

@@ -29,9 +29,17 @@ func (p part) ToSql() (sql string, args []interface{}, err error) {
return
}
func nestedToSql(s Sqlizer) (string, []interface{}, error) {
if raw, ok := s.(rawSqlizer); ok {
return raw.toSqlRaw()
} else {
return s.ToSql()
}
}
func appendToSql(parts []Sqlizer, w io.Writer, sep string, args []interface{}) ([]interface{}, error) {
for i, p := range parts {
partSql, partArgs, err := p.ToSql()
partSql, partArgs, err := nestedToSql(p)
if err != nil {
return nil, err
} else if len(partSql) == 0 {

18
vendor/github.com/Masterminds/squirrel/select.go сгенерированный поставляемый
Просмотреть файл

@@ -52,7 +52,7 @@ func (d *selectData) QueryRow() RowScanner {
}
func (d *selectData) ToSql() (sqlStr string, args []interface{}, err error) {
sqlStr, args, err = d.toSql()
sqlStr, args, err = d.toSqlRaw()
if err != nil {
return
}
@@ -62,10 +62,6 @@ func (d *selectData) ToSql() (sqlStr string, args []interface{}, err error) {
}
func (d *selectData) toSqlRaw() (sqlStr string, args []interface{}, err error) {
return d.toSql()
}
func (d *selectData) toSql() (sqlStr string, args []interface{}, err error) {
if len(d.Columns) == 0 {
err = fmt.Errorf("select statements must have at least one result column")
return
@@ -222,6 +218,13 @@ func (b SelectBuilder) ToSql() (string, []interface{}, error) {
return data.ToSql()
}
func (b SelectBuilder) toSqlRaw() (string, []interface{}, error) {
data := builder.GetStruct(b).(selectData)
return data.toSqlRaw()
}
// MustSql builds the query into a SQL string and bound args.
// It panics if there are any errors.
func (b SelectBuilder) MustSql() (string, []interface{}) {
sql, args, err := b.ToSql()
if err != nil {
@@ -230,11 +233,6 @@ func (b SelectBuilder) MustSql() (string, []interface{}) {
return sql, args
}
func (b SelectBuilder) toSqlRaw() (string, []interface{}, error) {
data := builder.GetStruct(b).(selectData)
return data.toSqlRaw()
}
// Prefix adds an expression to the beginning of the query
func (b SelectBuilder) Prefix(sql string, args ...interface{}) SelectBuilder {
return b.PrefixExpr(Expr(sql, args...))

10
vendor/github.com/Masterminds/squirrel/update.go сгенерированный поставляемый
Просмотреть файл

@@ -187,6 +187,16 @@ func (b UpdateBuilder) ToSql() (string, []interface{}, error) {
return data.ToSql()
}
// MustSql builds the query into a SQL string and bound args.
// It panics if there are any errors.
func (b UpdateBuilder) MustSql() (string, []interface{}) {
sql, args, err := b.ToSql()
if err != nil {
panic(err)
}
return sql, args
}
// Prefix adds an expression to the beginning of the query
func (b UpdateBuilder) Prefix(sql string, args ...interface{}) UpdateBuilder {
return b.PrefixExpr(Expr(sql, args...))

31
vendor/github.com/PuerkitoBio/goquery/.travis.yml сгенерированный поставляемый
Просмотреть файл

@@ -1,31 +0,0 @@
arch:
- amd64
- ppc64le
language: go
go:
- 1.7.x
- 1.8.x
- 1.9.x
- 1.10.x
- 1.11.x
- 1.12.x
- 1.13.x
- 1.14.x
- 1.15.x
- tip
jobs:
exclude:
- arch: ppc64le
go: 1.7.x
- arch: ppc64le
go: 1.8.x
- arch: ppc64le
go: 1.9.x
- arch: ppc64le
go: 1.10.x
- arch: ppc64le
go: 1.11.x
- arch: ppc64le
go: 1.12.x

4
vendor/github.com/PuerkitoBio/goquery/README.md сгенерированный поставляемый
Просмотреть файл

@@ -1,7 +1,6 @@
# goquery - a little like that j-thing, only in Go
[![builds.sr.ht status](https://builds.sr.ht/~mna/goquery/commits/fedora.yml.svg)](https://builds.sr.ht/~mna/goquery/commits/fedora.yml?)
[![build status](https://secure.travis-ci.org/PuerkitoBio/goquery.svg?branch=master)](http://travis-ci.org/PuerkitoBio/goquery)
[![Build Status](https://github.com/PuerkitoBio/goquery/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/PuerkitoBio/goquery/actions)
[![Go Reference](https://pkg.go.dev/badge/github.com/PuerkitoBio/goquery.svg)](https://pkg.go.dev/github.com/PuerkitoBio/goquery)
[![Sourcegraph Badge](https://sourcegraph.com/github.com/PuerkitoBio/goquery/-/badge.svg)](https://sourcegraph.com/github.com/PuerkitoBio/goquery?badge)
@@ -41,6 +40,7 @@ Please note that because of the net/html dependency, goquery requires Go1.1+ and
**Note that goquery's API is now stable, and will not break.**
* **2021-10-25 (v1.8.0)** : Add `Render` function to render a `Selection` to an `io.Writer` (thanks [@anthonygedeon](https://github.com/anthonygedeon)).
* **2021-07-11 (v1.7.1)** : Update go.mod dependencies and add dependabot config (thanks [@jauderho](https://github.com/jauderho)).
* **2021-06-14 (v1.7.0)** : Add `Single` and `SingleMatcher` functions to optimize first-match selection (thanks [@gdollardollar](https://github.com/gdollardollar)).
* **2021-01-11 (v1.6.1)** : Fix panic when calling `{Prepend,Append,Set}Html` on a `Selection` that contains non-Element nodes.

4
vendor/github.com/PuerkitoBio/goquery/go.mod сгенерированный поставляемый
Просмотреть файл

@@ -1,8 +1,8 @@
module github.com/PuerkitoBio/goquery
require (
github.com/andybalholm/cascadia v1.2.0
golang.org/x/net v0.0.0-20210614182718-04defd469f4e
github.com/andybalholm/cascadia v1.3.1
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8
)
go 1.13

9
vendor/github.com/PuerkitoBio/goquery/go.sum сгенерированный поставляемый
Просмотреть файл

@@ -1,8 +1,7 @@
github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE=
github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

21
vendor/github.com/PuerkitoBio/goquery/utilities.go сгенерированный поставляемый
Просмотреть файл

@@ -2,6 +2,7 @@ package goquery
import (
"bytes"
"io"
"golang.org/x/net/html"
)
@@ -50,13 +51,24 @@ func nodeName(node *html.Node) string {
case html.ElementNode, html.DoctypeNode:
return node.Data
default:
if node.Type >= 0 && int(node.Type) < len(nodeNames) {
if int(node.Type) < len(nodeNames) {
return nodeNames[node.Type]
}
return ""
}
}
// Render renders the html of the first element from selector and writes it to
// the writer. It behaves the same as OuterHtml but writes to w instead of
// returning the string.
func Render(w io.Writer, s *Selection) error {
if s.Length() == 0 {
return nil
}
n := s.Get(0)
return html.Render(w, n)
}
// OuterHtml returns the outer HTML rendering of the first item in
// the selection - that is, the HTML including the first element's
// tag and attributes.
@@ -66,12 +78,7 @@ func nodeName(node *html.Node) string {
// a property provided by the DOM).
func OuterHtml(s *Selection) (string, error) {
var buf bytes.Buffer
if s.Length() == 0 {
return "", nil
}
n := s.Get(0)
if err := html.Render(&buf, n); err != nil {
if err := Render(&buf, s); err != nil {
return "", err
}
return buf.String(), nil

162
vendor/github.com/andybalholm/brotli/cluster_command.go сгенерированный поставляемый
Просмотреть файл

@@ -1,7 +1,5 @@
package brotli
import "math"
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
@@ -164,163 +162,3 @@ func histogramBitCostDistanceCommand(histogram *histogramCommand, candidate *his
return populationCostCommand(&tmp) - candidate.bit_cost_
}
}
/* Find the best 'out' histogram for each of the 'in' histograms.
When called, clusters[0..num_clusters) contains the unique values from
symbols[0..in_size), but this property is not preserved in this function.
Note: we assume that out[]->bit_cost_ is already up-to-date. */
func histogramRemapCommand(in []histogramCommand, in_size uint, clusters []uint32, num_clusters uint, out []histogramCommand, symbols []uint32) {
var i uint
for i = 0; i < in_size; i++ {
var best_out uint32
if i == 0 {
best_out = symbols[0]
} else {
best_out = symbols[i-1]
}
var best_bits float64 = histogramBitCostDistanceCommand(&in[i], &out[best_out])
var j uint
for j = 0; j < num_clusters; j++ {
var cur_bits float64 = histogramBitCostDistanceCommand(&in[i], &out[clusters[j]])
if cur_bits < best_bits {
best_bits = cur_bits
best_out = clusters[j]
}
}
symbols[i] = best_out
}
/* Recompute each out based on raw and symbols. */
for i = 0; i < num_clusters; i++ {
histogramClearCommand(&out[clusters[i]])
}
for i = 0; i < in_size; i++ {
histogramAddHistogramCommand(&out[symbols[i]], &in[i])
}
}
/* Reorders elements of the out[0..length) array and changes values in
symbols[0..length) array in the following way:
* when called, symbols[] contains indexes into out[], and has N unique
values (possibly N < length)
* on return, symbols'[i] = f(symbols[i]) and
out'[symbols'[i]] = out[symbols[i]], for each 0 <= i < length,
where f is a bijection between the range of symbols[] and [0..N), and
the first occurrences of values in symbols'[i] come in consecutive
increasing order.
Returns N, the number of unique values in symbols[]. */
var histogramReindexCommand_kInvalidIndex uint32 = math.MaxUint32
func histogramReindexCommand(out []histogramCommand, symbols []uint32, length uint) uint {
var new_index []uint32 = make([]uint32, length)
var next_index uint32
var tmp []histogramCommand
var i uint
for i = 0; i < length; i++ {
new_index[i] = histogramReindexCommand_kInvalidIndex
}
next_index = 0
for i = 0; i < length; i++ {
if new_index[symbols[i]] == histogramReindexCommand_kInvalidIndex {
new_index[symbols[i]] = next_index
next_index++
}
}
/* TODO: by using idea of "cycle-sort" we can avoid allocation of
tmp and reduce the number of copying by the factor of 2. */
tmp = make([]histogramCommand, next_index)
next_index = 0
for i = 0; i < length; i++ {
if new_index[symbols[i]] == next_index {
tmp[next_index] = out[symbols[i]]
next_index++
}
symbols[i] = new_index[symbols[i]]
}
new_index = nil
for i = 0; uint32(i) < next_index; i++ {
out[i] = tmp[i]
}
tmp = nil
return uint(next_index)
}
func clusterHistogramsCommand(in []histogramCommand, in_size uint, max_histograms uint, out []histogramCommand, out_size *uint, histogram_symbols []uint32) {
var cluster_size []uint32 = make([]uint32, in_size)
var clusters []uint32 = make([]uint32, in_size)
var num_clusters uint = 0
var max_input_histograms uint = 64
var pairs_capacity uint = max_input_histograms * max_input_histograms / 2
var pairs []histogramPair = make([]histogramPair, (pairs_capacity + 1))
var i uint
/* For the first pass of clustering, we allow all pairs. */
for i = 0; i < in_size; i++ {
cluster_size[i] = 1
}
for i = 0; i < in_size; i++ {
out[i] = in[i]
out[i].bit_cost_ = populationCostCommand(&in[i])
histogram_symbols[i] = uint32(i)
}
for i = 0; i < in_size; i += max_input_histograms {
var num_to_combine uint = brotli_min_size_t(in_size-i, max_input_histograms)
var num_new_clusters uint
var j uint
for j = 0; j < num_to_combine; j++ {
clusters[num_clusters+j] = uint32(i + j)
}
num_new_clusters = histogramCombineCommand(out, cluster_size, histogram_symbols[i:], clusters[num_clusters:], pairs, num_to_combine, num_to_combine, max_histograms, pairs_capacity)
num_clusters += num_new_clusters
}
{
/* For the second pass, we limit the total number of histogram pairs.
After this limit is reached, we only keep searching for the best pair. */
var max_num_pairs uint = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
if pairs_capacity < (max_num_pairs + 1) {
var _new_size uint
if pairs_capacity == 0 {
_new_size = max_num_pairs + 1
} else {
_new_size = pairs_capacity
}
var new_array []histogramPair
for _new_size < (max_num_pairs + 1) {
_new_size *= 2
}
new_array = make([]histogramPair, _new_size)
if pairs_capacity != 0 {
copy(new_array, pairs[:pairs_capacity])
}
pairs = new_array
pairs_capacity = _new_size
}
/* Collapse similar histograms. */
num_clusters = histogramCombineCommand(out, cluster_size, histogram_symbols, clusters, pairs, num_clusters, in_size, max_histograms, max_num_pairs)
}
pairs = nil
cluster_size = nil
/* Find the optimal map from original histograms to the final ones. */
histogramRemapCommand(in, in_size, clusters, num_clusters, out, histogram_symbols)
clusters = nil
/* Convert the context map to a canonical form. */
*out_size = histogramReindexCommand(out, histogram_symbols, in_size)
}

2
vendor/github.com/andybalholm/brotli/compress_fragment.go сгенерированный поставляемый
Просмотреть файл

@@ -604,7 +604,7 @@ emit_commands:
assert(candidate < ip)
table[hash] = int(ip - base_ip)
if !(!isMatch5(in[ip:], in[candidate:])) {
if isMatch5(in[ip:], in[candidate:]) {
break
}
}

60
vendor/github.com/andybalholm/brotli/decode.go сгенерированный поставляемый
Просмотреть файл

@@ -50,21 +50,6 @@ const (
decoderErrorUnreachable = -31
)
/**
* The value of the last error code, negative integer.
*
* All other error code values are in the range from ::lastErrorCode
* to @c -1. There are also 4 other possible non-error codes @c 0 .. @c 3 in
* ::BrotliDecoderErrorCode enumeration.
*/
const lastErrorCode = decoderErrorUnreachable
/** Options to be used with ::BrotliDecoderSetParameter. */
const (
decoderParamDisableRingBufferReallocation = 0
decoderParamLargeWindow = 1
)
const huffmanTableBits = 8
const huffmanTableMask = 0xFF
@@ -81,28 +66,6 @@ var kCodeLengthPrefixLength = [16]byte{2, 2, 2, 3, 2, 2, 2, 4, 2, 2, 2, 3, 2, 2,
var kCodeLengthPrefixValue = [16]byte{0, 4, 3, 2, 0, 4, 3, 1, 0, 4, 3, 2, 0, 4, 3, 5}
func decoderSetParameter(state *Reader, p int, value uint32) bool {
if state.state != stateUninited {
return false
}
switch p {
case decoderParamDisableRingBufferReallocation:
if !(value == 0) {
state.canny_ringbuffer_allocation = 0
} else {
state.canny_ringbuffer_allocation = 1
}
return true
case decoderParamLargeWindow:
state.large_window = (!(value == 0))
return true
default:
return false
}
}
/* Saves error code and converts it to BrotliDecoderResult. */
func saveErrorCode(s *Reader, e int) int {
s.error_code = int(e)
@@ -1125,10 +1088,8 @@ func decodeContextMap(context_map_size uint32, num_htrees *uint32, context_map_a
Reads 3..54 bits. */
func decodeBlockTypeAndLength(safe int, s *Reader, tree_type int) bool {
var max_block_type uint32 = s.num_block_types[tree_type]
var type_tree []huffmanCode
type_tree = s.block_type_trees[tree_type*huffmanMaxSize258:]
var len_tree []huffmanCode
len_tree = s.block_len_trees[tree_type*huffmanMaxSize26:]
type_tree := s.block_type_trees[tree_type*huffmanMaxSize258:]
len_tree := s.block_len_trees[tree_type*huffmanMaxSize26:]
var br *bitReader = &s.br
var ringbuffer []uint32 = s.block_type_rb[tree_type*2:]
var block_type uint32
@@ -1280,8 +1241,7 @@ func unwrittenBytes(s *Reader, wrap bool) uint {
Returns BROTLI_DECODER_NEEDS_MORE_OUTPUT only if there is more output to push
and either ring-buffer is as big as window size, or |force| is true. */
func writeRingBuffer(s *Reader, available_out *uint, next_out *[]byte, total_out *uint, force bool) int {
var start []byte
start = s.ringbuffer[s.partial_pos_out&uint(s.ringbuffer_mask):]
start := s.ringbuffer[s.partial_pos_out&uint(s.ringbuffer_mask):]
var to_write uint = unwrittenBytes(s, true)
var num_written uint = *available_out
if num_written > to_write {
@@ -1412,8 +1372,7 @@ func copyUncompressedBlockToOutput(available_out *uint, next_out *[]byte, total_
case stateUncompressedWrite:
{
var result int
result = writeRingBuffer(s, available_out, next_out, total_out, false)
result := writeRingBuffer(s, available_out, next_out, total_out, false)
if result != decoderSuccess {
return result
}
@@ -1931,8 +1890,7 @@ CommandPostDecodeLiterals:
}
if transform_idx < int(trans.num_transforms) {
var word []byte
word = words.data[offset:]
word := words.data[offset:]
var len int = i
if transform_idx == int(trans.cutOffTransforms[0]) {
copy(s.ringbuffer[pos:], word[:uint(len)])
@@ -1954,10 +1912,8 @@ CommandPostDecodeLiterals:
}
} else {
var src_start int = (pos - s.distance_code) & s.ringbuffer_mask
var copy_dst []byte
copy_dst = s.ringbuffer[pos:]
var copy_src []byte
copy_src = s.ringbuffer[src_start:]
copy_dst := s.ringbuffer[pos:]
copy_src := s.ringbuffer[src_start:]
var dst_end int = pos + i
var src_end int = src_start + i
@@ -2494,8 +2450,6 @@ func decoderDecompressStream(s *Reader, available_in *uint, next_in *[]byte, ava
} else {
s.state = stateCommandBegin
}
break
} else if s.state == stateCommandPostWrite2 {
s.state = stateCommandPostWrapCopy /* BROTLI_STATE_COMMAND_INNER_WRITE */
} else {

3
vendor/github.com/andybalholm/brotli/encode.go сгенерированный поставляемый
Просмотреть файл

@@ -920,8 +920,7 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
REQUIRED: |header| should be 8-byte aligned and at least 16 bytes long.
REQUIRED: |block_size| <= (1 << 24). */
func writeMetadataHeader(s *Writer, block_size uint, header []byte) uint {
var storage_ix uint
storage_ix = uint(s.last_bytes_bits_)
storage_ix := uint(s.last_bytes_bits_)
header[0] = byte(s.last_bytes_)
header[1] = byte(s.last_bytes_ >> 8)
s.last_bytes_ = 0

16
vendor/github.com/andybalholm/brotli/fast_log.go сгенерированный поставляемый
Просмотреть файл

@@ -1,6 +1,9 @@
package brotli
import "math"
import (
"math"
"math/bits"
)
/* Copyright 2013 Google Inc. All Rights Reserved.
@@ -11,16 +14,7 @@ import "math"
/* Utilities for fast computation of logarithms. */
func log2FloorNonZero(n uint) uint32 {
/* TODO: generalize and move to platform.h */
var result uint32 = 0
for {
n >>= 1
if n == 0 {
break
}
result++
}
return result
return uint32(bits.Len(n)) - 1
}
/* A lookup table for small values of log2(int) to be used in entropy

2
vendor/github.com/andybalholm/brotli/hash.go сгенерированный поставляемый
Просмотреть файл

@@ -29,8 +29,6 @@ type hasherHandle interface {
Store(data []byte, mask uint, ix uint)
}
type score_t uint
const kCutoffTransformsCount uint32 = 10
/* 0, 12, 27, 23, 42, 63, 56, 48, 59, 64 */

3
vendor/github.com/andybalholm/brotli/hash_forgetful_chain.go сгенерированный поставляемый
Просмотреть файл

@@ -110,8 +110,7 @@ func (h *hashForgetfulChain) Prepare(one_shot bool, input_size uint, data []byte
func (h *hashForgetfulChain) Store(data []byte, mask uint, ix uint) {
var key uint = h.HashBytes(data[ix&mask:])
var bank uint = key & (h.numBanks - 1)
var idx uint
idx = uint(h.free_slot_idx[bank]) & ((1 << h.bankBits) - 1)
idx := uint(h.free_slot_idx[bank]) & ((1 << h.bankBits) - 1)
h.free_slot_idx[bank]++
var delta uint = ix - uint(h.addr[key])
h.tiny_hash[uint16(ix)] = byte(key)

1
vendor/github.com/andybalholm/brotli/hash_rolling.go сгенерированный поставляемый
Просмотреть файл

@@ -48,7 +48,6 @@ type hashRolling struct {
state uint32
table []uint32
next_ix uint
chunk_len uint32
factor uint32
factor_remove uint32
}

4
vendor/github.com/andybalholm/brotli/http.go сгенерированный поставляемый
Просмотреть файл

@@ -180,8 +180,8 @@ func init() {
var t octetType
isCtl := c <= 31 || c == 127
isChar := 0 <= c && c <= 127
isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0
if strings.IndexRune(" \t\r\n", rune(c)) >= 0 {
isSeparator := strings.ContainsRune(" \t\"(),/:;<=>?@[]\\{}", rune(c))
if strings.ContainsRune(" \t\r\n", rune(c)) {
t |= isSpace
}
if isChar && !isCtl && !isSeparator {

12
vendor/github.com/andybalholm/brotli/static_dict.go сгенерированный поставляемый
Просмотреть файл

@@ -77,8 +77,7 @@ func findAllStaticDictionaryMatches(dict *encoderDictionary, data []byte, min_le
var offset uint = uint(dict.buckets[hash(data)])
var end bool = offset == 0
for !end {
var w dictWord
w = dict.dict_words[offset]
w := dict.dict_words[offset]
offset++
var l uint = uint(w.len) & 0x1F
var n uint = uint(1) << dict.words.size_bits_by_length[l]
@@ -431,8 +430,7 @@ func findAllStaticDictionaryMatches(dict *encoderDictionary, data []byte, min_le
var offset uint = uint(dict.buckets[hash(data[1:])])
var end bool = offset == 0
for !end {
var w dictWord
w = dict.dict_words[offset]
w := dict.dict_words[offset]
offset++
var l uint = uint(w.len) & 0x1F
var n uint = uint(1) << dict.words.size_bits_by_length[l]
@@ -596,8 +594,7 @@ func findAllStaticDictionaryMatches(dict *encoderDictionary, data []byte, min_le
var offset uint = uint(dict.buckets[hash(data[2:])])
var end bool = offset == 0
for !end {
var w dictWord
w = dict.dict_words[offset]
w := dict.dict_words[offset]
offset++
var l uint = uint(w.len) & 0x1F
var n uint = uint(1) << dict.words.size_bits_by_length[l]
@@ -629,8 +626,7 @@ func findAllStaticDictionaryMatches(dict *encoderDictionary, data []byte, min_le
var offset uint = uint(dict.buckets[hash(data[5:])])
var end bool = offset == 0
for !end {
var w dictWord
w = dict.dict_words[offset]
w := dict.dict_words[offset]
offset++
var l uint = uint(w.len) & 0x1F
var n uint = uint(1) << dict.words.size_bits_by_length[l]

3
vendor/github.com/andybalholm/brotli/utf8_util.go сгенерированный поставляемый
Просмотреть файл

@@ -58,8 +58,7 @@ func isMostlyUTF8(data []byte, pos uint, mask uint, length uint, min_fraction fl
var i uint = 0
for i < length {
var symbol int
var current_data []byte
current_data = data[(pos+i)&mask:]
current_data := data[(pos+i)&mask:]
var bytes_read uint = parseAsUTF8(&symbol, current_data, length-i)
i += bytes_read
if symbol < 0x110000 {

1
vendor/github.com/andybalholm/brotli/writer.go сгенерированный поставляемый
Просмотреть файл

@@ -61,6 +61,7 @@ func (w *Writer) Reset(dst io.Writer) {
w.params.lgwin = uint(w.options.LGWin)
}
w.dst = dst
w.err = nil
}
func (w *Writer) writeChunk(p []byte, op int) (n int, err error) {

13
vendor/github.com/aws/aws-sdk-go/aws/client/client.go сгенерированный поставляемый
Просмотреть файл

@@ -10,12 +10,13 @@ import (
// A Config provides configuration to a service client instance.
type Config struct {
Config *aws.Config
Handlers request.Handlers
PartitionID string
Endpoint string
SigningRegion string
SigningName string
Config *aws.Config
Handlers request.Handlers
PartitionID string
Endpoint string
SigningRegion string
SigningName string
ResolvedRegion string
// States that the signing name did not come from a modeled source but
// was derived based on other data. Used by service client constructors

19
vendor/github.com/aws/aws-sdk-go/aws/client/metadata/client_info.go сгенерированный поставляемый
Просмотреть файл

@@ -2,13 +2,14 @@ package metadata
// ClientInfo wraps immutable data from the client.Client structure.
type ClientInfo struct {
ServiceName string
ServiceID string
APIVersion string
PartitionID string
Endpoint string
SigningName string
SigningRegion string
JSONVersion string
TargetPrefix string
ServiceName string
ServiceID string
APIVersion string
PartitionID string
Endpoint string
SigningName string
SigningRegion string
JSONVersion string
TargetPrefix string
ResolvedRegion string
}

23
vendor/github.com/aws/aws-sdk-go/aws/config.go сгенерированный поставляемый
Просмотреть файл

@@ -208,8 +208,19 @@ type Config struct {
// svc := s3.New(sess, &aws.Config{
// UseDualStack: aws.Bool(true),
// })
//
// Deprecated: This option will continue to function for S3 and S3 Control for backwards compatibility.
// UseDualStackEndpoint should be used to enable usage of a service's dual-stack endpoint for all service clients
// moving forward. For S3 and S3 Control, when UseDualStackEndpoint is set to a non-zero value it takes higher
// precedence then this option.
UseDualStack *bool
// Sets the resolver to resolve a dual-stack endpoint for the service.
UseDualStackEndpoint endpoints.DualStackEndpointState
// UseFIPSEndpoint specifies the resolver must resolve a FIPS endpoint.
UseFIPSEndpoint endpoints.FIPSEndpointState
// SleepDelay is an override for the func the SDK will call when sleeping
// during the lifecycle of a request. Specifically this will be used for
// request delays. This value should only be used for testing. To adjust
@@ -554,6 +565,10 @@ func mergeInConfig(dst *Config, other *Config) {
dst.UseDualStack = other.UseDualStack
}
if other.UseDualStackEndpoint != endpoints.DualStackEndpointStateUnset {
dst.UseDualStackEndpoint = other.UseDualStackEndpoint
}
if other.EC2MetadataDisableTimeoutOverride != nil {
dst.EC2MetadataDisableTimeoutOverride = other.EC2MetadataDisableTimeoutOverride
}
@@ -589,6 +604,14 @@ func mergeInConfig(dst *Config, other *Config) {
if other.LowerCaseHeaderMaps != nil {
dst.LowerCaseHeaderMaps = other.LowerCaseHeaderMaps
}
if other.UseDualStackEndpoint != endpoints.DualStackEndpointStateUnset {
dst.UseDualStackEndpoint = other.UseDualStackEndpoint
}
if other.UseFIPSEndpoint != endpoints.FIPSEndpointStateUnset {
dst.UseFIPSEndpoint = other.UseFIPSEndpoint
}
}
// Copy will return a shallow copy of the Config object. If any additional

59
vendor/github.com/aws/aws-sdk-go/aws/endpoints/decode.go сгенерированный поставляемый
Просмотреть файл

@@ -81,7 +81,6 @@ func decodeV3Endpoints(modelDef modelDefinition, opts DecodeModelOptions) (Resol
// Customization
for i := 0; i < len(ps); i++ {
p := &ps[i]
custAddS3DualStack(p)
custRegionalS3(p)
custRmIotDataService(p)
custFixAppAutoscalingChina(p)
@@ -91,15 +90,6 @@ func decodeV3Endpoints(modelDef modelDefinition, opts DecodeModelOptions) (Resol
return ps, nil
}
func custAddS3DualStack(p *partition) {
if !(p.ID == "aws" || p.ID == "aws-cn" || p.ID == "aws-us-gov") {
return
}
custAddDualstack(p, "s3")
custAddDualstack(p, "s3-control")
}
func custRegionalS3(p *partition) {
if p.ID != "aws" {
return
@@ -110,35 +100,28 @@ func custRegionalS3(p *partition) {
return
}
const awsGlobal = "aws-global"
const usEast1 = "us-east-1"
// If global endpoint already exists no customization needed.
if _, ok := service.Endpoints["aws-global"]; ok {
if _, ok := service.Endpoints[endpointKey{Region: awsGlobal}]; ok {
return
}
service.PartitionEndpoint = "aws-global"
service.Endpoints["us-east-1"] = endpoint{}
service.Endpoints["aws-global"] = endpoint{
service.PartitionEndpoint = awsGlobal
if _, ok := service.Endpoints[endpointKey{Region: usEast1}]; !ok {
service.Endpoints[endpointKey{Region: usEast1}] = endpoint{}
}
service.Endpoints[endpointKey{Region: awsGlobal}] = endpoint{
Hostname: "s3.amazonaws.com",
CredentialScope: credentialScope{
Region: "us-east-1",
Region: usEast1,
},
}
p.Services["s3"] = service
}
func custAddDualstack(p *partition, svcName string) {
s, ok := p.Services[svcName]
if !ok {
return
}
s.Defaults.HasDualStack = boxedTrue
s.Defaults.DualStackHostname = "{service}.dualstack.{region}.{dnsSuffix}"
p.Services[svcName] = s
}
func custRmIotDataService(p *partition) {
delete(p.Services, "data.iot")
}
@@ -155,12 +138,13 @@ func custFixAppAutoscalingChina(p *partition) {
}
const expectHostname = `autoscaling.{region}.amazonaws.com`
if e, a := s.Defaults.Hostname, expectHostname; e != a {
serviceDefault := s.Defaults[defaultKey{}]
if e, a := expectHostname, serviceDefault.Hostname; e != a {
fmt.Printf("custFixAppAutoscalingChina: ignoring customization, expected %s, got %s\n", e, a)
return
}
s.Defaults.Hostname = expectHostname + ".cn"
serviceDefault.Hostname = expectHostname + ".cn"
s.Defaults[defaultKey{}] = serviceDefault
p.Services[serviceName] = s
}
@@ -175,18 +159,25 @@ func custFixAppAutoscalingUsGov(p *partition) {
return
}
if a := s.Defaults.CredentialScope.Service; a != "" {
serviceDefault := s.Defaults[defaultKey{}]
if a := serviceDefault.CredentialScope.Service; a != "" {
fmt.Printf("custFixAppAutoscalingUsGov: ignoring customization, expected empty credential scope service, got %s\n", a)
return
}
if a := s.Defaults.Hostname; a != "" {
if a := serviceDefault.Hostname; a != "" {
fmt.Printf("custFixAppAutoscalingUsGov: ignoring customization, expected empty hostname, got %s\n", a)
return
}
s.Defaults.CredentialScope.Service = "application-autoscaling"
s.Defaults.Hostname = "autoscaling.{region}.amazonaws.com"
serviceDefault.CredentialScope.Service = "application-autoscaling"
serviceDefault.Hostname = "autoscaling.{region}.amazonaws.com"
if s.Defaults == nil {
s.Defaults = make(endpointDefaults)
}
s.Defaults[defaultKey{}] = serviceDefault
p.Services[serviceName] = s
}

27409
vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go сгенерированный поставляемый

Разница между файлами не показана из-за своего большого размера Загрузить разницу

103
vendor/github.com/aws/aws-sdk-go/aws/endpoints/endpoints.go сгенерированный поставляемый
Просмотреть файл

@@ -8,6 +8,41 @@ import (
"github.com/aws/aws-sdk-go/aws/awserr"
)
// A Logger is a minimalistic interface for the SDK to log messages to.
type Logger interface {
Log(...interface{})
}
// DualStackEndpointState is a constant to describe the dual-stack endpoint resolution
// behavior.
type DualStackEndpointState uint
const (
// DualStackEndpointStateUnset is the default value behavior for dual-stack endpoint
// resolution.
DualStackEndpointStateUnset DualStackEndpointState = iota
// DualStackEndpointStateEnabled enable dual-stack endpoint resolution for endpoints.
DualStackEndpointStateEnabled
// DualStackEndpointStateDisabled disables dual-stack endpoint resolution for endpoints.
DualStackEndpointStateDisabled
)
// FIPSEndpointState is a constant to describe the FIPS endpoint resolution behavior.
type FIPSEndpointState uint
const (
// FIPSEndpointStateUnset is the default value behavior for FIPS endpoint resolution.
FIPSEndpointStateUnset FIPSEndpointState = iota
// FIPSEndpointStateEnabled enables FIPS endpoint resolution for service endpoints.
FIPSEndpointStateEnabled
// FIPSEndpointStateDisabled disables FIPS endpoint resolution for endpoints.
FIPSEndpointStateDisabled
)
// Options provide the configuration needed to direct how the
// endpoints will be resolved.
type Options struct {
@@ -21,8 +56,19 @@ type Options struct {
// be returned. This endpoint may not be valid. If StrictMatching is
// enabled only services that are known to support dualstack will return
// dualstack endpoints.
//
// Deprecated: This option will continue to function for S3 and S3 Control for backwards compatibility.
// UseDualStackEndpoint should be used to enable usage of a service's dual-stack endpoint for all service clients
// moving forward. For S3 and S3 Control, when UseDualStackEndpoint is set to a non-zero value it takes higher
// precedence then this option.
UseDualStack bool
// Sets the resolver to resolve a dual-stack endpoint for the service.
UseDualStackEndpoint DualStackEndpointState
// UseFIPSEndpoint specifies the resolver must resolve a FIPS endpoint.
UseFIPSEndpoint FIPSEndpointState
// Enables strict matching of services and regions resolved endpoints.
// If the partition doesn't enumerate the exact service and region an
// error will be returned. This option will prevent returning endpoints
@@ -56,6 +102,30 @@ type Options struct {
// S3 Regional Endpoint flag helps with resolving the S3 endpoint
S3UsEast1RegionalEndpoint S3UsEast1RegionalEndpoint
// ResolvedRegion is the resolved region string. If provided (non-zero length) it takes priority
// over the region name passed to the ResolveEndpoint call.
ResolvedRegion string
// Logger is the logger that will be used to log messages.
Logger Logger
// Determines whether logging of deprecated endpoints usage is enabled.
LogDeprecated bool
}
func (o Options) getEndpointVariant(service string) (v endpointVariant) {
const s3 = "s3"
const s3Control = "s3-control"
if (o.UseDualStackEndpoint == DualStackEndpointStateEnabled) ||
((service == s3 || service == s3Control) && (o.UseDualStackEndpoint == DualStackEndpointStateUnset && o.UseDualStack)) {
v |= dualStackVariant
}
if o.UseFIPSEndpoint == FIPSEndpointStateEnabled {
v |= fipsVariant
}
return v
}
// EC2IMDSEndpointModeState is an enum configuration variable describing the client endpoint mode.
@@ -196,10 +266,25 @@ func DisableSSLOption(o *Options) {
// UseDualStackOption sets the UseDualStack option. Can be used as a functional
// option when resolving endpoints.
//
// Deprecated: UseDualStackEndpointOption should be used to enable usage of a service's dual-stack endpoint.
// When DualStackEndpointState is set to a non-zero value it takes higher precedence then this option.
func UseDualStackOption(o *Options) {
o.UseDualStack = true
}
// UseDualStackEndpointOption sets the UseDualStackEndpoint option to enabled. Can be used as a functional
// option when resolving endpoints.
func UseDualStackEndpointOption(o *Options) {
o.UseDualStackEndpoint = DualStackEndpointStateEnabled
}
// UseFIPSEndpointOption sets the UseFIPSEndpoint option to enabled. Can be used as a functional
// option when resolving endpoints.
func UseFIPSEndpointOption(o *Options) {
o.UseFIPSEndpoint = FIPSEndpointStateEnabled
}
// StrictMatchingOption sets the StrictMatching option. Can be used as a functional
// option when resolving endpoints.
func StrictMatchingOption(o *Options) {
@@ -407,7 +492,7 @@ func (r Region) ResolveEndpoint(service string, opts ...func(*Options)) (Resolve
func (r Region) Services() map[string]Service {
ss := map[string]Service{}
for id, s := range r.p.Services {
if _, ok := s.Endpoints[r.id]; ok {
if _, ok := s.Endpoints[endpointKey{Region: r.id}]; ok {
ss[id] = Service{
id: id,
p: r.p,
@@ -452,9 +537,12 @@ func (s Service) Regions() map[string]Region {
}
for id := range service.Endpoints {
if r, ok := s.p.Regions[id]; ok {
rs[id] = Region{
id: id,
if id.Variant != 0 {
continue
}
if r, ok := s.p.Regions[id.Region]; ok {
rs[id.Region] = Region{
id: id.Region,
desc: r.Description,
p: s.p,
}
@@ -472,8 +560,11 @@ func (s Service) Regions() map[string]Region {
func (s Service) Endpoints() map[string]Endpoint {
es := make(map[string]Endpoint, len(s.p.Services[s.id].Endpoints))
for id := range s.p.Services[s.id].Endpoints {
es[id] = Endpoint{
id: id,
if id.Variant != 0 {
continue
}
es[id.Region] = Endpoint{
id: id.Region,
serviceID: s.id,
p: s.p,
}

297
vendor/github.com/aws/aws-sdk-go/aws/endpoints/v3model.go сгенерированный поставляемый
Просмотреть файл

@@ -1,6 +1,7 @@
package endpoints
import (
"encoding/json"
"fmt"
"regexp"
"strconv"
@@ -12,6 +13,34 @@ const (
ec2MetadataEndpointIPv4 = "http://169.254.169.254/latest"
)
const dnsSuffixTemplateKey = "{dnsSuffix}"
// defaultKey is a compound map key of a variant and other values.
type defaultKey struct {
Variant endpointVariant
ServiceVariant serviceVariant
}
// endpointKey is a compound map key of a region and associated variant value.
type endpointKey struct {
Region string
Variant endpointVariant
}
// endpointVariant is a bit field to describe the endpoints attributes.
type endpointVariant uint64
// serviceVariant is a bit field to describe the service endpoint attributes.
type serviceVariant uint64
const (
// fipsVariant indicates that the endpoint is FIPS capable.
fipsVariant endpointVariant = 1 << (64 - 1 - iota)
// dualStackVariant indicates that the endpoint is DualStack capable.
dualStackVariant
)
var regionValidationRegex = regexp.MustCompile(`^[[:alnum:]]([[:alnum:]\-]*[[:alnum:]])?$`)
type partitions []partition
@@ -20,8 +49,12 @@ func (ps partitions) EndpointFor(service, region string, opts ...func(*Options))
var opt Options
opt.Set(opts...)
if len(opt.ResolvedRegion) > 0 {
region = opt.ResolvedRegion
}
for i := 0; i < len(ps); i++ {
if !ps[i].canResolveEndpoint(service, region, opt.StrictMatching) {
if !ps[i].canResolveEndpoint(service, region, opt) {
continue
}
@@ -49,14 +82,76 @@ func (ps partitions) Partitions() []Partition {
return parts
}
type endpointWithVariants struct {
endpoint
Variants []endpointWithTags `json:"variants"`
}
type endpointWithTags struct {
endpoint
Tags []string `json:"tags"`
}
type endpointDefaults map[defaultKey]endpoint
func (p *endpointDefaults) UnmarshalJSON(data []byte) error {
if *p == nil {
*p = make(endpointDefaults)
}
var e endpointWithVariants
if err := json.Unmarshal(data, &e); err != nil {
return err
}
(*p)[defaultKey{Variant: 0}] = e.endpoint
e.Hostname = ""
e.DNSSuffix = ""
for _, variant := range e.Variants {
endpointVariant, unknown := parseVariantTags(variant.Tags)
if unknown {
continue
}
var ve endpoint
ve.mergeIn(e.endpoint)
ve.mergeIn(variant.endpoint)
(*p)[defaultKey{Variant: endpointVariant}] = ve
}
return nil
}
func parseVariantTags(tags []string) (ev endpointVariant, unknown bool) {
if len(tags) == 0 {
unknown = true
return
}
for _, tag := range tags {
switch {
case strings.EqualFold("fips", tag):
ev |= fipsVariant
case strings.EqualFold("dualstack", tag):
ev |= dualStackVariant
default:
unknown = true
}
}
return ev, unknown
}
type partition struct {
ID string `json:"partition"`
Name string `json:"partitionName"`
DNSSuffix string `json:"dnsSuffix"`
RegionRegex regionRegex `json:"regionRegex"`
Defaults endpoint `json:"defaults"`
Regions regions `json:"regions"`
Services services `json:"services"`
ID string `json:"partition"`
Name string `json:"partitionName"`
DNSSuffix string `json:"dnsSuffix"`
RegionRegex regionRegex `json:"regionRegex"`
Defaults endpointDefaults `json:"defaults"`
Regions regions `json:"regions"`
Services services `json:"services"`
}
func (p partition) Partition() Partition {
@@ -67,15 +162,18 @@ func (p partition) Partition() Partition {
}
}
func (p partition) canResolveEndpoint(service, region string, strictMatch bool) bool {
func (p partition) canResolveEndpoint(service, region string, options Options) bool {
s, hasService := p.Services[service]
_, hasEndpoint := s.Endpoints[region]
_, hasEndpoint := s.Endpoints[endpointKey{
Region: region,
Variant: options.getEndpointVariant(service),
}]
if hasEndpoint && hasService {
return true
}
if strictMatch {
if options.StrictMatching {
return false
}
@@ -106,6 +204,10 @@ func (p partition) EndpointFor(service, region string, opts ...func(*Options)) (
var opt Options
opt.Set(opts...)
if len(opt.ResolvedRegion) > 0 {
region = opt.ResolvedRegion
}
s, hasService := p.Services[service]
if service == Ec2metadataServiceID && !hasService {
@@ -123,21 +225,44 @@ func (p partition) EndpointFor(service, region string, opts ...func(*Options)) (
region = s.PartitionEndpoint
}
if (service == "sts" && opt.STSRegionalEndpoint != RegionalSTSEndpoint) ||
(service == "s3" && opt.S3UsEast1RegionalEndpoint != RegionalS3UsEast1Endpoint) {
if _, ok := legacyGlobalRegions[service][region]; ok {
region = "aws-global"
}
if r, ok := isLegacyGlobalRegion(service, region, opt); ok {
region = r
}
e, hasEndpoint := s.endpointForRegion(region)
if len(region) == 0 || (!hasEndpoint && opt.StrictMatching) {
return resolved, NewUnknownEndpointError(p.ID, service, region, endpointList(s.Endpoints))
variant := opt.getEndpointVariant(service)
endpoints := s.Endpoints
serviceDefaults, hasServiceDefault := s.Defaults[defaultKey{Variant: variant}]
// If we searched for a variant which may have no explicit service defaults,
// then we need to inherit the standard service defaults except the hostname and dnsSuffix
if variant != 0 && !hasServiceDefault {
serviceDefaults = s.Defaults[defaultKey{}]
serviceDefaults.Hostname = ""
serviceDefaults.DNSSuffix = ""
}
defs := []endpoint{p.Defaults, s.Defaults}
partitionDefaults, hasPartitionDefault := p.Defaults[defaultKey{Variant: variant}]
return e.resolve(service, p.ID, region, p.DNSSuffix, defs, opt)
var dnsSuffix string
if len(serviceDefaults.DNSSuffix) > 0 {
dnsSuffix = serviceDefaults.DNSSuffix
} else if variant == 0 {
// For legacy reasons the partition dnsSuffix is not in the defaults, so if we looked for
// a non-variant endpoint then we need to set the dnsSuffix.
dnsSuffix = p.DNSSuffix
}
noDefaults := !hasServiceDefault && !hasPartitionDefault
e, hasEndpoint := s.endpointForRegion(region, endpoints, variant)
if len(region) == 0 || (!hasEndpoint && (opt.StrictMatching || noDefaults)) {
return resolved, NewUnknownEndpointError(p.ID, service, region, endpointList(endpoints, variant))
}
defs := []endpoint{partitionDefaults, serviceDefaults}
return e.resolve(service, p.ID, region, dnsSuffixTemplateKey, dnsSuffix, defs, opt)
}
func getEC2MetadataEndpoint(partitionID, service string, mode EC2IMDSEndpointModeState) ResolvedEndpoint {
@@ -165,6 +290,31 @@ func getEC2MetadataEndpoint(partitionID, service string, mode EC2IMDSEndpointMod
}
}
func isLegacyGlobalRegion(service string, region string, opt Options) (string, bool) {
if opt.getEndpointVariant(service) != 0 {
return "", false
}
const (
sts = "sts"
s3 = "s3"
awsGlobal = "aws-global"
)
switch {
case service == sts && opt.STSRegionalEndpoint == RegionalSTSEndpoint:
return region, false
case service == s3 && opt.S3UsEast1RegionalEndpoint == RegionalS3UsEast1Endpoint:
return region, false
default:
if _, ok := legacyGlobalRegions[service][region]; ok {
return awsGlobal, true
}
}
return region, false
}
func serviceList(ss services) []string {
list := make([]string, 0, len(ss))
for k := range ss {
@@ -172,10 +322,13 @@ func serviceList(ss services) []string {
}
return list
}
func endpointList(es endpoints) []string {
func endpointList(es serviceEndpoints, variant endpointVariant) []string {
list := make([]string, 0, len(es))
for k := range es {
list = append(list, k)
if k.Variant != variant {
continue
}
list = append(list, k.Region)
}
return list
}
@@ -207,19 +360,19 @@ type region struct {
type services map[string]service
type service struct {
PartitionEndpoint string `json:"partitionEndpoint"`
IsRegionalized boxedBool `json:"isRegionalized,omitempty"`
Defaults endpoint `json:"defaults"`
Endpoints endpoints `json:"endpoints"`
PartitionEndpoint string `json:"partitionEndpoint"`
IsRegionalized boxedBool `json:"isRegionalized,omitempty"`
Defaults endpointDefaults `json:"defaults"`
Endpoints serviceEndpoints `json:"endpoints"`
}
func (s *service) endpointForRegion(region string) (endpoint, bool) {
if e, ok := s.Endpoints[region]; ok {
func (s *service) endpointForRegion(region string, endpoints serviceEndpoints, variant endpointVariant) (endpoint, bool) {
if e, ok := endpoints[endpointKey{Region: region, Variant: variant}]; ok {
return e, true
}
if s.IsRegionalized == boxedFalse {
return s.Endpoints[s.PartitionEndpoint], region == s.PartitionEndpoint
return endpoints[endpointKey{Region: s.PartitionEndpoint, Variant: variant}], region == s.PartitionEndpoint
}
// Unable to find any matching endpoint, return
@@ -227,22 +380,73 @@ func (s *service) endpointForRegion(region string) (endpoint, bool) {
return endpoint{}, false
}
type endpoints map[string]endpoint
type serviceEndpoints map[endpointKey]endpoint
func (s *serviceEndpoints) UnmarshalJSON(data []byte) error {
if *s == nil {
*s = make(serviceEndpoints)
}
var regionToEndpoint map[string]endpointWithVariants
if err := json.Unmarshal(data, &regionToEndpoint); err != nil {
return err
}
for region, e := range regionToEndpoint {
(*s)[endpointKey{Region: region}] = e.endpoint
e.Hostname = ""
e.DNSSuffix = ""
for _, variant := range e.Variants {
endpointVariant, unknown := parseVariantTags(variant.Tags)
if unknown {
continue
}
var ve endpoint
ve.mergeIn(e.endpoint)
ve.mergeIn(variant.endpoint)
(*s)[endpointKey{Region: region, Variant: endpointVariant}] = ve
}
}
return nil
}
type endpoint struct {
Hostname string `json:"hostname"`
Protocols []string `json:"protocols"`
CredentialScope credentialScope `json:"credentialScope"`
// Custom fields not modeled
HasDualStack boxedBool `json:"-"`
DualStackHostname string `json:"-"`
DNSSuffix string `json:"dnsSuffix"`
// Signature Version not used
SignatureVersions []string `json:"signatureVersions"`
// SSLCommonName not used.
SSLCommonName string `json:"sslCommonName"`
Deprecated boxedBool `json:"deprecated"`
}
// isZero returns whether the endpoint structure is an empty (zero) value.
func (e endpoint) isZero() bool {
switch {
case len(e.Hostname) != 0:
return false
case len(e.Protocols) != 0:
return false
case e.CredentialScope != (credentialScope{}):
return false
case len(e.SignatureVersions) != 0:
return false
case len(e.SSLCommonName) != 0:
return false
}
return true
}
const (
@@ -271,7 +475,7 @@ func getByPriority(s []string, p []string, def string) string {
return s[0]
}
func (e endpoint) resolve(service, partitionID, region, dnsSuffix string, defs []endpoint, opts Options) (ResolvedEndpoint, error) {
func (e endpoint) resolve(service, partitionID, region, dnsSuffixTemplateVariable, dnsSuffix string, defs []endpoint, opts Options) (ResolvedEndpoint, error) {
var merged endpoint
for _, def := range defs {
merged.mergeIn(def)
@@ -292,23 +496,26 @@ func (e endpoint) resolve(service, partitionID, region, dnsSuffix string, defs [
}
hostname := e.Hostname
// Offset the hostname for dualstack if enabled
if opts.UseDualStack && e.HasDualStack == boxedTrue {
hostname = e.DualStackHostname
region = signingRegion
}
if !validateInputRegion(region) {
return ResolvedEndpoint{}, fmt.Errorf("invalid region identifier format provided")
}
if len(merged.DNSSuffix) > 0 {
dnsSuffix = merged.DNSSuffix
}
u := strings.Replace(hostname, "{service}", service, 1)
u = strings.Replace(u, "{region}", region, 1)
u = strings.Replace(u, "{dnsSuffix}", dnsSuffix, 1)
u = strings.Replace(u, dnsSuffixTemplateVariable, dnsSuffix, 1)
scheme := getEndpointScheme(e.Protocols, opts.DisableSSL)
u = fmt.Sprintf("%s://%s", scheme, u)
if e.Deprecated == boxedTrue && opts.LogDeprecated && opts.Logger != nil {
opts.Logger.Log(fmt.Sprintf("endpoint identifier %q, url %q marked as deprecated", region, u))
}
return ResolvedEndpoint{
URL: u,
PartitionID: partitionID,
@@ -346,11 +553,11 @@ func (e *endpoint) mergeIn(other endpoint) {
if len(other.SSLCommonName) > 0 {
e.SSLCommonName = other.SSLCommonName
}
if other.HasDualStack != boxedBoolUnset {
e.HasDualStack = other.HasDualStack
if len(other.DNSSuffix) > 0 {
e.DNSSuffix = other.DNSSuffix
}
if len(other.DualStackHostname) > 0 {
e.DualStackHostname = other.DualStackHostname
if other.Deprecated != boxedBoolUnset {
e.Deprecated = other.Deprecated
}
}

106
vendor/github.com/aws/aws-sdk-go/aws/endpoints/v3model_codegen.go сгенерированный поставляемый
Просмотреть файл

@@ -155,18 +155,71 @@ func serviceSet(ps partitions) map[string]struct{} {
return set
}
func endpointVariantSetter(variant endpointVariant) (string, error) {
if variant == 0 {
return "0", nil
}
if variant > (fipsVariant | dualStackVariant) {
return "", fmt.Errorf("unknown endpoint variant")
}
var symbols []string
if variant&fipsVariant != 0 {
symbols = append(symbols, "fipsVariant")
}
if variant&dualStackVariant != 0 {
symbols = append(symbols, "dualStackVariant")
}
v := strings.Join(symbols, "|")
return v, nil
}
func endpointKeySetter(e endpointKey) (string, error) {
var sb strings.Builder
sb.WriteString("endpointKey{\n")
sb.WriteString(fmt.Sprintf("Region: %q,\n", e.Region))
if e.Variant != 0 {
variantSetter, err := endpointVariantSetter(e.Variant)
if err != nil {
return "", err
}
sb.WriteString(fmt.Sprintf("Variant: %s,\n", variantSetter))
}
sb.WriteString("}")
return sb.String(), nil
}
func defaultKeySetter(e defaultKey) (string, error) {
var sb strings.Builder
sb.WriteString("defaultKey{\n")
if e.Variant != 0 {
variantSetter, err := endpointVariantSetter(e.Variant)
if err != nil {
return "", err
}
sb.WriteString(fmt.Sprintf("Variant: %s,\n", variantSetter))
}
sb.WriteString("}")
return sb.String(), nil
}
var funcMap = template.FuncMap{
"ToSymbol": toSymbol,
"QuoteString": quoteString,
"RegionConst": regionConstName,
"PartitionGetter": partitionGetter,
"PartitionVarName": partitionVarName,
"ListPartitionNames": listPartitionNames,
"BoxedBoolIfSet": boxedBoolIfSet,
"StringIfSet": stringIfSet,
"StringSliceIfSet": stringSliceIfSet,
"EndpointIsSet": endpointIsSet,
"ServicesSet": serviceSet,
"ToSymbol": toSymbol,
"QuoteString": quoteString,
"RegionConst": regionConstName,
"PartitionGetter": partitionGetter,
"PartitionVarName": partitionVarName,
"ListPartitionNames": listPartitionNames,
"BoxedBoolIfSet": boxedBoolIfSet,
"StringIfSet": stringIfSet,
"StringSliceIfSet": stringSliceIfSet,
"EndpointIsSet": endpointIsSet,
"ServicesSet": serviceSet,
"EndpointVariantSetter": endpointVariantSetter,
"EndpointKeySetter": endpointKeySetter,
"DefaultKeySetter": defaultKeySetter,
}
const v3Tmpl = `
@@ -272,9 +325,9 @@ partition{
{{ StringIfSet "Name: %q,\n" .Name -}}
{{ StringIfSet "DNSSuffix: %q,\n" .DNSSuffix -}}
RegionRegex: {{ template "gocode RegionRegex" .RegionRegex }},
{{ if EndpointIsSet .Defaults -}}
Defaults: {{ template "gocode Endpoint" .Defaults }},
{{- end }}
{{ if (gt (len .Defaults) 0) -}}
Defaults: {{ template "gocode Defaults" .Defaults -}},
{{ end -}}
Regions: {{ template "gocode Regions" .Regions }},
Services: {{ template "gocode Services" .Services }},
}
@@ -315,19 +368,27 @@ services{
service{
{{ StringIfSet "PartitionEndpoint: %q,\n" .PartitionEndpoint -}}
{{ BoxedBoolIfSet "IsRegionalized: %s,\n" .IsRegionalized -}}
{{ if EndpointIsSet .Defaults -}}
Defaults: {{ template "gocode Endpoint" .Defaults -}},
{{- end }}
{{ if (gt (len .Defaults) 0) -}}
Defaults: {{ template "gocode Defaults" .Defaults -}},
{{ end -}}
{{ if .Endpoints -}}
Endpoints: {{ template "gocode Endpoints" .Endpoints }},
{{- end }}
}
{{- end }}
{{ define "gocode Endpoints" -}}
endpoints{
{{ define "gocode Defaults" -}}
endpointDefaults{
{{ range $id, $endpoint := . -}}
"{{ $id }}": {{ template "gocode Endpoint" $endpoint }},
{{ DefaultKeySetter $id }}: {{ template "gocode Endpoint" $endpoint }},
{{ end }}
}
{{- end }}
{{ define "gocode Endpoints" -}}
serviceEndpoints{
{{ range $id, $endpoint := . -}}
{{ EndpointKeySetter $id }}: {{ template "gocode Endpoint" $endpoint }},
{{ end }}
}
{{- end }}
@@ -335,6 +396,7 @@ endpoints{
{{ define "gocode Endpoint" -}}
endpoint{
{{ StringIfSet "Hostname: %q,\n" .Hostname -}}
{{ StringIfSet "DNSSuffix: %q,\n" .DNSSuffix -}}
{{ StringIfSet "SSLCommonName: %q,\n" .SSLCommonName -}}
{{ StringSliceIfSet "Protocols: []string{%s},\n" .Protocols -}}
{{ StringSliceIfSet "SignatureVersions: []string{%s},\n" .SignatureVersions -}}
@@ -344,9 +406,7 @@ endpoint{
{{ StringIfSet "Service: %q,\n" .CredentialScope.Service -}}
},
{{- end }}
{{ BoxedBoolIfSet "HasDualStack: %s,\n" .HasDualStack -}}
{{ StringIfSet "DualStackHostname: %q,\n" .DualStackHostname -}}
{{ BoxedBoolIfSet "Deprecated: %s,\n" .Deprecated -}}
}
{{- end }}
`

3
vendor/github.com/aws/aws-sdk-go/aws/logger.go сгенерированный поставляемый
Просмотреть файл

@@ -77,6 +77,9 @@ const (
// wire unmarshaled message content of requests and responses made while
// using the SDK Will also enable LogDebug.
LogDebugWithEventStreamBody
// LogDebugWithDeprecated states the SDK should log details about deprecated functionality.
LogDebugWithDeprecated
)
// A Logger is a minimalistic interface for the SDK to log messages to. Should

78
vendor/github.com/aws/aws-sdk-go/aws/session/doc.go сгенерированный поставляемый
Просмотреть файл

@@ -285,5 +285,83 @@ The custom EC2 IMDS endpoint can also be specified via the Session options.
sess, err := session.NewSessionWithOptions(session.Options{
EC2MetadataEndpoint: "http://[::1]",
})
FIPS and DualStack Endpoints
The SDK can be configured to resolve an endpoint with certain capabilities such as FIPS and DualStack.
You can configure a FIPS endpoint using an environment variable, shared config ($HOME/.aws/config),
or programmatically.
To configure a FIPS endpoint set the environment variable set the AWS_USE_FIPS_ENDPOINT to true or false to enable
or disable FIPS endpoint resolution.
AWS_USE_FIPS_ENDPOINT=true
To configure a FIPS endpoint using shared config, set use_fips_endpoint to true or false to enable
or disable FIPS endpoint resolution.
[profile myprofile]
region=us-west-2
use_fips_endpoint=true
To configure a FIPS endpoint programmatically
// Option 1: Configure it on a session for all clients
sess, err := session.NewSessionWithOptions(session.Options{
UseFIPSEndpoint: endpoints.FIPSEndpointStateEnabled,
})
if err != nil {
// handle error
}
client := s3.New(sess)
// Option 2: Configure it per client
sess, err := session.NewSession()
if err != nil {
// handle error
}
client := s3.New(sess, &aws.Config{
UseFIPSEndpoint: endpoints.FIPSEndpointStateEnabled,
})
You can configure a DualStack endpoint using an environment variable, shared config ($HOME/.aws/config),
or programmatically.
To configure a DualStack endpoint set the environment variable set the AWS_USE_DUALSTACK_ENDPOINT to true or false to
enable or disable DualStack endpoint resolution.
AWS_USE_DUALSTACK_ENDPOINT=true
To configure a DualStack endpoint using shared config, set use_dualstack_endpoint to true or false to enable
or disable DualStack endpoint resolution.
[profile myprofile]
region=us-west-2
use_dualstack_endpoint=true
To configure a DualStack endpoint programmatically
// Option 1: Configure it on a session for all clients
sess, err := session.NewSessionWithOptions(session.Options{
UseDualStackEndpoint: endpoints.DualStackEndpointStateEnabled,
})
if err != nil {
// handle error
}
client := s3.New(sess)
// Option 2: Configure it per client
sess, err := session.NewSession()
if err != nil {
// handle error
}
client := s3.New(sess, &aws.Config{
UseDualStackEndpoint: endpoints.DualStackEndpointStateEnabled,
})
*/
package session

68
vendor/github.com/aws/aws-sdk-go/aws/session/env_config.go сгенерированный поставляемый
Просмотреть файл

@@ -170,6 +170,18 @@ type envConfig struct {
//
// AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE=IPv6
EC2IMDSEndpointMode endpoints.EC2IMDSEndpointModeState
// Specifies that SDK clients must resolve a dual-stack endpoint for
// services.
//
// AWS_USE_DUALSTACK_ENDPOINT=true
UseDualStackEndpoint endpoints.DualStackEndpointState
// Specifies that SDK clients must resolve a FIPS endpoint for
// services.
//
// AWS_USE_FIPS_ENDPOINT=true
UseFIPSEndpoint endpoints.FIPSEndpointState
}
var (
@@ -248,6 +260,12 @@ var (
useClientTLSKey = []string{
"AWS_SDK_GO_CLIENT_TLS_KEY",
}
awsUseDualStackEndpoint = []string{
"AWS_USE_DUALSTACK_ENDPOINT",
}
awsUseFIPSEndpoint = []string{
"AWS_USE_FIPS_ENDPOINT",
}
)
// loadEnvConfig retrieves the SDK's environment configuration.
@@ -376,6 +394,14 @@ func envConfigLoad(enableSharedConfig bool) (envConfig, error) {
return envConfig{}, err
}
if err := setUseDualStackEndpointFromEnvVal(&cfg.UseDualStackEndpoint, awsUseDualStackEndpoint); err != nil {
return cfg, err
}
if err := setUseFIPSEndpointFromEnvVal(&cfg.UseFIPSEndpoint, awsUseFIPSEndpoint); err != nil {
return cfg, err
}
return cfg, nil
}
@@ -401,3 +427,45 @@ func setEC2IMDSEndpointMode(mode *endpoints.EC2IMDSEndpointModeState, keys []str
}
return nil
}
func setUseDualStackEndpointFromEnvVal(dst *endpoints.DualStackEndpointState, keys []string) error {
for _, k := range keys {
value := os.Getenv(k)
if len(value) == 0 {
continue // skip if empty
}
switch {
case strings.EqualFold(value, "true"):
*dst = endpoints.DualStackEndpointStateEnabled
case strings.EqualFold(value, "false"):
*dst = endpoints.DualStackEndpointStateDisabled
default:
return fmt.Errorf(
"invalid value for environment variable, %s=%s, need true, false",
k, value)
}
}
return nil
}
func setUseFIPSEndpointFromEnvVal(dst *endpoints.FIPSEndpointState, keys []string) error {
for _, k := range keys {
value := os.Getenv(k)
if len(value) == 0 {
continue // skip if empty
}
switch {
case strings.EqualFold(value, "true"):
*dst = endpoints.FIPSEndpointStateEnabled
case strings.EqualFold(value, "false"):
*dst = endpoints.FIPSEndpointStateDisabled
default:
return fmt.Errorf(
"invalid value for environment variable, %s=%s, need true, false",
k, value)
}
}
return nil
}

55
vendor/github.com/aws/aws-sdk-go/aws/session/session.go сгенерированный поставляемый
Просмотреть файл

@@ -8,6 +8,7 @@ import (
"io/ioutil"
"net/http"
"os"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
@@ -792,6 +793,20 @@ func mergeConfigSrcs(cfg, userCfg *aws.Config,
cfg.S3UseARNRegion = &sharedCfg.S3UseARNRegion
}
for _, v := range []endpoints.DualStackEndpointState{userCfg.UseDualStackEndpoint, envCfg.UseDualStackEndpoint, sharedCfg.UseDualStackEndpoint} {
if v != endpoints.DualStackEndpointStateUnset {
cfg.UseDualStackEndpoint = v
break
}
}
for _, v := range []endpoints.FIPSEndpointState{userCfg.UseFIPSEndpoint, envCfg.UseFIPSEndpoint, sharedCfg.UseFIPSEndpoint} {
if v != endpoints.FIPSEndpointStateUnset {
cfg.UseFIPSEndpoint = v
break
}
}
return nil
}
@@ -845,8 +860,10 @@ func (s *Session) Copy(cfgs ...*aws.Config) *Session {
func (s *Session) ClientConfig(service string, cfgs ...*aws.Config) client.Config {
s = s.Copy(cfgs...)
resolvedRegion := normalizeRegion(s.Config)
region := aws.StringValue(s.Config.Region)
resolved, err := s.resolveEndpoint(service, region, s.Config)
resolved, err := s.resolveEndpoint(service, region, resolvedRegion, s.Config)
if err != nil {
s.Handlers.Validate.PushBack(func(r *request.Request) {
if len(r.ClientInfo.Endpoint) != 0 {
@@ -867,12 +884,13 @@ func (s *Session) ClientConfig(service string, cfgs ...*aws.Config) client.Confi
SigningRegion: resolved.SigningRegion,
SigningNameDerived: resolved.SigningNameDerived,
SigningName: resolved.SigningName,
ResolvedRegion: resolvedRegion,
}
}
const ec2MetadataServiceID = "ec2metadata"
func (s *Session) resolveEndpoint(service, region string, cfg *aws.Config) (endpoints.ResolvedEndpoint, error) {
func (s *Session) resolveEndpoint(service, region, resolvedRegion string, cfg *aws.Config) (endpoints.ResolvedEndpoint, error) {
if ep := aws.StringValue(cfg.Endpoint); len(ep) != 0 {
return endpoints.ResolvedEndpoint{
@@ -884,7 +902,12 @@ func (s *Session) resolveEndpoint(service, region string, cfg *aws.Config) (endp
resolved, err := cfg.EndpointResolver.EndpointFor(service, region,
func(opt *endpoints.Options) {
opt.DisableSSL = aws.BoolValue(cfg.DisableSSL)
opt.UseDualStack = aws.BoolValue(cfg.UseDualStack)
opt.UseDualStackEndpoint = cfg.UseDualStackEndpoint
opt.UseFIPSEndpoint = cfg.UseFIPSEndpoint
// Support for STSRegionalEndpoint where the STSRegionalEndpoint is
// provided in envConfig or sharedConfig with envConfig getting
// precedence.
@@ -898,6 +921,11 @@ func (s *Session) resolveEndpoint(service, region string, cfg *aws.Config) (endp
// Support the condition where the service is modeled but its
// endpoint metadata is not available.
opt.ResolveUnknownService = true
opt.ResolvedRegion = resolvedRegion
opt.Logger = cfg.Logger
opt.LogDeprecated = cfg.LogLevel.Matches(aws.LogDebugWithDeprecated)
},
)
if err != nil {
@@ -913,6 +941,8 @@ func (s *Session) resolveEndpoint(service, region string, cfg *aws.Config) (endp
func (s *Session) ClientConfigNoResolveEndpoint(cfgs ...*aws.Config) client.Config {
s = s.Copy(cfgs...)
resolvedRegion := normalizeRegion(s.Config)
var resolved endpoints.ResolvedEndpoint
if ep := aws.StringValue(s.Config.Endpoint); len(ep) > 0 {
resolved.URL = endpoints.AddScheme(ep, aws.BoolValue(s.Config.DisableSSL))
@@ -926,6 +956,7 @@ func (s *Session) ClientConfigNoResolveEndpoint(cfgs ...*aws.Config) client.Conf
SigningRegion: resolved.SigningRegion,
SigningNameDerived: resolved.SigningNameDerived,
SigningName: resolved.SigningName,
ResolvedRegion: resolvedRegion,
}
}
@@ -939,3 +970,23 @@ func (s *Session) logDeprecatedNewSessionError(msg string, err error, cfgs []*aw
r.Error = err
})
}
// normalizeRegion resolves / normalizes the configured region (converts pseudo fips regions), and modifies the provided
// config to have the equivalent options for resolution and returns the resolved region name.
func normalizeRegion(cfg *aws.Config) (resolved string) {
const fipsInfix = "-fips-"
const fipsPrefix = "-fips"
const fipsSuffix = "fips-"
region := aws.StringValue(cfg.Region)
if strings.Contains(region, fipsInfix) ||
strings.Contains(region, fipsPrefix) ||
strings.Contains(region, fipsSuffix) {
resolved = strings.Replace(strings.Replace(strings.Replace(
region, fipsInfix, "-", -1), fipsPrefix, "", -1), fipsSuffix, "", -1)
cfg.UseFIPSEndpoint = endpoints.FIPSEndpointStateEnabled
}
return resolved
}

54
vendor/github.com/aws/aws-sdk-go/aws/session/shared_config.go сгенерированный поставляемый
Просмотреть файл

@@ -72,6 +72,12 @@ const (
// EC2 IMDS Endpoint
ec2MetadataServiceEndpointKey = "ec2_metadata_service_endpoint"
// Use DualStack Endpoint Resolution
useDualStackEndpoint = "use_dualstack_endpoint"
// Use FIPS Endpoint Resolution
useFIPSEndpointKey = "use_fips_endpoint"
)
// sharedConfig represents the configuration fields of the SDK config files.
@@ -161,6 +167,18 @@ type sharedConfig struct {
//
// ec2_metadata_service_endpoint=http://fd00:ec2::254
EC2IMDSEndpoint string
// Specifies that SDK clients must resolve a dual-stack endpoint for
// services.
//
// use_dualstack_endpoint=true
UseDualStackEndpoint endpoints.DualStackEndpointState
// Specifies that SDK clients must resolve a FIPS endpoint for
// services.
//
// use_fips_endpoint=true
UseFIPSEndpoint endpoints.FIPSEndpointState
}
type sharedConfigFile struct {
@@ -356,6 +374,10 @@ func (cfg *sharedConfig) setFromIniFile(profile string, file sharedConfigFile, e
ec2MetadataServiceEndpointModeKey, file.Filename, err)
}
updateString(&cfg.EC2IMDSEndpoint, section, ec2MetadataServiceEndpointKey)
updateUseDualStackEndpoint(&cfg.UseDualStackEndpoint, section, useDualStackEndpoint)
updateUseFIPSEndpoint(&cfg.UseFIPSEndpoint, section, useFIPSEndpointKey)
}
updateString(&cfg.CredentialProcess, section, credentialProcessKey)
@@ -673,3 +695,35 @@ func (e CredentialRequiresARNError) OrigErr() error {
func (e CredentialRequiresARNError) Error() string {
return awserr.SprintError(e.Code(), e.Message(), "", nil)
}
// updateEndpointDiscoveryType will only update the dst with the value in the section, if
// a valid key and corresponding EndpointDiscoveryType is found.
func updateUseDualStackEndpoint(dst *endpoints.DualStackEndpointState, section ini.Section, key string) {
if !section.Has(key) {
return
}
if section.Bool(key) {
*dst = endpoints.DualStackEndpointStateEnabled
} else {
*dst = endpoints.DualStackEndpointStateDisabled
}
return
}
// updateEndpointDiscoveryType will only update the dst with the value in the section, if
// a valid key and corresponding EndpointDiscoveryType is found.
func updateUseFIPSEndpoint(dst *endpoints.FIPSEndpointState, section ini.Section, key string) {
if !section.Has(key) {
return
}
if section.Bool(key) {
*dst = endpoints.FIPSEndpointStateEnabled
} else {
*dst = endpoints.FIPSEndpointStateDisabled
}
return
}

18
vendor/github.com/aws/aws-sdk-go/aws/signer/v4/v4.go сгенерированный поставляемый
Просмотреть файл

@@ -634,21 +634,25 @@ func (ctx *signingCtx) buildCanonicalHeaders(r rule, header http.Header) {
ctx.Query.Set("X-Amz-SignedHeaders", ctx.signedHeaders)
}
headerValues := make([]string, len(headers))
headerItems := make([]string, len(headers))
for i, k := range headers {
if k == "host" {
if ctx.Request.Host != "" {
headerValues[i] = "host:" + ctx.Request.Host
headerItems[i] = "host:" + ctx.Request.Host
} else {
headerValues[i] = "host:" + ctx.Request.URL.Host
headerItems[i] = "host:" + ctx.Request.URL.Host
}
} else {
headerValues[i] = k + ":" +
strings.Join(ctx.SignedHeaderVals[k], ",")
headerValues := make([]string, len(ctx.SignedHeaderVals[k]))
for i, v := range ctx.SignedHeaderVals[k] {
headerValues[i] = strings.TrimSpace(v)
}
headerItems[i] = k + ":" +
strings.Join(headerValues, ",")
}
}
stripExcessSpaces(headerValues)
ctx.canonicalHeaders = strings.Join(headerValues, "\n")
stripExcessSpaces(headerItems)
ctx.canonicalHeaders = strings.Join(headerItems, "\n")
}
func (ctx *signingCtx) buildCanonicalString() {

2
vendor/github.com/aws/aws-sdk-go/aws/version.go сгенерированный поставляемый
Просмотреть файл

@@ -5,4 +5,4 @@ package aws
const SDKName = "aws-sdk-go"
// SDKVersion is the version of this SDK
const SDKVersion = "1.40.57"
const SDKVersion = "1.42.11"

20
vendor/github.com/aws/aws-sdk-go/internal/ini/literal_tokens.go сгенерированный поставляемый
Просмотреть файл

@@ -4,6 +4,7 @@ import (
"fmt"
"strconv"
"strings"
"unicode"
)
var (
@@ -18,7 +19,7 @@ var literalValues = [][]rune{
func isBoolValue(b []rune) bool {
for _, lv := range literalValues {
if isLitValue(lv, b) {
if isCaselessLitValue(lv, b) {
return true
}
}
@@ -39,6 +40,21 @@ func isLitValue(want, have []rune) bool {
return true
}
// isCaselessLitValue is a caseless value comparison, assumes want is already lower-cased for efficiency.
func isCaselessLitValue(want, have []rune) bool {
if len(have) < len(want) {
return false
}
for i := 0; i < len(want); i++ {
if want[i] != unicode.ToLower(have[i]) {
return false
}
}
return true
}
// isNumberValue will return whether not the leading characters in
// a byte slice is a number. A number is delimited by whitespace or
// the newline token.
@@ -177,7 +193,7 @@ func newValue(t ValueType, base int, raw []rune) (Value, error) {
case QuotedStringType:
v.str = string(raw[1 : len(raw)-1])
case BoolType:
v.boolean = runeCompare(v.raw, runesTrue)
v.boolean = isCaselessLitValue(runesTrue, v.raw)
}
// issue 2253

2
vendor/github.com/aws/aws-sdk-go/internal/ini/value_util.go сгенерированный поставляемый
Просмотреть файл

@@ -57,7 +57,7 @@ func getBoolValue(b []rune) (int, error) {
continue
}
if isLitValue(lv, b) {
if isCaselessLitValue(lv, b) {
n = len(lv)
}
}

10
vendor/github.com/aws/aws-sdk-go/private/protocol/json/jsonutil/build.go сгенерированный поставляемый
Просмотреть файл

@@ -82,13 +82,17 @@ func buildStruct(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag)
field, _ := value.Type().FieldByName(payload)
tag = field.Tag
value = elemOf(value.FieldByName(payload))
if !value.IsValid() {
if !value.IsValid() && tag.Get("type") != "structure" {
return nil
}
}
buf.WriteByte('{')
defer buf.WriteString("}")
if !value.IsValid() {
return nil
}
t := value.Type()
first := true
@@ -144,8 +148,6 @@ func buildStruct(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag)
}
buf.WriteString("}")
return nil
}

5
vendor/github.com/aws/aws-sdk-go/private/protocol/jsonrpc/jsonrpc.go сгенерированный поставляемый
Просмотреть файл

@@ -49,9 +49,8 @@ func Build(req *request.Request) {
buf = emptyJSON
}
if req.ClientInfo.TargetPrefix != "" || string(buf) != "{}" {
req.SetBufferBody(buf)
}
// Always serialize the body, don't suppress it.
req.SetBufferBody(buf)
if req.ClientInfo.TargetPrefix != "" {
target := req.ClientInfo.TargetPrefix + "." + req.Operation.Name

11
vendor/github.com/aws/aws-sdk-go/private/protocol/rest/payload.go сгенерированный поставляемый
Просмотреть файл

@@ -28,18 +28,27 @@ func PayloadMember(i interface{}) interface{} {
return nil
}
// PayloadType returns the type of a payload field member of i if there is one, or "".
const nopayloadPayloadType = "nopayload"
// PayloadType returns the type of a payload field member of i if there is one,
// or "".
func PayloadType(i interface{}) string {
v := reflect.Indirect(reflect.ValueOf(i))
if !v.IsValid() {
return ""
}
if field, ok := v.Type().FieldByName("_"); ok {
if noPayload := field.Tag.Get(nopayloadPayloadType); noPayload != "" {
return nopayloadPayloadType
}
if payloadName := field.Tag.Get("payload"); payloadName != "" {
if member, ok := v.Type().FieldByName(payloadName); ok {
return member.Tag.Get("type")
}
}
}
return ""
}

23
vendor/github.com/aws/aws-sdk-go/service/marketplacemetering/service.go сгенерированный поставляемый
Просмотреть файл

@@ -52,24 +52,25 @@ func New(p client.ConfigProvider, cfgs ...*aws.Config) *MarketplaceMetering {
if c.SigningNameDerived || len(c.SigningName) == 0 {
c.SigningName = "aws-marketplace"
}
return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName)
return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName, c.ResolvedRegion)
}
// newClient creates, initializes and returns a new service client instance.
func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint, signingRegion, signingName string) *MarketplaceMetering {
func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint, signingRegion, signingName, resolvedRegion string) *MarketplaceMetering {
svc := &MarketplaceMetering{
Client: client.New(
cfg,
metadata.ClientInfo{
ServiceName: ServiceName,
ServiceID: ServiceID,
SigningName: signingName,
SigningRegion: signingRegion,
PartitionID: partitionID,
Endpoint: endpoint,
APIVersion: "2016-01-14",
JSONVersion: "1.1",
TargetPrefix: "AWSMPMeteringService",
ServiceName: ServiceName,
ServiceID: ServiceID,
SigningName: signingName,
SigningRegion: signingRegion,
PartitionID: partitionID,
Endpoint: endpoint,
APIVersion: "2016-01-14",
ResolvedRegion: resolvedRegion,
JSONVersion: "1.1",
TargetPrefix: "AWSMPMeteringService",
},
handlers,
),

8
vendor/github.com/aws/aws-sdk-go/service/sso/api.go сгенерированный поставляемый
Просмотреть файл

@@ -550,7 +550,7 @@ func (s *AccountInfo) SetEmailAddress(v string) *AccountInfo {
}
type GetRoleCredentialsInput struct {
_ struct{} `type:"structure"`
_ struct{} `type:"structure" nopayload:"true"`
// The token issued by the CreateToken API call. For more information, see CreateToken
// (https://docs.aws.amazon.com/singlesignon/latest/OIDCAPIReference/API_CreateToken.html)
@@ -726,7 +726,7 @@ func (s *InvalidRequestException) RequestID() string {
}
type ListAccountRolesInput struct {
_ struct{} `type:"structure"`
_ struct{} `type:"structure" nopayload:"true"`
// The token issued by the CreateToken API call. For more information, see CreateToken
// (https://docs.aws.amazon.com/singlesignon/latest/OIDCAPIReference/API_CreateToken.html)
@@ -855,7 +855,7 @@ func (s *ListAccountRolesOutput) SetRoleList(v []*RoleInfo) *ListAccountRolesOut
}
type ListAccountsInput struct {
_ struct{} `type:"structure"`
_ struct{} `type:"structure" nopayload:"true"`
// The token issued by the CreateToken API call. For more information, see CreateToken
// (https://docs.aws.amazon.com/singlesignon/latest/OIDCAPIReference/API_CreateToken.html)
@@ -970,7 +970,7 @@ func (s *ListAccountsOutput) SetNextToken(v string) *ListAccountsOutput {
}
type LogoutInput struct {
_ struct{} `type:"structure"`
_ struct{} `type:"structure" nopayload:"true"`
// The token issued by the CreateToken API call. For more information, see CreateToken
// (https://docs.aws.amazon.com/singlesignon/latest/OIDCAPIReference/API_CreateToken.html)

19
vendor/github.com/aws/aws-sdk-go/service/sso/service.go сгенерированный поставляемый
Просмотреть файл

@@ -52,22 +52,23 @@ func New(p client.ConfigProvider, cfgs ...*aws.Config) *SSO {
if c.SigningNameDerived || len(c.SigningName) == 0 {
c.SigningName = "awsssoportal"
}
return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName)
return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName, c.ResolvedRegion)
}
// newClient creates, initializes and returns a new service client instance.
func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint, signingRegion, signingName string) *SSO {
func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint, signingRegion, signingName, resolvedRegion string) *SSO {
svc := &SSO{
Client: client.New(
cfg,
metadata.ClientInfo{
ServiceName: ServiceName,
ServiceID: ServiceID,
SigningName: signingName,
SigningRegion: signingRegion,
PartitionID: partitionID,
Endpoint: endpoint,
APIVersion: "2019-06-10",
ServiceName: ServiceName,
ServiceID: ServiceID,
SigningName: signingName,
SigningRegion: signingRegion,
PartitionID: partitionID,
Endpoint: endpoint,
APIVersion: "2019-06-10",
ResolvedRegion: resolvedRegion,
},
handlers,
),

168
vendor/github.com/aws/aws-sdk-go/service/sts/api.go сгенерированный поставляемый
Просмотреть файл

@@ -63,14 +63,15 @@ func (c *STS) AssumeRoleRequest(input *AssumeRoleInput) (req *request.Request, o
// or for cross-account access. For a comparison of AssumeRole with other API
// operations that produce temporary credentials, see Requesting Temporary Security
// Credentials (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html)
// and Comparing the STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
// and Comparing the Amazon Web Services STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
// in the IAM User Guide.
//
// Permissions
//
// The temporary security credentials created by AssumeRole can be used to make
// API calls to any Amazon Web Services service with the following exception:
// You cannot call the STS GetFederationToken or GetSessionToken API operations.
// You cannot call the Amazon Web Services STS GetFederationToken or GetSessionToken
// API operations.
//
// (Optional) You can pass inline or managed session policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session)
// to this operation. You can pass a single JSON policy document to use as an
@@ -86,26 +87,33 @@ func (c *STS) AssumeRoleRequest(input *AssumeRoleInput) (req *request.Request, o
// For more information, see Session Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session)
// in the IAM User Guide.
//
// To assume a role from a different account, your account must be trusted by
// the role. The trust relationship is defined in the role's trust policy when
// the role is created. That trust policy states which accounts are allowed
// to delegate that access to users in the account.
// When you create a role, you create two policies: A role trust policy that
// specifies who can assume the role and a permissions policy that specifies
// what can be done with the role. You specify the trusted principal who is
// allowed to assume the role in the role trust policy.
//
// To assume a role from a different account, your Amazon Web Services account
// must be trusted by the role. The trust relationship is defined in the role's
// trust policy when the role is created. That trust policy states which accounts
// are allowed to delegate that access to users in the account.
//
// A user who wants to access a role in a different account must also have permissions
// that are delegated from the user account administrator. The administrator
// must attach a policy that allows the user to call AssumeRole for the ARN
// of the role in the other account. If the user is in the same account as the
// role, then you can do either of the following:
// of the role in the other account.
//
// * Attach a policy to the user (identical to the previous user in a different
// account).
// To allow a user to assume a role in the same account, you can do either of
// the following:
//
// * Attach a policy to the user that allows the user to call AssumeRole
// (as long as the role's trust policy trusts the account).
//
// * Add the user as a principal directly in the role's trust policy.
//
// In this case, the trust policy acts as an IAM resource-based policy. Users
// in the same account as the role do not need explicit permission to assume
// the role. For more information about trust policies and resource-based policies,
// see IAM Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html)
// You can do either because the roles trust policy acts as an IAM resource-based
// policy. When a resource-based policy grants access to a principal in the
// same account, no additional identity-based policy is required. For more information
// about trust policies and resource-based policies, see IAM Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html)
// in the IAM User Guide.
//
// Tags
@@ -170,7 +178,7 @@ func (c *STS) AssumeRoleRequest(input *AssumeRoleInput) (req *request.Request, o
//
// You could receive this error even though you meet other defined session policy
// and session tag limits. For more information, see IAM and STS Entity Character
// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html)
// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html#reference_iam-limits-entity-length)
// in the IAM User Guide.
//
// * ErrCodeRegionDisabledException "RegionDisabledException"
@@ -258,7 +266,7 @@ func (c *STS) AssumeRoleWithSAMLRequest(input *AssumeRoleWithSAMLInput) (req *re
// Services access without user-specific credentials or configuration. For a
// comparison of AssumeRoleWithSAML with the other API operations that produce
// temporary credentials, see Requesting Temporary Security Credentials (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html)
// and Comparing the STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
// and Comparing the Amazon Web Services STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
// in the IAM User Guide.
//
// The temporary security credentials returned by this operation consist of
@@ -403,7 +411,7 @@ func (c *STS) AssumeRoleWithSAMLRequest(input *AssumeRoleWithSAMLInput) (req *re
//
// You could receive this error even though you meet other defined session policy
// and session tag limits. For more information, see IAM and STS Entity Character
// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html)
// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html#reference_iam-limits-entity-length)
// in the IAM User Guide.
//
// * ErrCodeIDPRejectedClaimException "IDPRejectedClaim"
@@ -523,7 +531,7 @@ func (c *STS) AssumeRoleWithWebIdentityRequest(input *AssumeRoleWithWebIdentityI
// by using a token from the web identity provider. For a comparison of AssumeRoleWithWebIdentity
// with the other API operations that produce temporary credentials, see Requesting
// Temporary Security Credentials (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html)
// and Comparing the STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
// and Comparing the Amazon Web Services STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
// in the IAM User Guide.
//
// The temporary security credentials returned by this API consist of an access
@@ -661,7 +669,7 @@ func (c *STS) AssumeRoleWithWebIdentityRequest(input *AssumeRoleWithWebIdentityI
//
// You could receive this error even though you meet other defined session policy
// and session tag limits. For more information, see IAM and STS Entity Character
// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html)
// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html#reference_iam-limits-entity-length)
// in the IAM User Guide.
//
// * ErrCodeIDPRejectedClaimException "IDPRejectedClaim"
@@ -776,10 +784,11 @@ func (c *STS) DecodeAuthorizationMessageRequest(input *DecodeAuthorizationMessag
// code.
//
// The message is encoded because the details of the authorization status can
// constitute privileged information that the user who requested the operation
// contain privileged information that the user who requested the operation
// should not see. To decode an authorization status message, a user must be
// granted permissions via an IAM policy to request the DecodeAuthorizationMessage
// (sts:DecodeAuthorizationMessage) action.
// granted permissions through an IAM policy (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html)
// to request the DecodeAuthorizationMessage (sts:DecodeAuthorizationMessage)
// action.
//
// The decoded message includes the following type of information:
//
@@ -1065,7 +1074,7 @@ func (c *STS) GetFederationTokenRequest(input *GetFederationTokenInput) (req *re
// For a comparison of GetFederationToken with the other API operations that
// produce temporary credentials, see Requesting Temporary Security Credentials
// (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html)
// and Comparing the STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
// and Comparing the Amazon Web Services STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
// in the IAM User Guide.
//
// You can create a mobile-based or browser-based app that can authenticate
@@ -1088,9 +1097,9 @@ func (c *STS) GetFederationTokenRequest(input *GetFederationTokenInput) (req *re
//
// The temporary credentials are valid for the specified duration, from 900
// seconds (15 minutes) up to a maximum of 129,600 seconds (36 hours). The default
// session duration is 43,200 seconds (12 hours). Temporary credentials that
// are obtained by using Amazon Web Services account root user credentials have
// a maximum duration of 3,600 seconds (1 hour).
// session duration is 43,200 seconds (12 hours). Temporary credentials obtained
// by using the Amazon Web Services account root user credentials have a maximum
// duration of 3,600 seconds (1 hour).
//
// Permissions
//
@@ -1141,63 +1150,6 @@ func (c *STS) GetFederationTokenRequest(input *GetFederationTokenInput) (req *re
// (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#api_assumerolewithwebidentity)
// in the IAM User Guide.
//
// You can also call GetFederationToken using the security credentials of an
// Amazon Web Services account root user, but we do not recommend it. Instead,
// we recommend that you create an IAM user for the purpose of the proxy application.
// Then attach a policy to the IAM user that limits federated users to only
// the actions and resources that they need to access. For more information,
// see IAM Best Practices (https://docs.aws.amazon.com/IAM/latest/UserGuide/best-practices.html)
// in the IAM User Guide.
//
// Session duration
//
// The temporary credentials are valid for the specified duration, from 900
// seconds (15 minutes) up to a maximum of 129,600 seconds (36 hours). The default
// session duration is 43,200 seconds (12 hours). Temporary credentials that
// are obtained by using Amazon Web Services account root user credentials have
// a maximum duration of 3,600 seconds (1 hour).
//
// Permissions
//
// You can use the temporary credentials created by GetFederationToken in any
// Amazon Web Services service except the following:
//
// * You cannot call any IAM operations using the CLI or the Amazon Web Services
// API.
//
// * You cannot call any STS operations except GetCallerIdentity.
//
// You must pass an inline or managed session policy (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session)
// to this operation. You can pass a single JSON policy document to use as an
// inline session policy. You can also specify up to 10 managed policies to
// use as managed session policies. The plain text that you use for both inline
// and managed session policies can't exceed 2,048 characters.
//
// Though the session policy parameters are optional, if you do not pass a policy,
// then the resulting federated user session has no permissions. When you pass
// session policies, the session permissions are the intersection of the IAM
// user policies and the session policies that you pass. This gives you a way
// to further restrict the permissions for a federated user. You cannot use
// session policies to grant more permissions than those that are defined in
// the permissions policy of the IAM user. For more information, see Session
// Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session)
// in the IAM User Guide. For information about using GetFederationToken to
// create temporary security credentials, see GetFederationToken—Federation
// Through a Custom Identity Broker (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#api_getfederationtoken).
//
// You can use the credentials to access a resource that has a resource-based
// policy. If that policy specifically references the federated user session
// in the Principal element of the policy, the session has the permissions allowed
// by the policy. These permissions are granted in addition to the permissions
// granted by the session policies.
//
// Tags
//
// (Optional) You can pass tag key-value pairs to your session. These are called
// session tags. For more information about session tags, see Passing Session
// Tags in STS (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html)
// in the IAM User Guide.
//
// An administrator must grant you the permissions necessary to pass session
// tags. The administrator can also create granular permissions to allow you
// to pass only specific session tags. For more information, see Tutorial: Using
@@ -1234,7 +1186,7 @@ func (c *STS) GetFederationTokenRequest(input *GetFederationTokenInput) (req *re
//
// You could receive this error even though you meet other defined session policy
// and session tag limits. For more information, see IAM and STS Entity Character
// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html)
// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html#reference_iam-limits-entity-length)
// in the IAM User Guide.
//
// * ErrCodeRegionDisabledException "RegionDisabledException"
@@ -1323,7 +1275,7 @@ func (c *STS) GetSessionTokenRequest(input *GetSessionTokenInput) (req *request.
// then the API returns an access denied error. For a comparison of GetSessionToken
// with the other API operations that produce temporary credentials, see Requesting
// Temporary Security Credentials (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html)
// and Comparing the STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
// and Comparing the Amazon Web Services STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
// in the IAM User Guide.
//
// Session Duration
@@ -1404,15 +1356,23 @@ func (c *STS) GetSessionTokenWithContext(ctx aws.Context, input *GetSessionToken
type AssumeRoleInput struct {
_ struct{} `type:"structure"`
// The duration, in seconds, of the role session. The value specified can can
// range from 900 seconds (15 minutes) up to the maximum session duration that
// is set for the role. The maximum session duration setting can have a value
// from 1 hour to 12 hours. If you specify a value higher than this setting
// or the administrator setting (whichever is lower), the operation fails. For
// example, if you specify a session duration of 12 hours, but your administrator
// set the maximum session duration to 6 hours, your operation fails. To learn
// how to view the maximum value for your role, see View the Maximum Session
// Duration Setting for a Role (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html#id_roles_use_view-role-max-session)
// The duration, in seconds, of the role session. The value specified can range
// from 900 seconds (15 minutes) up to the maximum session duration set for
// the role. The maximum session duration setting can have a value from 1 hour
// to 12 hours. If you specify a value higher than this setting or the administrator
// setting (whichever is lower), the operation fails. For example, if you specify
// a session duration of 12 hours, but your administrator set the maximum session
// duration to 6 hours, your operation fails.
//
// Role chaining limits your Amazon Web Services CLI or Amazon Web Services
// API role session to a maximum of one hour. When you use the AssumeRole API
// operation to assume a role, you can specify the duration of your role session
// with the DurationSeconds parameter. You can specify a parameter value of
// up to 43200 seconds (12 hours), depending on the maximum session duration
// setting for your role. However, if you assume a role using role chaining
// and provide a DurationSeconds parameter value greater than one hour, the
// operation fails. To learn how to view the maximum value for your role, see
// View the Maximum Session Duration Setting for a Role (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html#id_roles_use_view-role-max-session)
// in the IAM User Guide.
//
// By default, the value is set to 3600 seconds.
@@ -1422,7 +1382,7 @@ type AssumeRoleInput struct {
// to the federation endpoint for a console sign-in token takes a SessionDuration
// parameter that specifies the maximum length of the console session. For more
// information, see Creating a URL that Enables Federated Users to Access the
// Management Console (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_enable-console-custom-url.html)
// Amazon Web Services Management Console (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_enable-console-custom-url.html)
// in the IAM User Guide.
DurationSeconds *int64 `min:"900" type:"integer"`
@@ -1548,7 +1508,7 @@ type AssumeRoleInput struct {
// A list of session tags that you want to pass. Each session tag consists of
// a key name and an associated value. For more information about session tags,
// see Tagging STS Sessions (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html)
// see Tagging Amazon Web Services STS Sessions (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html)
// in the IAM User Guide.
//
// This parameter is optional. You can pass up to 50 session tags. The plaintext
@@ -1858,7 +1818,7 @@ type AssumeRoleWithSAMLInput struct {
// to the federation endpoint for a console sign-in token takes a SessionDuration
// parameter that specifies the maximum length of the console session. For more
// information, see Creating a URL that Enables Federated Users to Access the
// Management Console (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_enable-console-custom-url.html)
// Amazon Web Services Management Console (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_enable-console-custom-url.html)
// in the IAM User Guide.
DurationSeconds *int64 `min:"900" type:"integer"`
@@ -2205,7 +2165,7 @@ type AssumeRoleWithWebIdentityInput struct {
// to the federation endpoint for a console sign-in token takes a SessionDuration
// parameter that specifies the maximum length of the console session. For more
// information, see Creating a URL that Enables Federated Users to Access the
// Management Console (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_enable-console-custom-url.html)
// Amazon Web Services Management Console (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_enable-console-custom-url.html)
// in the IAM User Guide.
DurationSeconds *int64 `min:"900" type:"integer"`
@@ -2705,7 +2665,7 @@ func (s *DecodeAuthorizationMessageInput) SetEncodedMessage(v string) *DecodeAut
type DecodeAuthorizationMessageOutput struct {
_ struct{} `type:"structure"`
// An XML document that contains the decoded message.
// The API returns a response with the decoded message.
DecodedMessage *string `type:"string"`
}
@@ -3237,8 +3197,8 @@ type GetSessionTokenInput struct {
// user has a policy that requires MFA authentication. The value is either the
// serial number for a hardware device (such as GAHT12345678) or an Amazon Resource
// Name (ARN) for a virtual device (such as arn:aws:iam::123456789012:mfa/user).
// You can find the device for an IAM user by going to the Management Console
// and viewing the user's security credentials.
// You can find the device for an IAM user by going to the Amazon Web Services
// Management Console and viewing the user's security credentials.
//
// The regex used to validate this parameter is a string of characters consisting
// of upper- and lower-case alphanumeric characters with no spaces. You can
@@ -3400,9 +3360,9 @@ func (s *PolicyDescriptorType) SetArn(v string) *PolicyDescriptorType {
// You can pass custom key-value pair attributes when you assume a role or federate
// a user. These are called session tags. You can then use the session tags
// to control access to resources. For more information, see Tagging STS Sessions
// (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html) in
// the IAM User Guide.
// to control access to resources. For more information, see Tagging Amazon
// Web Services STS Sessions (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html)
// in the IAM User Guide.
type Tag struct {
_ struct{} `type:"structure"`

2
vendor/github.com/aws/aws-sdk-go/service/sts/errors.go сгенерированный поставляемый
Просмотреть файл

@@ -67,7 +67,7 @@ const (
//
// You could receive this error even though you meet other defined session policy
// and session tag limits. For more information, see IAM and STS Entity Character
// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html)
// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html#reference_iam-limits-entity-length)
// in the IAM User Guide.
ErrCodePackedPolicyTooLargeException = "PackedPolicyTooLarge"

19
vendor/github.com/aws/aws-sdk-go/service/sts/service.go сгенерированный поставляемый
Просмотреть файл

@@ -48,22 +48,23 @@ const (
// svc := sts.New(mySession, aws.NewConfig().WithRegion("us-west-2"))
func New(p client.ConfigProvider, cfgs ...*aws.Config) *STS {
c := p.ClientConfig(EndpointsID, cfgs...)
return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName)
return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName, c.ResolvedRegion)
}
// newClient creates, initializes and returns a new service client instance.
func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint, signingRegion, signingName string) *STS {
func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint, signingRegion, signingName, resolvedRegion string) *STS {
svc := &STS{
Client: client.New(
cfg,
metadata.ClientInfo{
ServiceName: ServiceName,
ServiceID: ServiceID,
SigningName: signingName,
SigningRegion: signingRegion,
PartitionID: partitionID,
Endpoint: endpoint,
APIVersion: "2011-06-15",
ServiceName: ServiceName,
ServiceID: ServiceID,
SigningName: signingName,
SigningRegion: signingRegion,
PartitionID: partitionID,
Endpoint: endpoint,
APIVersion: "2011-06-15",
ResolvedRegion: resolvedRegion,
},
handlers,
),

3
vendor/github.com/dsnet/compress/go.mod сгенерированный поставляемый
Просмотреть файл

@@ -4,7 +4,8 @@ go 1.9
require (
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780
github.com/google/go-cmp v0.5.5 // indirect
github.com/klauspost/compress v1.4.1
github.com/klauspost/cpuid v1.2.0 // indirect
github.com/ulikunitz/xz v0.5.6
github.com/ulikunitz/xz v0.5.8
)

8
vendor/github.com/dsnet/compress/go.sum сгенерированный поставляемый
Просмотреть файл

@@ -1,8 +1,12 @@
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780 h1:tFh1tRc4CA31yP6qDcu+Trax5wW5GuMxvkIba07qVLY=
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8=
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ=
github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

23
vendor/github.com/fcjr/aia-transport-go/.travis.yml сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,23 @@
language: go
go:
- "1.13.x"
- "1.x"
# - tip
os:
- linux
- osx
- windows
before_install:
- go get -t -v ./...
before_script:
- go vet ./...
script:
- go test -race -coverprofile=coverage.txt -covermode=atomic
after_success:
- bash <(curl -s https://codecov.io/bash)

21
vendor/github.com/fcjr/aia-transport-go/LICENSE сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2020 Frank Chiarulli Jr.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

55
vendor/github.com/fcjr/aia-transport-go/README.md сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,55 @@
# AIA (Authority Information Access) Transport Go
[![GoDoc][doc-img]][doc] [![Build Status][ci-img]][ci] [![Go Report Card][report-card-img]][report-card] [![GolangCI][golangci-lint-img]][golangci-lint] [![Coverage Status][cov-img]][cov]
AIA-Transport-Go provides an http.Transport which uses the AIA (Authority Information Access) X.509 extension to resolve incomplete certificate chains during the tls handshake. See [rfc3280](https://tools.ietf.org/html/rfc3280#section-4.2.2.1) for more details.
## Installation
`go get github.com/fcjr/aia-transport-go`
## Usage
```go
tr, err := aia.NewTransport()
if err != nil {
log.Fatal(err)
}
client := http.Client{
Transport: tr,
}
res, err := client.Get("https://incomplete-chain.badssl.com/")
if err != nil {
log.Fatal(err)
}
fmt.Println(res.Status)
```
## Todos
* [X] Follow single incomplete AIA chain
* [X] Tests
* [X] CI & Code Coverage
* [X] Documentation
* [X] Chain Caching
* [ ] Certificate Caching
* [ ] Follow all possible issuing urls
* [ ] Benchmarks
## Caveats
This library relies on [x509.SystemCertPool()](https://golang.org/pkg/crypto/x509/#SystemCertPool) to gather the initial system root certificates to validate against. This function is not implemented on windows, however windows will resolve incomplete certificate chains via AIA automatically so this library simply returns a default http.Transport on windows which should be sufficient.[^1]
[^1]: https://github.com/golang/go/issues/31773#issuecomment-582176197
[doc-img]: https://img.shields.io/static/v1?label=godoc&message=reference&color=blue
[doc]: https://pkg.go.dev/github.com/fcjr/aia-transport-go?tab=doc
[ci-img]: https://travis-ci.org/fcjr/aia-transport-go.svg?branch=master
[ci]: https://travis-ci.org/fcjr/aia-transport-go
[report-card-img]: https://goreportcard.com/badge/github.com/fcjr/aia-transport-go
[report-card]: https://goreportcard.com/report/github.com/fcjr/aia-transport-go
[golangci-lint-img]: https://github.com/fcjr/aia-transport-go/workflows/golangci-lint/badge.svg
[golangci-lint]: https://github.com/fcjr/aia-transport-go/actions?query=workflow%3Agolangci-lint
[cov-img]: https://codecov.io/gh/fcjr/aia-transport-go/branch/master/graph/badge.svg
[cov]: https://codecov.io/gh/fcjr/aia-transport-go

15
vendor/github.com/fcjr/aia-transport-go/codecov.yml сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,15 @@
codecov:
require_ci_to_pass: yes
notify:
after_n_builds: 6
wait_for_ci: yes
coverage:
range: 60..80
status:
project:
default:
threshold: 1 # check will fail if coverage drops >1%
patch:
default:
threshold: 1 # check will fail if coverage drops >1%

22
vendor/github.com/fcjr/aia-transport-go/doc.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,22 @@
// Package aia provides an http.Transport which uses the AIA (Authority Information Access)
// X.509 extension to resolve incomplete certificate chains during the tls handshake.
// See https://tools.ietf.org/html/rfc3280#section-4.2.2.1 for more details.
//
// Usage
//
// To use simply create a new transport via NewTransport()
// and use it in your http.Client.
//
// tr, err := aia.NewTransport()
// if err != nil {
// log.Fatal(err)
// }
// client := http.Client{
// Transport: tr,
// }
// res, err := client.Get("https://incomplete-chain.badssl.com/")
// if err != nil {
// log.Fatal(err)
// }
// fmt.Println(res.Status)
package aia //import "github.com/fcjr/aia-transport-go"

3
vendor/github.com/fcjr/aia-transport-go/go.mod сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,3 @@
module github.com/fcjr/aia-transport-go
go 1.13

117
vendor/github.com/fcjr/aia-transport-go/transport.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,117 @@
package aia
import (
"crypto/tls"
"crypto/x509"
"errors"
"io/ioutil"
"net"
"net/http"
"runtime"
"time"
)
// NewTransport returns a http.Transport that supports AIA (Authority Information Access) resolution
// for incomplete certificate chains.
func NewTransport() (*http.Transport, error) {
// Support windows.
if runtime.GOOS == "windows" {
return &http.Transport{}, nil
}
rootCAs, err := x509.SystemCertPool()
if err != nil {
return nil, err
}
return &http.Transport{
TLSClientConfig: &tls.Config{
RootCAs: rootCAs,
},
DialTLS: func(network, addr string) (net.Conn, error) {
conn, err := tls.Dial(network, addr, &tls.Config{
InsecureSkipVerify: true,
RootCAs: rootCAs,
VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
serverName, _, err := net.SplitHostPort(addr)
if err != nil {
return err
}
return verifyPeerCerts(rootCAs, serverName, rawCerts)
},
})
if err != nil {
return conn, err
}
return conn, nil
},
}, nil
}
func verifyPeerCerts(rootCAs *x509.CertPool, serverName string, rawCerts [][]byte) error {
certs := make([]*x509.Certificate, len(rawCerts))
for i, asn1Data := range rawCerts {
cert, err := x509.ParseCertificate(asn1Data)
if err != nil {
return errors.New("failed to parse certificate from server: " + err.Error())
}
certs[i] = cert
}
opts := &x509.VerifyOptions{
Roots: rootCAs,
CurrentTime: time.Now(),
DNSName: serverName,
Intermediates: x509.NewCertPool(),
}
for _, cert := range certs[1:] {
opts.Intermediates.AddCert(cert)
}
_, err := certs[0].Verify(*opts)
if err != nil {
if _, ok := err.(x509.UnknownAuthorityError); ok {
if len(certs[0].IssuingCertificateURL) >= 1 && certs[0].IssuingCertificateURL[0] != "" {
return verifyIncompleteChain(certs[0].IssuingCertificateURL[0], certs[0], opts)
}
}
return err
}
return nil
}
func verifyIncompleteChain(issuingCertificateURL string, baseCert *x509.Certificate, opts *x509.VerifyOptions) error {
issuer, err := getCert(issuingCertificateURL)
if err != nil {
return err
}
opts.Intermediates.AddCert(issuer)
_, err = baseCert.Verify(*opts)
if err != nil {
if _, ok := err.(x509.UnknownAuthorityError); ok {
if len(issuer.IssuingCertificateURL) >= 1 && issuer.IssuingCertificateURL[0] != "" {
return verifyIncompleteChain(issuer.IssuingCertificateURL[0], baseCert, opts)
}
}
return err
}
return nil
}
func getCert(url string) (*x509.Certificate, error) {
resp, err := http.Get(url)
if resp != nil {
defer resp.Body.Close()
}
if err != nil {
return nil, err
}
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return x509.ParseCertificate(data)
}

21
vendor/github.com/go-resty/resty/v2/.travis.yml сгенерированный поставляемый
Просмотреть файл

@@ -1,21 +0,0 @@
language: go
os: linux
go: # use travis ci resource effectively, keep always latest 2 versions and tip :)
- 1.15.x
- 1.14.x
- tip
install:
- go get -v -t ./...
script:
- go test ./... -race -coverprofile=coverage.txt -covermode=atomic
after_success:
- bash <(curl -s https://codecov.io/bash)
jobs:
allow_failures:
- go: tip

60
vendor/github.com/go-resty/resty/v2/BUILD.bazel сгенерированный поставляемый
Просмотреть файл

@@ -1,36 +1,48 @@
package(default_visibility = ["//visibility:private"])
load("@bazel_gazelle//:def.bzl", "gazelle")
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
load("@bazel_gazelle//:def.bzl", "gazelle")
gazelle(
name = "gazelle",
command = "fix",
prefix = "github.com/go-resty/resty/v2",
)
# gazelle:prefix github.com/go-resty/resty/v2
# gazelle:go_naming_convention import_alias
gazelle(name = "gazelle")
go_library(
name = "go_default_library",
srcs = glob(
["*.go"],
exclude = ["*_test.go"],
),
name = "resty",
srcs = [
"client.go",
"middleware.go",
"redirect.go",
"request.go",
"response.go",
"resty.go",
"retry.go",
"trace.go",
"transport.go",
"transport112.go",
"util.go",
],
importpath = "github.com/go-resty/resty/v2",
visibility = ["//visibility:public"],
deps = ["@org_golang_x_net//publicsuffix:go_default_library"],
)
go_test(
name = "go_default_test",
srcs =
glob(
["*_test.go"],
exclude = ["example_test.go"],
),
data = glob([".testdata/*"]),
embed = [":go_default_library"],
importpath = "github.com/go-resty/resty/v2",
deps = [
"@org_golang_x_net//proxy:go_default_library",
name = "resty_test",
srcs = [
"client_test.go",
"context_test.go",
"example_test.go",
"request_test.go",
"resty_test.go",
"retry_test.go",
"util_test.go",
],
data = glob([".testdata/*"]),
embed = [":resty"],
deps = ["@org_golang_x_net//proxy:go_default_library"],
)
alias(
name = "go_default_library",
actual = ":resty",
visibility = ["//visibility:public"],
)

31
vendor/github.com/go-resty/resty/v2/README.md сгенерированный поставляемый
Просмотреть файл

@@ -4,7 +4,7 @@
<p align="center"><a href="#features">Features</a> section describes in detail about Resty capabilities</p>
</p>
<p align="center">
<p align="center"><a href="https://travis-ci.org/go-resty/resty"><img src="https://travis-ci.org/go-resty/resty.svg?branch=master" alt="Build Status"></a> <a href="https://codecov.io/gh/go-resty/resty/branch/master"><img src="https://codecov.io/gh/go-resty/resty/branch/master/graph/badge.svg" alt="Code Coverage"></a> <a href="https://goreportcard.com/report/go-resty/resty"><img src="https://goreportcard.com/badge/go-resty/resty" alt="Go Report Card"></a> <a href="https://github.com/go-resty/resty/releases/latest"><img src="https://img.shields.io/badge/version-2.6.0-blue.svg" alt="Release Version"></a> <a href="https://pkg.go.dev/github.com/go-resty/resty/v2"><img src="https://pkg.go.dev/badge/github.com/go-resty/resty" alt="GoDoc"></a> <a href="LICENSE"><img src="https://img.shields.io/github/license/go-resty/resty.svg" alt="License"></a> <a href="https://github.com/avelino/awesome-go"><img src="https://awesome.re/mentioned-badge.svg" alt="Mentioned in Awesome Go"></a></p>
<p align="center"><a href="https://github.com/go-resty/resty/actions/workflows/ci.yml?query=branch%3Amaster"><img src="https://github.com/go-resty/resty/actions/workflows/ci.yml/badge.svg" alt="Build Status"></a> <a href="https://codecov.io/gh/go-resty/resty/branch/master"><img src="https://codecov.io/gh/go-resty/resty/branch/master/graph/badge.svg" alt="Code Coverage"></a> <a href="https://goreportcard.com/report/go-resty/resty"><img src="https://goreportcard.com/badge/go-resty/resty" alt="Go Report Card"></a> <a href="https://github.com/go-resty/resty/releases/latest"><img src="https://img.shields.io/badge/version-2.7.0-blue.svg" alt="Release Version"></a> <a href="https://pkg.go.dev/github.com/go-resty/resty/v2"><img src="https://pkg.go.dev/badge/github.com/go-resty/resty" alt="GoDoc"></a> <a href="LICENSE"><img src="https://img.shields.io/github/license/go-resty/resty.svg" alt="License"></a> <a href="https://github.com/avelino/awesome-go"><img src="https://awesome.re/mentioned-badge.svg" alt="Mentioned in Awesome Go"></a></p>
</p>
<p align="center">
<h4 align="center">Resty Communication Channels</h4>
@@ -13,7 +13,7 @@
## News
* v2.6.0 [released](https://github.com/go-resty/resty/releases/tag/v2.6.0) and tagged on Apr 09, 2021.
* v2.7.0 [released](https://github.com/go-resty/resty/releases/tag/v2.7.0) and tagged on Nov 03, 2021.
* v2.0.0 [released](https://github.com/go-resty/resty/releases/tag/v2.0.0) and tagged on Jul 16, 2019.
* v1.12.0 [released](https://github.com/go-resty/resty/releases/tag/v1.12.0) and tagged on Feb 27, 2019.
* v1.0 released and tagged on Sep 25, 2017. - Resty's first version was released on Sep 15, 2015 then it grew gradually as a very handy and helpful library. Its been a two years since first release. I'm very thankful to Resty users and its [contributors](https://github.com/go-resty/resty/graphs/contributors).
@@ -36,6 +36,7 @@
- Success scenario [Request.SetResult()](https://pkg.go.dev/github.com/go-resty/resty/v2#Request.SetResult) and [Response.Result()](https://pkg.go.dev/github.com/go-resty/resty/v2#Response.Result).
- Error scenario [Request.SetError()](https://pkg.go.dev/github.com/go-resty/resty/v2#Request.SetError) and [Response.Error()](https://pkg.go.dev/github.com/go-resty/resty/v2#Response.Error).
- Supports [RFC7807](https://tools.ietf.org/html/rfc7807) - `application/problem+json` & `application/problem+xml`
* Resty provides an option to override [JSON Marshal/Unmarshal and XML Marshal/Unmarshal](#override-json--xml-marshalunmarshal)
* Easy to upload one or more file(s) via `multipart/form-data`
* Auto detects file content type
* Request URL [Path Params (aka URI Params)](https://pkg.go.dev/github.com/go-resty/resty/v2#Request.SetPathParams)
@@ -107,7 +108,7 @@ Resty author also published following projects for Go Community.
```bash
# Go Modules
require github.com/go-resty/resty/v2 v2.4.0
require github.com/go-resty/resty/v2 v2.7.0
```
## Usage
@@ -359,6 +360,24 @@ resp, err := client.R().
Options("https://myapp.com/servers/nyc-dc-01")
```
#### Override JSON & XML Marshal/Unmarshal
User could register choice of JSON/XML library into resty or write your own. By default resty registers standard `encoding/json` and `encoding/xml` respectively.
```go
// Example of registering json-iterator
import jsoniter "github.com/json-iterator/go"
json := jsoniter.ConfigCompatibleWithStandardLibrary
client := resty.New()
client.JSONMarshal = json.Marshal
client.JSONUnmarshal = json.Unmarshal
// similarly user could do for XML too with -
client.XMLMarshal
client.XMLUnmarshal
```
### Multipart File(s) upload
#### Using io.Reader
@@ -829,13 +848,13 @@ client.SetTransport(&transport).SetScheme("http").SetHostURL(unixSocket)
client.R().Get("/index.html")
```
#### Bazel support
#### Bazel Support
Resty can be built, tested and depended upon via [Bazel](https://bazel.build).
For example, to run all tests:
```shell
bazel test :go_default_test
bazel test :resty_test
```
#### Mocking http requests using [httpmock](https://github.com/jarcoal/httpmock) library
@@ -876,7 +895,7 @@ BTW, I'd like to know what you think about `Resty`. Kindly open an issue or send
## Core Team
Have a look on [Members](https://github.com/orgs/go-resty/teams/core/members) page.
Have a look on [Members](https://github.com/orgs/go-resty/people) page.
## Contributors

28
vendor/github.com/go-resty/resty/v2/WORKSPACE сгенерированный поставляемый
Просмотреть файл

@@ -1,26 +1,30 @@
workspace(name = "resty")
git_repository(
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
http_archive(
name = "io_bazel_rules_go",
remote = "https://github.com/bazelbuild/rules_go.git",
tag = "0.13.0",
sha256 = "69de5c704a05ff37862f7e0f5534d4f479418afc21806c887db544a316f3cb6b",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.27.0/rules_go-v0.27.0.tar.gz",
"https://github.com/bazelbuild/rules_go/releases/download/v0.27.0/rules_go-v0.27.0.tar.gz",
],
)
git_repository(
http_archive(
name = "bazel_gazelle",
remote = "https://github.com/bazelbuild/bazel-gazelle.git",
tag = "0.13.0",
sha256 = "62ca106be173579c0a167deb23358fdfe71ffa1e4cfdddf5582af26520f1c66f",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.23.0/bazel-gazelle-v0.23.0.tar.gz",
"https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.23.0/bazel-gazelle-v0.23.0.tar.gz",
],
)
load(
"@io_bazel_rules_go//go:def.bzl",
"go_rules_dependencies",
"go_register_toolchains",
)
load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
go_rules_dependencies()
go_register_toolchains()
go_register_toolchains(version = "1.16")
load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")

50
vendor/github.com/go-resty/resty/v2/client.go сгенерированный поставляемый
Просмотреть файл

@@ -10,6 +10,7 @@ import (
"crypto/tls"
"crypto/x509"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"io"
@@ -92,9 +93,11 @@ type (
// Resty also provides an options to override most of the client settings
// at request level.
type Client struct {
HostURL string
BaseURL string
HostURL string // Deprecated: use BaseURL instead. To be removed in v3.0.0 release.
QueryParam url.Values
FormData url.Values
PathParams map[string]string
Header http.Header
UserInfo *User
Token string
@@ -112,6 +115,8 @@ type Client struct {
RetryAfter RetryAfterFunc
JSONMarshal func(v interface{}) ([]byte, error)
JSONUnmarshal func(data []byte, v interface{}) error
XMLMarshal func(v interface{}) ([]byte, error)
XMLUnmarshal func(data []byte, v interface{}) error
// HeaderAuthorizationKey is used to set/access Request Authorization header
// value when `SetAuthToken` option is used.
@@ -125,7 +130,6 @@ type Client struct {
debugBodySizeLimit int64
outputDirectory string
scheme string
pathParams map[string]string
log Logger
httpClient *http.Client
proxyURL *url.URL
@@ -154,8 +158,25 @@ type User struct {
//
// // Setting HTTPS address
// client.SetHostURL("https://myjeeva.com")
//
// Deprecated: use SetBaseURL instead. To be removed in v3.0.0 release.
func (c *Client) SetHostURL(url string) *Client {
c.HostURL = strings.TrimRight(url, "/")
c.SetBaseURL(url)
return c
}
// SetBaseURL method is to set Base URL in the client instance. It will be used with request
// raised from this client with relative URL
// // Setting HTTP address
// client.SetBaseURL("http://myjeeva.com")
//
// // Setting HTTPS address
// client.SetBaseURL("https://myjeeva.com")
//
// Since v2.7.0
func (c *Client) SetBaseURL(url string) *Client {
c.BaseURL = strings.TrimRight(url, "/")
c.HostURL = c.BaseURL
return c
}
@@ -367,7 +388,7 @@ func (c *Client) R() *Request {
client: c,
multipartFiles: []*File{},
multipartFields: []*MultipartField{},
pathParams: map[string]string{},
PathParams: map[string]string{},
jsonEscapeHTML: true,
}
return r
@@ -589,6 +610,9 @@ func (c *Client) SetRetryAfter(callback RetryAfterFunc) *Client {
// AddRetryCondition method adds a retry condition function to array of functions
// that are checked to determine if the request is retried. The request will
// retry if any of the functions return true and error is nil.
//
// Note: These retry conditions are applied on all Request made using this Client.
// For Request specific retry conditions check *Request.AddRetryCondition
func (c *Client) AddRetryCondition(condition RetryConditionFunc) *Client {
c.RetryConditions = append(c.RetryConditions, condition)
return c
@@ -758,7 +782,7 @@ func (c *Client) SetTransport(transport http.RoundTripper) *Client {
// client.SetScheme("http")
func (c *Client) SetScheme(scheme string) *Client {
if !IsStringEmpty(scheme) {
c.scheme = scheme
c.scheme = strings.TrimSpace(scheme)
}
return c
}
@@ -793,7 +817,7 @@ func (c *Client) SetDoNotParseResponse(parse bool) *Client {
// Also it can be overridden at request level Path Params options,
// see `Request.SetPathParam` or `Request.SetPathParams`.
func (c *Client) SetPathParam(param, value string) *Client {
c.pathParams[param] = value
c.PathParams[param] = value
return c
}
@@ -869,7 +893,6 @@ func (c *Client) GetClient() *http.Client {
// Executes method executes the given `Request` object and returns response
// error.
func (c *Client) execute(req *Request) (*Response, error) {
defer releaseBuffer(req.bodyBuf)
// Apply Request middleware
var err error
@@ -903,6 +926,8 @@ func (c *Client) execute(req *Request) (*Response, error) {
return nil, wrapNoRetryErr(err)
}
req.RawRequest.Body = newRequestBodyReleaser(req.RawRequest.Body, req.bodyBuf)
req.Time = time.Now()
resp, err := c.httpClient.Do(req.RawRequest)
@@ -1052,13 +1077,16 @@ func createClient(hc *http.Client) *Client {
Cookies: make([]*http.Cookie, 0),
RetryWaitTime: defaultWaitTime,
RetryMaxWaitTime: defaultMaxWaitTime,
PathParams: make(map[string]string),
JSONMarshal: json.Marshal,
JSONUnmarshal: json.Unmarshal,
XMLMarshal: xml.Marshal,
XMLUnmarshal: xml.Unmarshal,
HeaderAuthorizationKey: http.CanonicalHeaderKey("Authorization"),
jsonEscapeHTML: true,
httpClient: hc,
debugBodySizeLimit: math.MaxInt32,
pathParams: make(map[string]string),
jsonEscapeHTML: true,
httpClient: hc,
debugBodySizeLimit: math.MaxInt32,
}
// Logger

2
vendor/github.com/go-resty/resty/v2/go.mod сгенерированный поставляемый
Просмотреть файл

@@ -1,5 +1,5 @@
module github.com/go-resty/resty/v2
require golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
require golang.org/x/net v0.0.0-20211029224645-99673261e6eb
go 1.11

8
vendor/github.com/go-resty/resty/v2/go.sum сгенерированный поставляемый
Просмотреть файл

@@ -1,7 +1,7 @@
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20211029224645-99673261e6eb h1:pirldcYWx7rx7kE5r+9WsOXPXK0+WH5+uZ7uPmJ44uM=
golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

24
vendor/github.com/go-resty/resty/v2/middleware.go сгенерированный поставляемый
Просмотреть файл

@@ -6,7 +6,6 @@ package resty
import (
"bytes"
"encoding/xml"
"errors"
"fmt"
"io"
@@ -29,13 +28,13 @@ const debugRequestLogKey = "__restyDebugRequestLog"
func parseRequestURL(c *Client, r *Request) error {
// GitHub #103 Path Params
if len(r.pathParams) > 0 {
for p, v := range r.pathParams {
if len(r.PathParams) > 0 {
for p, v := range r.PathParams {
r.URL = strings.Replace(r.URL, "{"+p+"}", url.PathEscape(v), -1)
}
}
if len(c.pathParams) > 0 {
for p, v := range c.pathParams {
if len(c.PathParams) > 0 {
for p, v := range c.PathParams {
r.URL = strings.Replace(r.URL, "{"+p+"}", url.PathEscape(v), -1)
}
}
@@ -60,6 +59,11 @@ func parseRequestURL(c *Client, r *Request) error {
}
}
// GH #407 && #318
if reqURL.Scheme == "" && len(c.scheme) > 0 {
reqURL.Scheme = c.scheme
}
// Adding Query Param
query := make(url.Values)
for k, v := range c.QueryParam {
@@ -191,12 +195,6 @@ func createHTTPRequest(c *Client, r *Request) (err error) {
r.RawRequest.AddCookie(cookie)
}
// it's for non-http scheme option
if r.RawRequest.URL != nil && r.RawRequest.URL.Scheme == "" {
r.RawRequest.URL.Scheme = c.scheme
r.RawRequest.URL.Host = r.URL
}
// Enable trace
if c.trace || r.trace {
r.clientTrace = &clientTrace{}
@@ -458,12 +456,12 @@ func handleRequestBody(c *Client, r *Request) (err error) {
bodyBytes = []byte(s)
} else if IsJSONType(contentType) &&
(kind == reflect.Struct || kind == reflect.Map || kind == reflect.Slice) {
bodyBytes, err = jsonMarshal(c, r, r.Body)
r.bodyBuf, err = jsonMarshal(c, r, r.Body)
if err != nil {
return
}
} else if IsXMLType(contentType) && (kind == reflect.Struct) {
bodyBytes, err = xml.Marshal(r.Body)
bodyBytes, err = c.XMLMarshal(r.Body)
if err != nil {
return
}

46
vendor/github.com/go-resty/resty/v2/request.go сгенерированный поставляемый
Просмотреть файл

@@ -33,6 +33,7 @@ type Request struct {
AuthScheme string
QueryParam url.Values
FormData url.Values
PathParams map[string]string
Header http.Header
Time time.Time
Body interface{}
@@ -60,13 +61,13 @@ type Request struct {
fallbackContentType string
forceContentType string
ctx context.Context
pathParams map[string]string
values map[string]interface{}
client *Client
bodyBuf *bytes.Buffer
clientTrace *clientTrace
multipartFiles []*File
multipartFields []*MultipartField
retryConditions []RetryConditionFunc
}
// Context method returns the Context if its already set in request
@@ -117,6 +118,22 @@ func (r *Request) SetHeaders(headers map[string]string) *Request {
return r
}
// SetHeaderMultiValues sets multiple headers fields and its values is list of strings at one go in the current request.
//
// For Example: To set `Accept` as `text/html, application/xhtml+xml, application/xml;q=0.9, image/webp, */*;q=0.8`
//
// client.R().
// SetHeaderMultiValues(map[string][]string{
// "Accept": []string{"text/html", "application/xhtml+xml", "application/xml;q=0.9", "image/webp", "*/*;q=0.8"},
// })
// Also you can override header value, which was set at client instance level.
func (r *Request) SetHeaderMultiValues(headers map[string][]string) *Request {
for key, values := range headers {
r.SetHeader(key, strings.Join(values, ", "))
}
return r
}
// SetHeaderVerbatim method is to set a single header field and its value verbatim in the current request.
//
// For Example: To set `all_lowercase` and `UPPERCASE` as `available`.
@@ -494,7 +511,7 @@ func (r *Request) SetDoNotParseResponse(parse bool) *Request {
// It replaces the value of the key while composing the request URL. Also you can
// override Path Params value, which was set at client instance level.
func (r *Request) SetPathParam(param, value string) *Request {
r.pathParams[param] = value
r.PathParams[param] = value
return r
}
@@ -577,6 +594,18 @@ func (r *Request) SetCookies(rs []*http.Cookie) *Request {
return r
}
// AddRetryCondition method adds a retry condition function to the request's
// array of functions that are checked to determine if the request is retried.
// The request will retry if any of the functions return true and error is nil.
//
// Note: These retry conditions are checked before all retry conditions of the client.
//
// Since v2.7.0
func (r *Request) AddRetryCondition(condition RetryConditionFunc) *Request {
r.retryConditions = append(r.retryConditions, condition)
return r
}
//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
// HTTP request tracing
//_______________________________________________________________________
@@ -747,7 +776,7 @@ func (r *Request) Execute(method, url string) (*Response, error) {
Retries(r.client.RetryCount),
WaitTime(r.client.RetryWaitTime),
MaxWaitTime(r.client.RetryMaxWaitTime),
RetryConditions(r.client.RetryConditions),
RetryConditions(append(r.retryConditions, r.client.RetryConditions...)),
RetryHooks(r.client.RetryHooks),
)
@@ -854,11 +883,14 @@ func (r *Request) initValuesMap() {
}
}
var noescapeJSONMarshal = func(v interface{}) ([]byte, error) {
var noescapeJSONMarshal = func(v interface{}) (*bytes.Buffer, error) {
buf := acquireBuffer()
defer releaseBuffer(buf)
encoder := json.NewEncoder(buf)
encoder.SetEscapeHTML(false)
err := encoder.Encode(v)
return buf.Bytes(), err
if err := encoder.Encode(v); err != nil {
releaseBuffer(buf)
return nil, err
}
return buf, nil
}

2
vendor/github.com/go-resty/resty/v2/resty.go сгенерированный поставляемый
Просмотреть файл

@@ -14,7 +14,7 @@ import (
)
// Version # of resty
const Version = "2.6.0"
const Version = "2.7.0"
// New method creates a new Resty client.
func New() *Client {

6
vendor/github.com/go-resty/resty/v2/retry.go сгенерированный поставляемый
Просмотреть файл

@@ -129,6 +129,12 @@ func Backoff(operation func() (*Response, error), options ...Option) error {
hook(resp, err)
}
// Don't need to wait when no retries left.
// Still run retry hooks even on last retry to keep compatibility.
if attempt == opts.maxRetries {
return err
}
waitTime, err2 := sleepDuration(resp, opts.waitTime, opts.maxWaitTime, attempt)
if err2 != nil {
if err == nil {

50
vendor/github.com/go-resty/resty/v2/util.go сгенерированный поставляемый
Просмотреть файл

@@ -6,7 +6,6 @@ package resty
import (
"bytes"
"encoding/xml"
"fmt"
"io"
"log"
@@ -19,6 +18,7 @@ import (
"runtime"
"sort"
"strings"
"sync"
)
//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
@@ -108,7 +108,7 @@ func Unmarshalc(c *Client, ct string, b []byte, d interface{}) (err error) {
if IsJSONType(ct) {
err = c.JSONUnmarshal(b, d)
} else if IsXMLType(ct) {
err = xml.Unmarshal(b, d)
err = c.XMLUnmarshal(b, d)
}
return
@@ -139,13 +139,19 @@ type ResponseLog struct {
//_______________________________________________________________________
// way to disable the HTML escape as opt-in
func jsonMarshal(c *Client, r *Request, d interface{}) ([]byte, error) {
if !r.jsonEscapeHTML {
return noescapeJSONMarshal(d)
} else if !c.jsonEscapeHTML {
func jsonMarshal(c *Client, r *Request, d interface{}) (*bytes.Buffer, error) {
if !r.jsonEscapeHTML || !c.jsonEscapeHTML {
return noescapeJSONMarshal(d)
}
return c.JSONMarshal(d)
data, err := c.JSONMarshal(d)
if err != nil {
return nil, err
}
buf := acquireBuffer()
_, _ = buf.Write(data)
return buf, nil
}
func firstNonEmpty(v ...string) string {
@@ -195,7 +201,7 @@ func writeMultipartFormFile(w *multipart.Writer, fieldName, fileName string, r i
// Auto detect actual multipart content type
cbuf := make([]byte, 512)
size, err := r.Read(cbuf)
if err != nil {
if err != nil && err != io.EOF {
return err
}
@@ -283,6 +289,34 @@ func releaseBuffer(buf *bytes.Buffer) {
}
}
// requestBodyReleaser wraps requests's body and implements custom Close for it.
// The Close method closes original body and releases request body back to sync.Pool.
type requestBodyReleaser struct {
releaseOnce sync.Once
reqBuf *bytes.Buffer
io.ReadCloser
}
func newRequestBodyReleaser(respBody io.ReadCloser, reqBuf *bytes.Buffer) io.ReadCloser {
if reqBuf == nil {
return respBody
}
return &requestBodyReleaser{
reqBuf: reqBuf,
ReadCloser: respBody,
}
}
func (rr *requestBodyReleaser) Close() error {
err := rr.ReadCloser.Close()
rr.releaseOnce.Do(func() {
releaseBuffer(rr.reqBuf)
})
return err
}
func closeq(v interface{}) {
if c, ok := v.(io.Closer); ok {
silently(c.Close())

8
vendor/github.com/golang-migrate/migrate/v4/Dockerfile.github-actions сгенерированный поставляемый
Просмотреть файл

@@ -2,8 +2,10 @@ FROM alpine:3.13
RUN apk add --no-cache ca-certificates
COPY migrate /usr/bin/migrate
RUN ln -s /usr/bin/migrate /migrate
COPY migrate /usr/local/bin/migrate
RUN ln -s /usr/local/bin/migrate /usr/bin/migrate
RUN ln -s /usr/local/bin/migrate /migrate
ENTRYPOINT ["migrate"]
CMD ["--help"]
CMD ["--help"]

37
vendor/github.com/golang-migrate/migrate/v4/go.mod сгенерированный поставляемый
Просмотреть файл

@@ -5,27 +5,24 @@ require (
cloud.google.com/go/storage v1.10.0
github.com/Azure/go-autorest/autorest/adal v0.9.14
github.com/ClickHouse/clickhouse-go v1.4.3
github.com/apache/arrow/go/arrow v0.0.0-20210521153258-78c88a9f517b // indirect
github.com/Microsoft/go-winio v0.5.0 // indirect
github.com/apache/arrow/go/arrow v0.0.0-20211013220434-5962184e7a30 // indirect
github.com/aws/aws-sdk-go v1.17.7
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.2.1 // indirect
github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 // indirect
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.4 // indirect
github.com/cenkalti/backoff/v4 v4.0.2
github.com/cockroachdb/cockroach-go/v2 v2.1.1
github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369 // indirect
github.com/denisenkom/go-mssqldb v0.10.0
github.com/dhui/dktest v0.3.4
github.com/docker/docker v17.12.0-ce-rc1.0.20210128214336-420b1d36250f+incompatible
github.com/dhui/dktest v0.3.7
github.com/docker/docker v20.10.9+incompatible
github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 // indirect
github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
github.com/fsouza/fake-gcs-server v1.17.0
github.com/gabriel-vasile/mimetype v1.4.0 // indirect
github.com/go-sql-driver/mysql v1.5.0
github.com/gobuffalo/here v0.6.0
github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4
github.com/gocql/gocql v0.0.0-20210515062232-b7ef815b4556
github.com/gofrs/uuid v4.0.0+incompatible // indirect
github.com/google/flatbuffers v2.0.0+incompatible // indirect
github.com/google/go-github/v35 v35.2.0
github.com/google/uuid v1.2.0 // indirect
github.com/gorilla/mux v1.7.4 // indirect
github.com/hashicorp/go-multierror v1.1.0
github.com/jackc/pgconn v1.8.0
@@ -33,29 +30,29 @@ require (
github.com/jackc/pgproto3/v2 v2.0.7 // indirect
github.com/jackc/pgx/v4 v4.10.1
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
github.com/klauspost/compress v1.12.2 // indirect
github.com/klauspost/compress v1.13.6 // indirect
github.com/ktrysmt/go-bitbucket v0.6.4
github.com/lib/pq v1.10.0
github.com/markbates/pkger v0.15.1
github.com/mattn/go-sqlite3 v1.14.6
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
github.com/mutecomm/go-sqlcipher/v4 v4.4.0
github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8
github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba
github.com/pierrec/lz4/v4 v4.1.7 // indirect
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/snowflakedb/gosnowflake v1.4.3
github.com/stretchr/testify v1.6.1
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
github.com/snowflakedb/gosnowflake v1.6.3
github.com/stretchr/testify v1.7.0
github.com/xanzy/go-gitlab v0.15.0
gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b // indirect
go.mongodb.org/mongo-driver v1.7.0
go.uber.org/atomic v1.6.0
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a // indirect
golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 // indirect
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
golang.org/x/net v0.0.0-20211013171255-e13a2654a71e // indirect
golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect
golang.org/x/tools v0.1.5
google.golang.org/api v0.51.0
google.golang.org/genproto v0.0.0-20210726143408-b02e89920bf0
google.golang.org/genproto v0.0.0-20211013025323-ce878158c4d4
google.golang.org/grpc v1.41.0 // indirect
modernc.org/b v1.0.0 // indirect
modernc.org/db v1.0.0 // indirect
modernc.org/file v1.0.0 // indirect

790
vendor/github.com/golang-migrate/migrate/v4/go.sum сгенерированный поставляемый

Разница между файлами не показана из-за своего большого размера Загрузить разницу

11
vendor/github.com/hashicorp/memberlist/config.go сгенерированный поставляемый
Просмотреть файл

@@ -21,6 +21,17 @@ type Config struct {
// make a NetTransport using BindAddr and BindPort from this structure.
Transport Transport
// Label is an optional set of bytes to include on the outside of each
// packet and stream.
//
// If gossip encryption is enabled and this is set it is treated as GCM
// authenticated data.
Label string
// SkipInboundLabelCheck skips the check that inbound packets and gossip
// streams need to be label prefixed.
SkipInboundLabelCheck bool
// Configuration related to what address to bind to and ports to
// listen on. The port is used for both UDP and TCP gossip. It is
// assumed other nodes are running on this port, but they do not need

178
vendor/github.com/hashicorp/memberlist/label.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,178 @@
package memberlist
import (
"bufio"
"fmt"
"io"
"net"
)
// General approach is to prefix all packets and streams with the same structure:
//
// magic type byte (244): uint8
// length of label name: uint8 (because labels can't be longer than 255 bytes)
// label name: []uint8
// LabelMaxSize is the maximum length of a packet or stream label.
const LabelMaxSize = 255
// AddLabelHeaderToPacket prefixes outgoing packets with the correct header if
// the label is not empty.
func AddLabelHeaderToPacket(buf []byte, label string) ([]byte, error) {
if label == "" {
return buf, nil
}
if len(label) > LabelMaxSize {
return nil, fmt.Errorf("label %q is too long", label)
}
return makeLabelHeader(label, buf), nil
}
// RemoveLabelHeaderFromPacket removes any label header from the provided
// packet and returns it along with the remaining packet contents.
func RemoveLabelHeaderFromPacket(buf []byte) (newBuf []byte, label string, err error) {
if len(buf) == 0 {
return buf, "", nil // can't possibly be labeled
}
// [type:byte] [size:byte] [size bytes]
msgType := messageType(buf[0])
if msgType != hasLabelMsg {
return buf, "", nil
}
if len(buf) < 2 {
return nil, "", fmt.Errorf("cannot decode label; packet has been truncated")
}
size := int(buf[1])
if size < 1 {
return nil, "", fmt.Errorf("label header cannot be empty when present")
}
if len(buf) < 2+size {
return nil, "", fmt.Errorf("cannot decode label; packet has been truncated")
}
label = string(buf[2 : 2+size])
newBuf = buf[2+size:]
return newBuf, label, nil
}
// AddLabelHeaderToStream prefixes outgoing streams with the correct header if
// the label is not empty.
func AddLabelHeaderToStream(conn net.Conn, label string) error {
if label == "" {
return nil
}
if len(label) > LabelMaxSize {
return fmt.Errorf("label %q is too long", label)
}
header := makeLabelHeader(label, nil)
_, err := conn.Write(header)
return err
}
// RemoveLabelHeaderFromStream removes any label header from the beginning of
// the stream if present and returns it along with an updated conn with that
// header removed.
//
// Note that on error it is the caller's responsibility to close the
// connection.
func RemoveLabelHeaderFromStream(conn net.Conn) (net.Conn, string, error) {
br := bufio.NewReader(conn)
// First check for the type byte.
peeked, err := br.Peek(1)
if err != nil {
if err == io.EOF {
// It is safe to return the original net.Conn at this point because
// it never contained any data in the first place so we don't have
// to splice the buffer into the conn because both are empty.
return conn, "", nil
}
return nil, "", err
}
msgType := messageType(peeked[0])
if msgType != hasLabelMsg {
conn, err = newPeekedConnFromBufferedReader(conn, br, 0)
return conn, "", err
}
// We are guaranteed to get a size byte as well.
peeked, err = br.Peek(2)
if err != nil {
if err == io.EOF {
return nil, "", fmt.Errorf("cannot decode label; stream has been truncated")
}
return nil, "", err
}
size := int(peeked[1])
if size < 1 {
return nil, "", fmt.Errorf("label header cannot be empty when present")
}
// NOTE: we don't have to check this against LabelMaxSize because a byte
// already has a max value of 255.
// Once we know the size we can peek the label as well. Note that since we
// are using the default bufio.Reader size of 4096, the entire label header
// fits in the initial buffer fill so this should be free.
peeked, err = br.Peek(2 + size)
if err != nil {
if err == io.EOF {
return nil, "", fmt.Errorf("cannot decode label; stream has been truncated")
}
return nil, "", err
}
label := string(peeked[2 : 2+size])
conn, err = newPeekedConnFromBufferedReader(conn, br, 2+size)
if err != nil {
return nil, "", err
}
return conn, label, nil
}
// newPeekedConnFromBufferedReader will splice the buffer contents after the
// offset into the provided net.Conn and return the result so that the rest of
// the buffer contents are returned first when reading from the returned
// peekedConn before moving on to the unbuffered conn contents.
func newPeekedConnFromBufferedReader(conn net.Conn, br *bufio.Reader, offset int) (*peekedConn, error) {
// Extract any of the readahead buffer.
peeked, err := br.Peek(br.Buffered())
if err != nil {
return nil, err
}
return &peekedConn{
Peeked: peeked[offset:],
Conn: conn,
}, nil
}
func makeLabelHeader(label string, rest []byte) []byte {
newBuf := make([]byte, 2, 2+len(label)+len(rest))
newBuf[0] = byte(hasLabelMsg)
newBuf[1] = byte(len(label))
newBuf = append(newBuf, []byte(label)...)
if len(rest) > 0 {
newBuf = append(newBuf, []byte(rest)...)
}
return newBuf
}
func labelOverhead(label string) int {
if label == "" {
return 0
}
return 2 + len(label)
}

13
vendor/github.com/hashicorp/memberlist/memberlist.go сгенерированный поставляемый
Просмотреть файл

@@ -187,6 +187,17 @@ func newMemberlist(conf *Config) (*Memberlist, error) {
nodeAwareTransport = &shimNodeAwareTransport{transport}
}
if len(conf.Label) > LabelMaxSize {
return nil, fmt.Errorf("could not use %q as a label: too long", conf.Label)
}
if conf.Label != "" {
nodeAwareTransport = &labelWrappedTransport{
label: conf.Label,
NodeAwareTransport: nodeAwareTransport,
}
}
m := &Memberlist{
config: conf,
shutdownCh: make(chan struct{}),
@@ -262,7 +273,7 @@ func (m *Memberlist) Join(existing []string) (int, error) {
hp := joinHostPort(addr.ip.String(), addr.port)
a := Address{Addr: hp, Name: addr.nodeName}
if err := m.pushPullNode(a, true); err != nil {
err = fmt.Errorf("Failed to join %s: %v", addr.ip, err)
err = fmt.Errorf("Failed to join %s: %v", a.Addr, err)
errs = multierror.Append(errs, err)
m.logger.Printf("[DEBUG] memberlist: %v", err)
continue

132
vendor/github.com/hashicorp/memberlist/net.go сгенерированный поставляемый
Просмотреть файл

@@ -42,6 +42,9 @@ const (
type messageType uint8
// The list of available message types.
//
// WARNING: ONLY APPEND TO THIS LIST! The numeric values are part of the
// protocol itself.
const (
pingMsg messageType = iota
indirectPingMsg
@@ -59,6 +62,13 @@ const (
errMsg
)
const (
// hasLabelMsg has a deliberately high value so that you can disambiguate
// it from the encryptionVersion header which is either 0/1 right now and
// also any of the existing messageTypes
hasLabelMsg messageType = 244
)
// compressionType is used to specify the compression algorithm
type compressionType uint8
@@ -226,7 +236,32 @@ func (m *Memberlist) handleConn(conn net.Conn) {
metrics.IncrCounter([]string{"memberlist", "tcp", "accept"}, 1)
conn.SetDeadline(time.Now().Add(m.config.TCPTimeout))
msgType, bufConn, dec, err := m.readStream(conn)
var (
streamLabel string
err error
)
conn, streamLabel, err = RemoveLabelHeaderFromStream(conn)
if err != nil {
m.logger.Printf("[ERR] memberlist: failed to receive and remove the stream label header: %s %s", err, LogConn(conn))
return
}
if m.config.SkipInboundLabelCheck {
if streamLabel != "" {
m.logger.Printf("[ERR] memberlist: unexpected double stream label header: %s", LogConn(conn))
return
}
// Set this from config so that the auth data assertions work below.
streamLabel = m.config.Label
}
if m.config.Label != streamLabel {
m.logger.Printf("[ERR] memberlist: discarding stream with unacceptable label %q: %s", streamLabel, LogConn(conn))
return
}
msgType, bufConn, dec, err := m.readStream(conn, streamLabel)
if err != nil {
if err != io.EOF {
m.logger.Printf("[ERR] memberlist: failed to receive: %s %s", err, LogConn(conn))
@@ -238,7 +273,7 @@ func (m *Memberlist) handleConn(conn net.Conn) {
return
}
err = m.rawSendMsgStream(conn, out.Bytes())
err = m.rawSendMsgStream(conn, out.Bytes(), streamLabel)
if err != nil {
m.logger.Printf("[ERR] memberlist: Failed to send error: %s %s", err, LogConn(conn))
return
@@ -269,7 +304,7 @@ func (m *Memberlist) handleConn(conn net.Conn) {
return
}
if err := m.sendLocalState(conn, join); err != nil {
if err := m.sendLocalState(conn, join, streamLabel); err != nil {
m.logger.Printf("[ERR] memberlist: Failed to push local state: %s %s", err, LogConn(conn))
return
}
@@ -297,7 +332,7 @@ func (m *Memberlist) handleConn(conn net.Conn) {
return
}
err = m.rawSendMsgStream(conn, out.Bytes())
err = m.rawSendMsgStream(conn, out.Bytes(), streamLabel)
if err != nil {
m.logger.Printf("[ERR] memberlist: Failed to send ack: %s %s", err, LogConn(conn))
return
@@ -322,10 +357,35 @@ func (m *Memberlist) packetListen() {
}
func (m *Memberlist) ingestPacket(buf []byte, from net.Addr, timestamp time.Time) {
var (
packetLabel string
err error
)
buf, packetLabel, err = RemoveLabelHeaderFromPacket(buf)
if err != nil {
m.logger.Printf("[ERR] memberlist: %v %s", err, LogAddress(from))
return
}
if m.config.SkipInboundLabelCheck {
if packetLabel != "" {
m.logger.Printf("[ERR] memberlist: unexpected double packet label header: %s", LogAddress(from))
return
}
// Set this from config so that the auth data assertions work below.
packetLabel = m.config.Label
}
if m.config.Label != packetLabel {
m.logger.Printf("[ERR] memberlist: discarding packet with unacceptable label %q: %s", packetLabel, LogAddress(from))
return
}
// Check if encryption is enabled
if m.config.EncryptionEnabled() {
// Decrypt the payload
plain, err := decryptPayload(m.config.Keyring.GetKeys(), buf, nil)
authData := []byte(packetLabel)
plain, err := decryptPayload(m.config.Keyring.GetKeys(), buf, authData)
if err != nil {
if !m.config.GossipVerifyIncoming {
// Treat the message as plaintext
@@ -723,7 +783,7 @@ func (m *Memberlist) encodeAndSendMsg(a Address, msgType messageType, msg interf
// opportunistically create a compoundMsg and piggy back other broadcasts.
func (m *Memberlist) sendMsg(a Address, msg []byte) error {
// Check if we can piggy back any messages
bytesAvail := m.config.UDPBufferSize - len(msg) - compoundHeaderOverhead
bytesAvail := m.config.UDPBufferSize - len(msg) - compoundHeaderOverhead - labelOverhead(m.config.Label)
if m.config.EncryptionEnabled() && m.config.GossipVerifyOutgoing {
bytesAvail -= encryptOverhead(m.encryptionVersion())
}
@@ -795,9 +855,12 @@ func (m *Memberlist) rawSendMsgPacket(a Address, node *Node, msg []byte) error {
// Check if we have encryption enabled
if m.config.EncryptionEnabled() && m.config.GossipVerifyOutgoing {
// Encrypt the payload
var buf bytes.Buffer
primaryKey := m.config.Keyring.GetPrimaryKey()
err := encryptPayload(m.encryptionVersion(), primaryKey, msg, nil, &buf)
var (
primaryKey = m.config.Keyring.GetPrimaryKey()
packetLabel = []byte(m.config.Label)
buf bytes.Buffer
)
err := encryptPayload(m.encryptionVersion(), primaryKey, msg, packetLabel, &buf)
if err != nil {
m.logger.Printf("[ERR] memberlist: Encryption of message failed: %v", err)
return err
@@ -812,7 +875,7 @@ func (m *Memberlist) rawSendMsgPacket(a Address, node *Node, msg []byte) error {
// rawSendMsgStream is used to stream a message to another host without
// modification, other than applying compression and encryption if enabled.
func (m *Memberlist) rawSendMsgStream(conn net.Conn, sendBuf []byte) error {
func (m *Memberlist) rawSendMsgStream(conn net.Conn, sendBuf []byte, streamLabel string) error {
// Check if compression is enabled
if m.config.EnableCompression {
compBuf, err := compressPayload(sendBuf)
@@ -825,7 +888,7 @@ func (m *Memberlist) rawSendMsgStream(conn net.Conn, sendBuf []byte) error {
// Check if encryption is enabled
if m.config.EncryptionEnabled() && m.config.GossipVerifyOutgoing {
crypt, err := m.encryptLocalState(sendBuf)
crypt, err := m.encryptLocalState(sendBuf, streamLabel)
if err != nil {
m.logger.Printf("[ERROR] memberlist: Failed to encrypt local state: %v", err)
return err
@@ -871,7 +934,8 @@ func (m *Memberlist) sendUserMsg(a Address, sendBuf []byte) error {
if _, err := bufConn.Write(sendBuf); err != nil {
return err
}
return m.rawSendMsgStream(conn, bufConn.Bytes())
return m.rawSendMsgStream(conn, bufConn.Bytes(), m.config.Label)
}
// sendAndReceiveState is used to initiate a push/pull over a stream with a
@@ -891,12 +955,12 @@ func (m *Memberlist) sendAndReceiveState(a Address, join bool) ([]pushNodeState,
metrics.IncrCounter([]string{"memberlist", "tcp", "connect"}, 1)
// Send our state
if err := m.sendLocalState(conn, join); err != nil {
if err := m.sendLocalState(conn, join, m.config.Label); err != nil {
return nil, nil, err
}
conn.SetDeadline(time.Now().Add(m.config.TCPTimeout))
msgType, bufConn, dec, err := m.readStream(conn)
msgType, bufConn, dec, err := m.readStream(conn, m.config.Label)
if err != nil {
return nil, nil, err
}
@@ -921,7 +985,7 @@ func (m *Memberlist) sendAndReceiveState(a Address, join bool) ([]pushNodeState,
}
// sendLocalState is invoked to send our local state over a stream connection.
func (m *Memberlist) sendLocalState(conn net.Conn, join bool) error {
func (m *Memberlist) sendLocalState(conn net.Conn, join bool, streamLabel string) error {
// Setup a deadline
conn.SetDeadline(time.Now().Add(m.config.TCPTimeout))
@@ -978,11 +1042,11 @@ func (m *Memberlist) sendLocalState(conn net.Conn, join bool) error {
}
// Get the send buffer
return m.rawSendMsgStream(conn, bufConn.Bytes())
return m.rawSendMsgStream(conn, bufConn.Bytes(), streamLabel)
}
// encryptLocalState is used to help encrypt local state before sending
func (m *Memberlist) encryptLocalState(sendBuf []byte) ([]byte, error) {
func (m *Memberlist) encryptLocalState(sendBuf []byte, streamLabel string) ([]byte, error) {
var buf bytes.Buffer
// Write the encryptMsg byte
@@ -995,9 +1059,15 @@ func (m *Memberlist) encryptLocalState(sendBuf []byte) ([]byte, error) {
binary.BigEndian.PutUint32(sizeBuf, uint32(encLen))
buf.Write(sizeBuf)
// Authenticated Data is:
//
// [messageType; byte] [messageLength; uint32] [stream_label; optional]
//
dataBytes := appendBytes(buf.Bytes()[:5], []byte(streamLabel))
// Write the encrypted cipher text to the buffer
key := m.config.Keyring.GetPrimaryKey()
err := encryptPayload(encVsn, key, sendBuf, buf.Bytes()[:5], &buf)
err := encryptPayload(encVsn, key, sendBuf, dataBytes, &buf)
if err != nil {
return nil, err
}
@@ -1005,7 +1075,7 @@ func (m *Memberlist) encryptLocalState(sendBuf []byte) ([]byte, error) {
}
// decryptRemoteState is used to help decrypt the remote state
func (m *Memberlist) decryptRemoteState(bufConn io.Reader) ([]byte, error) {
func (m *Memberlist) decryptRemoteState(bufConn io.Reader, streamLabel string) ([]byte, error) {
// Read in enough to determine message length
cipherText := bytes.NewBuffer(nil)
cipherText.WriteByte(byte(encryptMsg))
@@ -1027,8 +1097,13 @@ func (m *Memberlist) decryptRemoteState(bufConn io.Reader) ([]byte, error) {
return nil, err
}
// Decrypt the cipherText
dataBytes := cipherText.Bytes()[:5]
// Decrypt the cipherText with some authenticated data
//
// Authenticated Data is:
//
// [messageType; byte] [messageLength; uint32] [label_data; optional]
//
dataBytes := appendBytes(cipherText.Bytes()[:5], []byte(streamLabel))
cipherBytes := cipherText.Bytes()[5:]
// Decrypt the payload
@@ -1036,15 +1111,18 @@ func (m *Memberlist) decryptRemoteState(bufConn io.Reader) ([]byte, error) {
return decryptPayload(keys, cipherBytes, dataBytes)
}
// readStream is used to read from a stream connection, decrypting and
// readStream is used to read messages from a stream connection, decrypting and
// decompressing the stream if necessary.
func (m *Memberlist) readStream(conn net.Conn) (messageType, io.Reader, *codec.Decoder, error) {
//
// The provided streamLabel if present will be authenticated during decryption
// of each message.
func (m *Memberlist) readStream(conn net.Conn, streamLabel string) (messageType, io.Reader, *codec.Decoder, error) {
// Created a buffered reader
var bufConn io.Reader = bufio.NewReader(conn)
// Read the message type
buf := [1]byte{0}
if _, err := bufConn.Read(buf[:]); err != nil {
if _, err := io.ReadFull(bufConn, buf[:]); err != nil {
return 0, nil, nil, err
}
msgType := messageType(buf[0])
@@ -1056,7 +1134,7 @@ func (m *Memberlist) readStream(conn net.Conn) (messageType, io.Reader, *codec.D
fmt.Errorf("Remote state is encrypted and encryption is not configured")
}
plain, err := m.decryptRemoteState(bufConn)
plain, err := m.decryptRemoteState(bufConn, streamLabel)
if err != nil {
return 0, nil, nil, err
}
@@ -1236,11 +1314,11 @@ func (m *Memberlist) sendPingAndWaitForAck(a Address, ping ping, deadline time.T
return false, err
}
if err = m.rawSendMsgStream(conn, out.Bytes()); err != nil {
if err = m.rawSendMsgStream(conn, out.Bytes(), m.config.Label); err != nil {
return false, err
}
msgType, _, dec, err := m.readStream(conn)
msgType, _, dec, err := m.readStream(conn, m.config.Label)
if err != nil {
return false, err
}

48
vendor/github.com/hashicorp/memberlist/peeked_conn.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,48 @@
// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Originally from: https://github.com/google/tcpproxy/blob/master/tcpproxy.go
// at f5c09fbedceb69e4b238dec52cdf9f2fe9a815e2
package memberlist
import "net"
// peekedConn is an incoming connection that has had some bytes read from it
// to determine how to route the connection. The Read method stitches
// the peeked bytes and unread bytes back together.
type peekedConn struct {
// Peeked are the bytes that have been read from Conn for the
// purposes of route matching, but have not yet been consumed
// by Read calls. It set to nil by Read when fully consumed.
Peeked []byte
// Conn is the underlying connection.
// It can be type asserted against *net.TCPConn or other types
// as needed. It should not be read from directly unless
// Peeked is nil.
net.Conn
}
func (c *peekedConn) Read(p []byte) (n int, err error) {
if len(c.Peeked) > 0 {
n = copy(p, c.Peeked)
c.Peeked = c.Peeked[n:]
if len(c.Peeked) == 0 {
c.Peeked = nil
}
return n, nil
}
return c.Conn.Read(p)
}

19
vendor/github.com/hashicorp/memberlist/security.go сгенерированный поставляемый
Просмотреть файл

@@ -199,3 +199,22 @@ func decryptPayload(keys [][]byte, msg []byte, data []byte) ([]byte, error) {
return nil, fmt.Errorf("No installed keys could decrypt the message")
}
func appendBytes(first []byte, second []byte) []byte {
hasFirst := len(first) > 0
hasSecond := len(second) > 0
switch {
case hasFirst && hasSecond:
out := make([]byte, 0, len(first)+len(second))
out = append(out, first...)
out = append(out, second...)
return out
case hasFirst:
return first
case hasSecond:
return second
default:
return nil
}
}

7
vendor/github.com/hashicorp/memberlist/state.go сгенерированный поставляемый
Просмотреть файл

@@ -274,6 +274,11 @@ func failedRemote(err error) bool {
case "dial", "read", "write":
return true
}
} else if strings.HasPrefix(t.Net, "udp") {
switch t.Op {
case "write":
return true
}
}
}
return false
@@ -587,7 +592,7 @@ func (m *Memberlist) gossip() {
m.nodeLock.RUnlock()
// Compute the bytes available
bytesAvail := m.config.UDPBufferSize - compoundHeaderOverhead
bytesAvail := m.config.UDPBufferSize - compoundHeaderOverhead - labelOverhead(m.config.Label)
if m.config.EncryptionEnabled() {
bytesAvail -= encryptOverhead(m.encryptionVersion())
}

47
vendor/github.com/hashicorp/memberlist/transport.go сгенерированный поставляемый
Просмотреть файл

@@ -111,3 +111,50 @@ func (t *shimNodeAwareTransport) WriteToAddress(b []byte, addr Address) (time.Ti
func (t *shimNodeAwareTransport) DialAddressTimeout(addr Address, timeout time.Duration) (net.Conn, error) {
return t.DialTimeout(addr.Addr, timeout)
}
type labelWrappedTransport struct {
label string
NodeAwareTransport
}
var _ NodeAwareTransport = (*labelWrappedTransport)(nil)
func (t *labelWrappedTransport) WriteToAddress(buf []byte, addr Address) (time.Time, error) {
var err error
buf, err = AddLabelHeaderToPacket(buf, t.label)
if err != nil {
return time.Time{}, fmt.Errorf("failed to add label header to packet: %w", err)
}
return t.NodeAwareTransport.WriteToAddress(buf, addr)
}
func (t *labelWrappedTransport) WriteTo(buf []byte, addr string) (time.Time, error) {
var err error
buf, err = AddLabelHeaderToPacket(buf, t.label)
if err != nil {
return time.Time{}, err
}
return t.NodeAwareTransport.WriteTo(buf, addr)
}
func (t *labelWrappedTransport) DialAddressTimeout(addr Address, timeout time.Duration) (net.Conn, error) {
conn, err := t.NodeAwareTransport.DialAddressTimeout(addr, timeout)
if err != nil {
return nil, err
}
if err := AddLabelHeaderToStream(conn, t.label); err != nil {
return nil, fmt.Errorf("failed to add label header to stream: %w", err)
}
return conn, nil
}
func (t *labelWrappedTransport) DialTimeout(addr string, timeout time.Duration) (net.Conn, error) {
conn, err := t.NodeAwareTransport.DialTimeout(addr, timeout)
if err != nil {
return nil, err
}
if err := AddLabelHeaderToStream(conn, t.label); err != nil {
return nil, fmt.Errorf("failed to add label header to stream: %w", err)
}
return conn, nil
}

8
vendor/github.com/hashicorp/yamux/session.go сгенерированный поставляемый
Просмотреть файл

@@ -80,7 +80,7 @@ type Session struct {
// or to directly send a header
type sendReady struct {
Hdr []byte
Body io.Reader
Body []byte
Err chan error
}
@@ -352,7 +352,7 @@ func (s *Session) keepalive() {
}
// waitForSendErr waits to send a header, checking for a potential shutdown
func (s *Session) waitForSend(hdr header, body io.Reader) error {
func (s *Session) waitForSend(hdr header, body []byte) error {
errCh := make(chan error, 1)
return s.waitForSendErr(hdr, body, errCh)
}
@@ -360,7 +360,7 @@ func (s *Session) waitForSend(hdr header, body io.Reader) error {
// waitForSendErr waits to send a header with optional data, checking for a
// potential shutdown. Since there's the expectation that sends can happen
// in a timely manner, we enforce the connection write timeout here.
func (s *Session) waitForSendErr(hdr header, body io.Reader, errCh chan error) error {
func (s *Session) waitForSendErr(hdr header, body []byte, errCh chan error) error {
t := timerPool.Get()
timer := t.(*time.Timer)
timer.Reset(s.config.ConnectionWriteTimeout)
@@ -440,7 +440,7 @@ func (s *Session) send() {
// Send data from a body if given
if ready.Body != nil {
_, err := io.Copy(s.conn, ready.Body)
_, err := s.conn.Write(ready.Body)
if err != nil {
s.logger.Printf("[ERR] yamux: Failed to write body: %v", err)
asyncSendErr(ready.Err, err)

4
vendor/github.com/hashicorp/yamux/stream.go сгенерированный поставляемый
Просмотреть файл

@@ -169,7 +169,7 @@ func (s *Stream) Write(b []byte) (n int, err error) {
func (s *Stream) write(b []byte) (n int, err error) {
var flags uint16
var max uint32
var body io.Reader
var body []byte
START:
s.stateLock.Lock()
switch s.state {
@@ -195,7 +195,7 @@ START:
// Send up to our send window
max = min(window, uint32(len(b)))
body = bytes.NewReader(b[:max])
body = b[:max]
// Send the header
s.sendHdr.encode(typeData, flags, s.id, max)

11
vendor/github.com/jaytaylor/html2text/README.md сгенерированный поставляемый
Просмотреть файл

@@ -6,7 +6,6 @@
### Converts HTML into text of the markdown-flavored variety
## Introduction
Ensure your emails are readable by all!
@@ -19,7 +18,6 @@ There are still lots of improvements to be had, but FWIW this has worked fine fo
It requires go 1.x or newer ;)
## Download the package
```bash
@@ -28,6 +26,8 @@ go get jaytaylor.com/html2text
## Example usage
### Library
```go
package main
@@ -110,6 +110,11 @@ Here is some more information:
+-------------+-------------+
```
### Command line
```
echo '<div>hi</div>' | html2text
```
## Unit-tests
@@ -119,12 +124,10 @@ Running the unit-tests is straightforward and standard:
go test
```
# License
Permissive MIT license.
## Contact
You are more than welcome to open issues and send pull requests if you find a bug or want a new feature.

23
vendor/github.com/jaytaylor/html2text/html2text.go сгенерированный поставляемый
Просмотреть файл

@@ -18,6 +18,7 @@ type Options struct {
PrettyTables bool // Turns on pretty ASCII rendering for table elements.
PrettyTablesOptions *PrettyTablesOptions // Configures pretty ASCII rendering for table elements.
OmitLinks bool // Turns on omitting links
TextOnly bool // Returns only plain text
}
// PrettyTablesOptions overrides tablewriter behaviors
@@ -157,6 +158,9 @@ func (ctx *textifyTraverseContext) handleElement(node *html.Node) error {
}
str := subCtx.buf.String()
if ctx.options.TextOnly {
return ctx.emit(str + ".\n\n")
}
dividerLen := 0
for _, line := range strings.Split(str, "\n") {
if lineLen := len([]rune(line)); lineLen-1 > dividerLen {
@@ -177,7 +181,9 @@ func (ctx *textifyTraverseContext) handleElement(node *html.Node) error {
case atom.Blockquote:
ctx.blockquoteLevel++
ctx.prefix = strings.Repeat(">", ctx.blockquoteLevel) + " "
if !ctx.options.TextOnly {
ctx.prefix = strings.Repeat(">", ctx.blockquoteLevel) + " "
}
if err := ctx.emit("\n"); err != nil {
return err
}
@@ -190,7 +196,9 @@ func (ctx *textifyTraverseContext) handleElement(node *html.Node) error {
return err
}
ctx.blockquoteLevel--
ctx.prefix = strings.Repeat(">", ctx.blockquoteLevel)
if !ctx.options.TextOnly {
ctx.prefix = strings.Repeat(">", ctx.blockquoteLevel)
}
if ctx.blockquoteLevel > 0 {
ctx.prefix += " "
}
@@ -213,8 +221,10 @@ func (ctx *textifyTraverseContext) handleElement(node *html.Node) error {
return err
case atom.Li:
if err := ctx.emit("* "); err != nil {
return err
if !ctx.options.TextOnly {
if err := ctx.emit("* "); err != nil {
return err
}
}
if err := ctx.traverseChildren(node); err != nil {
@@ -230,6 +240,9 @@ func (ctx *textifyTraverseContext) handleElement(node *html.Node) error {
return err
}
str := subCtx.buf.String()
if ctx.options.TextOnly {
return ctx.emit(str + ".")
}
return ctx.emit("*" + str + "*")
case atom.A:
@@ -254,7 +267,7 @@ func (ctx *textifyTraverseContext) handleElement(node *html.Node) error {
if attrVal := getAttrVal(node, "href"); attrVal != "" {
attrVal = ctx.normalizeHrefLink(attrVal)
// Don't print link href if it matches link element content or if the link is empty.
if !ctx.options.OmitLinks && attrVal != "" && linkText != attrVal {
if (!ctx.options.OmitLinks && attrVal != "" && linkText != attrVal) || !ctx.options.TextOnly {
hrefLink = "( " + attrVal + " )"
}
}

15
vendor/github.com/klauspost/compress/s2/.gitignore сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,15 @@
testdata/bench
# These explicitly listed benchmark data files are for an obsolete version of
# snappy_test.go.
testdata/alice29.txt
testdata/asyoulik.txt
testdata/fireworks.jpeg
testdata/geo.protodata
testdata/html
testdata/html_x_4
testdata/kppkn.gtb
testdata/lcet10.txt
testdata/paper-100k.pdf
testdata/plrabn12.txt
testdata/urls.10K

28
vendor/github.com/klauspost/compress/s2/LICENSE сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,28 @@
Copyright (c) 2011 The Snappy-Go Authors. All rights reserved.
Copyright (c) 2019 Klaus Post. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

717
vendor/github.com/klauspost/compress/s2/README.md сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,717 @@
# S2 Compression
S2 is an extension of [Snappy](https://github.com/google/snappy).
S2 is aimed for high throughput, which is why it features concurrent compression for bigger payloads.
Decoding is compatible with Snappy compressed content, but content compressed with S2 cannot be decompressed by Snappy.
This means that S2 can seamlessly replace Snappy without converting compressed content.
S2 can produce Snappy compatible output, faster and better than Snappy.
If you want full benefit of the changes you should use s2 without Snappy compatibility.
S2 is designed to have high throughput on content that cannot be compressed.
This is important, so you don't have to worry about spending CPU cycles on already compressed data.
## Benefits over Snappy
* Better compression
* Adjustable compression (3 levels)
* Concurrent stream compression
* Faster decompression, even for Snappy compatible content
* Ability to quickly skip forward in compressed stream
* Compatible with reading Snappy compressed content
* Smaller block size overhead on incompressible blocks
* Block concatenation
* Uncompressed stream mode
* Automatic stream size padding
* Snappy compatible block compression
## Drawbacks over Snappy
* Not optimized for 32 bit systems.
* Streams use slightly more memory due to larger blocks and concurrency (configurable).
# Usage
Installation: `go get -u github.com/klauspost/compress/s2`
Full package documentation:
[![godoc][1]][2]
[1]: https://godoc.org/github.com/klauspost/compress?status.svg
[2]: https://godoc.org/github.com/klauspost/compress/s2
## Compression
```Go
func EncodeStream(src io.Reader, dst io.Writer) error {
enc := s2.NewWriter(dst)
_, err := io.Copy(enc, src)
if err != nil {
enc.Close()
return err
}
// Blocks until compression is done.
return enc.Close()
}
```
You should always call `enc.Close()`, otherwise you will leak resources and your encode will be incomplete.
For the best throughput, you should attempt to reuse the `Writer` using the `Reset()` method.
The Writer in S2 is always buffered, therefore `NewBufferedWriter` in Snappy can be replaced with `NewWriter` in S2.
It is possible to flush any buffered data using the `Flush()` method.
This will block until all data sent to the encoder has been written to the output.
S2 also supports the `io.ReaderFrom` interface, which will consume all input from a reader.
As a final method to compress data, if you have a single block of data you would like to have encoded as a stream,
a slightly more efficient method is to use the `EncodeBuffer` method.
This will take ownership of the buffer until the stream is closed.
```Go
func EncodeStream(src []byte, dst io.Writer) error {
enc := s2.NewWriter(dst)
// The encoder owns the buffer until Flush or Close is called.
err := enc.EncodeBuffer(buf)
if err != nil {
enc.Close()
return err
}
// Blocks until compression is done.
return enc.Close()
}
```
Each call to `EncodeBuffer` will result in discrete blocks being created without buffering,
so it should only be used a single time per stream.
If you need to write several blocks, you should use the regular io.Writer interface.
## Decompression
```Go
func DecodeStream(src io.Reader, dst io.Writer) error {
dec := s2.NewReader(src)
_, err := io.Copy(dst, dec)
return err
}
```
Similar to the Writer, a Reader can be reused using the `Reset` method.
For the best possible throughput, there is a `EncodeBuffer(buf []byte)` function available.
However, it requires that the provided buffer isn't used after it is handed over to S2 and until the stream is flushed or closed.
For smaller data blocks, there is also a non-streaming interface: `Encode()`, `EncodeBetter()` and `Decode()`.
Do however note that these functions (similar to Snappy) does not provide validation of data,
so data corruption may be undetected. Stream encoding provides CRC checks of data.
It is possible to efficiently skip forward in a compressed stream using the `Skip()` method.
For big skips the decompressor is able to skip blocks without decompressing them.
## Single Blocks
Similar to Snappy S2 offers single block compression.
Blocks do not offer the same flexibility and safety as streams,
but may be preferable for very small payloads, less than 100K.
Using a simple `dst := s2.Encode(nil, src)` will compress `src` and return the compressed result.
It is possible to provide a destination buffer.
If the buffer has a capacity of `s2.MaxEncodedLen(len(src))` it will be used.
If not a new will be allocated.
Alternatively `EncodeBetter`/`EncodeBest` can also be used for better, but slightly slower compression.
Similarly to decompress a block you can use `dst, err := s2.Decode(nil, src)`.
Again an optional destination buffer can be supplied.
The `s2.DecodedLen(src)` can be used to get the minimum capacity needed.
If that is not satisfied a new buffer will be allocated.
Block function always operate on a single goroutine since it should only be used for small payloads.
# Commandline tools
Some very simply commandline tools are provided; `s2c` for compression and `s2d` for decompression.
Binaries can be downloaded on the [Releases Page](https://github.com/klauspost/compress/releases).
Installing then requires Go to be installed. To install them, use:
`go install github.com/klauspost/compress/s2/cmd/s2c && go install github.com/klauspost/compress/s2/cmd/s2d`
To build binaries to the current folder use:
`go build github.com/klauspost/compress/s2/cmd/s2c && go build github.com/klauspost/compress/s2/cmd/s2d`
## s2c
```
Usage: s2c [options] file1 file2
Compresses all files supplied as input separately.
Output files are written as 'filename.ext.s2' or 'filename.ext.snappy'.
By default output files will be overwritten.
Use - as the only file name to read from stdin and write to stdout.
Wildcards are accepted: testdir/*.txt will compress all files in testdir ending with .txt
Directories can be wildcards as well. testdir/*/*.txt will match testdir/subdir/b.txt
File names beginning with 'http://' and 'https://' will be downloaded and compressed.
Only http response code 200 is accepted.
Options:
-bench int
Run benchmark n times. No output will be written
-blocksize string
Max block size. Examples: 64K, 256K, 1M, 4M. Must be power of two and <= 4MB (default "4M")
-c Write all output to stdout. Multiple input files will be concatenated
-cpu int
Compress using this amount of threads (default 32)
-faster
Compress faster, but with a minor compression loss
-help
Display help
-o string
Write output to another file. Single input file only
-pad string
Pad size to a multiple of this value, Examples: 500, 64K, 256K, 1M, 4M, etc (default "1")
-q Don't write any output to terminal, except errors
-rm
Delete source file(s) after successful compression
-safe
Do not overwrite output files
-slower
Compress more, but a lot slower
-snappy
Generate Snappy compatible output stream
-verify
Verify written files
```
## s2d
```
Usage: s2d [options] file1 file2
Decompresses all files supplied as input. Input files must end with '.s2' or '.snappy'.
Output file names have the extension removed. By default output files will be overwritten.
Use - as the only file name to read from stdin and write to stdout.
Wildcards are accepted: testdir/*.txt will compress all files in testdir ending with .txt
Directories can be wildcards as well. testdir/*/*.txt will match testdir/subdir/b.txt
File names beginning with 'http://' and 'https://' will be downloaded and decompressed.
Extensions on downloaded files are ignored. Only http response code 200 is accepted.
Options:
-bench int
Run benchmark n times. No output will be written
-c Write all output to stdout. Multiple input files will be concatenated
-help
Display help
-o string
Write output to another file. Single input file only
-q Don't write any output to terminal, except errors
-rm
Delete source file(s) after successful decompression
-safe
Do not overwrite output files
-verify
Verify files, but do not write output
```
## s2sx: self-extracting archives
s2sx allows creating self-extracting archives with no dependencies.
By default, executables are created for the same platforms as the host os,
but this can be overridden with `-os` and `-arch` parameters.
Extracted files have 0666 permissions, except when untar option used.
```
Usage: s2sx [options] file1 file2
Compresses all files supplied as input separately.
If files have '.s2' extension they are assumed to be compressed already.
Output files are written as 'filename.s2sx' and with '.exe' for windows targets.
If output is big, an additional file with ".more" is written. This must be included as well.
By default output files will be overwritten.
Wildcards are accepted: testdir/*.txt will compress all files in testdir ending with .txt
Directories can be wildcards as well. testdir/*/*.txt will match testdir/subdir/b.txt
Options:
-arch string
Destination architecture (default "amd64")
-c Write all output to stdout. Multiple input files will be concatenated
-cpu int
Compress using this amount of threads (default 32)
-help
Display help
-max string
Maximum executable size. Rest will be written to another file. (default "1G")
-os string
Destination operating system (default "windows")
-q Don't write any output to terminal, except errors
-rm
Delete source file(s) after successful compression
-safe
Do not overwrite output files
-untar
Untar on destination
```
Available platforms are:
* darwin-amd64
* darwin-arm64
* linux-amd64
* linux-arm
* linux-arm64
* linux-mips64
* linux-ppc64le
* windows-386
* windows-amd64
By default, there is a size limit of 1GB for the output executable.
When this is exceeded the remaining file content is written to a file called
output+`.more`. This file must be included for a successful extraction and
placed alongside the executable for a successful extraction.
This file *must* have the same name as the executable, so if the executable is renamed,
so must the `.more` file.
This functionality is disabled with stdin/stdout.
### Self-extracting TAR files
If you wrap a TAR file you can specify `-untar` to make it untar on the destination host.
Files are extracted to the current folder with the path specified in the tar file.
Note that tar files are not validated before they are wrapped.
For security reasons files that move below the root folder are not allowed.
# Performance
This section will focus on comparisons to Snappy.
This package is solely aimed at replacing Snappy as a high speed compression package.
If you are mainly looking for better compression [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd)
gives better compression, but typically at speeds slightly below "better" mode in this package.
Compression is increased compared to Snappy, mostly around 5-20% and the throughput is typically 25-40% increased (single threaded) compared to the Snappy Go implementation.
Streams are concurrently compressed. The stream will be distributed among all available CPU cores for the best possible throughput.
A "better" compression mode is also available. This allows to trade a bit of speed for a minor compression gain.
The content compressed in this mode is fully compatible with the standard decoder.
Snappy vs S2 **compression** speed on 16 core (32 thread) computer, using all threads and a single thread (1 CPU):
| File | S2 speed | S2 Throughput | S2 % smaller | S2 "better" | "better" throughput | "better" % smaller |
|-----------------------------------------------------------------------------------------------------|----------|---------------|--------------|-------------|---------------------|--------------------|
| [rawstudio-mint14.tar](https://files.klauspost.com/compress/rawstudio-mint14.7z) | 12.70x | 10556 MB/s | 7.35% | 4.15x | 3455 MB/s | 12.79% |
| (1 CPU) | 1.14x | 948 MB/s | - | 0.42x | 349 MB/s | - |
| [github-june-2days-2019.json](https://files.klauspost.com/compress/github-june-2days-2019.json.zst) | 17.13x | 14484 MB/s | 31.60% | 10.09x | 8533 MB/s | 37.71% |
| (1 CPU) | 1.33x | 1127 MB/s | - | 0.70x | 589 MB/s | - |
| [github-ranks-backup.bin](https://files.klauspost.com/compress/github-ranks-backup.bin.zst) | 15.14x | 12000 MB/s | -5.79% | 6.59x | 5223 MB/s | 5.80% |
| (1 CPU) | 1.11x | 877 MB/s | - | 0.47x | 370 MB/s | - |
| [consensus.db.10gb](https://files.klauspost.com/compress/consensus.db.10gb.zst) | 14.62x | 12116 MB/s | 15.90% | 5.35x | 4430 MB/s | 16.08% |
| (1 CPU) | 1.38x | 1146 MB/s | - | 0.38x | 312 MB/s | - |
| [adresser.json](https://files.klauspost.com/compress/adresser.json.zst) | 8.83x | 17579 MB/s | 43.86% | 6.54x | 13011 MB/s | 47.23% |
| (1 CPU) | 1.14x | 2259 MB/s | - | 0.74x | 1475 MB/s | - |
| [gob-stream](https://files.klauspost.com/compress/gob-stream.7z) | 16.72x | 14019 MB/s | 24.02% | 10.11x | 8477 MB/s | 30.48% |
| (1 CPU) | 1.24x | 1043 MB/s | - | 0.70x | 586 MB/s | - |
| [10gb.tar](http://mattmahoney.net/dc/10gb.html) | 13.33x | 9254 MB/s | 1.84% | 6.75x | 4686 MB/s | 6.72% |
| (1 CPU) | 0.97x | 672 MB/s | - | 0.53x | 366 MB/s | - |
| sharnd.out.2gb | 2.11x | 12639 MB/s | 0.01% | 1.98x | 11833 MB/s | 0.01% |
| (1 CPU) | 0.93x | 5594 MB/s | - | 1.34x | 8030 MB/s | - |
| [enwik9](http://mattmahoney.net/dc/textdata.html) | 19.34x | 8220 MB/s | 3.98% | 7.87x | 3345 MB/s | 15.82% |
| (1 CPU) | 1.06x | 452 MB/s | - | 0.50x | 213 MB/s | - |
| [silesia.tar](http://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip) | 10.48x | 6124 MB/s | 5.67% | 3.76x | 2197 MB/s | 12.60% |
| (1 CPU) | 0.97x | 568 MB/s | - | 0.46x | 271 MB/s | - |
| [enwik10](https://encode.su/threads/3315-enwik10-benchmark-results) | 21.07x | 9020 MB/s | 6.36% | 6.91x | 2959 MB/s | 16.95% |
| (1 CPU) | 1.07x | 460 MB/s | - | 0.51x | 220 MB/s | - |
### Legend
* `S2 speed`: Speed of S2 compared to Snappy, using 16 cores and 1 core.
* `S2 throughput`: Throughput of S2 in MB/s.
* `S2 % smaller`: How many percent of the Snappy output size is S2 better.
* `S2 "better"`: Speed when enabling "better" compression mode in S2 compared to Snappy.
* `"better" throughput`: Speed when enabling "better" compression mode in S2 compared to Snappy.
* `"better" % smaller`: How many percent of the Snappy output size is S2 better when using "better" compression.
There is a good speedup across the board when using a single thread and a significant speedup when using multiple threads.
Machine generated data gets by far the biggest compression boost, with size being being reduced by up to 45% of Snappy size.
The "better" compression mode sees a good improvement in all cases, but usually at a performance cost.
Incompressible content (`sharnd.out.2gb`, 2GB random data) sees the smallest speedup.
This is likely dominated by synchronization overhead, which is confirmed by the fact that single threaded performance is higher (see above).
## Decompression
S2 attempts to create content that is also fast to decompress, except in "better" mode where the smallest representation is used.
S2 vs Snappy **decompression** speed. Both operating on single core:
| File | S2 Throughput | vs. Snappy | Better Throughput | vs. Snappy |
|-----------------------------------------------------------------------------------------------------|---------------|------------|-------------------|------------|
| [rawstudio-mint14.tar](https://files.klauspost.com/compress/rawstudio-mint14.7z) | 2117 MB/s | 1.14x | 1738 MB/s | 0.94x |
| [github-june-2days-2019.json](https://files.klauspost.com/compress/github-june-2days-2019.json.zst) | 2401 MB/s | 1.25x | 2307 MB/s | 1.20x |
| [github-ranks-backup.bin](https://files.klauspost.com/compress/github-ranks-backup.bin.zst) | 2075 MB/s | 0.98x | 1764 MB/s | 0.83x |
| [consensus.db.10gb](https://files.klauspost.com/compress/consensus.db.10gb.zst) | 2967 MB/s | 1.05x | 2885 MB/s | 1.02x |
| [adresser.json](https://files.klauspost.com/compress/adresser.json.zst) | 4141 MB/s | 1.07x | 4184 MB/s | 1.08x |
| [gob-stream](https://files.klauspost.com/compress/gob-stream.7z) | 2264 MB/s | 1.12x | 2185 MB/s | 1.08x |
| [10gb.tar](http://mattmahoney.net/dc/10gb.html) | 1525 MB/s | 1.03x | 1347 MB/s | 0.91x |
| sharnd.out.2gb | 3813 MB/s | 0.79x | 3900 MB/s | 0.81x |
| [enwik9](http://mattmahoney.net/dc/textdata.html) | 1246 MB/s | 1.29x | 967 MB/s | 1.00x |
| [silesia.tar](http://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip) | 1433 MB/s | 1.12x | 1203 MB/s | 0.94x |
| [enwik10](https://encode.su/threads/3315-enwik10-benchmark-results) | 1284 MB/s | 1.32x | 1010 MB/s | 1.04x |
### Legend
* `S2 Throughput`: Decompression speed of S2 encoded content.
* `Better Throughput`: Decompression speed of S2 "better" encoded content.
* `vs Snappy`: Decompression speed of S2 "better" mode compared to Snappy and absolute speed.
While the decompression code hasn't changed, there is a significant speedup in decompression speed.
S2 prefers longer matches and will typically only find matches that are 6 bytes or longer.
While this reduces compression a bit, it improves decompression speed.
The "better" compression mode will actively look for shorter matches, which is why it has a decompression speed quite similar to Snappy.
Without assembly decompression is also very fast; single goroutine decompression speed. No assembly:
| File | S2 Throughput | S2 throughput |
|--------------------------------|--------------|---------------|
| consensus.db.10gb.s2 | 1.84x | 2289.8 MB/s |
| 10gb.tar.s2 | 1.30x | 867.07 MB/s |
| rawstudio-mint14.tar.s2 | 1.66x | 1329.65 MB/s |
| github-june-2days-2019.json.s2 | 2.36x | 1831.59 MB/s |
| github-ranks-backup.bin.s2 | 1.73x | 1390.7 MB/s |
| enwik9.s2 | 1.67x | 681.53 MB/s |
| adresser.json.s2 | 3.41x | 4230.53 MB/s |
| silesia.tar.s2 | 1.52x | 811.58 |
Even though S2 typically compresses better than Snappy, decompression speed is always better.
## Block compression
When compressing blocks no concurrent compression is performed just as Snappy.
This is because blocks are for smaller payloads and generally will not benefit from concurrent compression.
An important change is that incompressible blocks will not be more than at most 10 bytes bigger than the input.
In rare, worst case scenario Snappy blocks could be significantly bigger than the input.
### Mixed content blocks
The most reliable is a wide dataset.
For this we use [`webdevdata.org-2015-01-07-subset`](https://files.klauspost.com/compress/webdevdata.org-2015-01-07-4GB-subset.7z),
53927 files, total input size: 4,014,735,833 bytes. Single goroutine used.
| * | Input | Output | Reduction | MB/s |
|-------------------|------------|------------|-----------|--------|
| S2 | 4014735833 | 1059723369 | 73.60% | **934.34** |
| S2 Better | 4014735833 | 969670507 | 75.85% | 532.70 |
| S2 Best | 4014735833 | 906625668 | **77.85%** | 46.84 |
| Snappy | 4014735833 | 1128706759 | 71.89% | 762.59 |
| S2, Snappy Output | 4014735833 | 1093821420 | 72.75% | 908.60 |
| LZ4 | 4014735833 | 1079259294 | 73.12% | 526.94 |
S2 delivers both the best single threaded throughput with regular mode and the best compression rate with "best".
"Better" mode provides the same compression speed as LZ4 with better compression ratio.
When outputting Snappy compatible output it still delivers better throughput (150MB/s more) and better compression.
As can be seen from the other benchmarks decompression should also be easier on the S2 generated output.
Though they cannot be compared due to different decompression speeds here are the speed/size comparisons for
other Go compressors:
| * | Input | Output | Reduction | MB/s |
|-------------------|------------|------------|-----------|--------|
| Zstd Fastest (Go) | 4014735833 | 794608518 | 80.21% | 236.04 |
| Zstd Best (Go) | 4014735833 | 704603356 | 82.45% | 35.63 |
| Deflate (Go) l1 | 4014735833 | 871294239 | 78.30% | 214.04 |
| Deflate (Go) l9 | 4014735833 | 730389060 | 81.81% | 41.17 |
### Standard block compression
Benchmarking single block performance is subject to a lot more variation since it only tests a limited number of file patterns.
So individual benchmarks should only be seen as a guideline and the overall picture is more important.
These micro-benchmarks are with data in cache and trained branch predictors. For a more realistic benchmark see the mixed content above.
Block compression. Parallel benchmark running on 16 cores, 16 goroutines.
AMD64 assembly is use for both S2 and Snappy.
| Absolute Perf | Snappy size | S2 Size | Snappy Speed | S2 Speed | Snappy dec | S2 dec |
|-----------------------|-------------|---------|--------------|-------------|-------------|-------------|
| html | 22843 | 21111 | 16246 MB/s | 17438 MB/s | 40972 MB/s | 49263 MB/s |
| urls.10K | 335492 | 287326 | 7943 MB/s | 9693 MB/s | 22523 MB/s | 26484 MB/s |
| fireworks.jpeg | 123034 | 123100 | 349544 MB/s | 273889 MB/s | 718321 MB/s | 827552 MB/s |
| fireworks.jpeg (200B) | 146 | 155 | 8869 MB/s | 17773 MB/s | 33691 MB/s | 52421 MB/s |
| paper-100k.pdf | 85304 | 84459 | 167546 MB/s | 101263 MB/s | 326905 MB/s | 291944 MB/s |
| html_x_4 | 92234 | 21113 | 15194 MB/s | 50670 MB/s | 30843 MB/s | 32217 MB/s |
| alice29.txt | 88034 | 85975 | 5936 MB/s | 6139 MB/s | 12882 MB/s | 20044 MB/s |
| asyoulik.txt | 77503 | 79650 | 5517 MB/s | 6366 MB/s | 12735 MB/s | 22806 MB/s |
| lcet10.txt | 234661 | 220670 | 6235 MB/s | 6067 MB/s | 14519 MB/s | 18697 MB/s |
| plrabn12.txt | 319267 | 317985 | 5159 MB/s | 5726 MB/s | 11923 MB/s | 19901 MB/s |
| geo.protodata | 23335 | 18690 | 21220 MB/s | 26529 MB/s | 56271 MB/s | 62540 MB/s |
| kppkn.gtb | 69526 | 65312 | 9732 MB/s | 8559 MB/s | 18491 MB/s | 18969 MB/s |
| alice29.txt (128B) | 80 | 82 | 6691 MB/s | 15489 MB/s | 31883 MB/s | 38874 MB/s |
| alice29.txt (1000B) | 774 | 774 | 12204 MB/s | 13000 MB/s | 48056 MB/s | 52341 MB/s |
| alice29.txt (10000B) | 6648 | 6933 | 10044 MB/s | 12806 MB/s | 32378 MB/s | 46322 MB/s |
| alice29.txt (20000B) | 12686 | 13574 | 7733 MB/s | 11210 MB/s | 30566 MB/s | 58969 MB/s |
| Relative Perf | Snappy size | S2 size improved | S2 Speed | S2 Dec Speed |
|-----------------------|-------------|------------------|----------|--------------|
| html | 22.31% | 7.58% | 1.07x | 1.20x |
| urls.10K | 47.78% | 14.36% | 1.22x | 1.18x |
| fireworks.jpeg | 99.95% | -0.05% | 0.78x | 1.15x |
| fireworks.jpeg (200B) | 73.00% | -6.16% | 2.00x | 1.56x |
| paper-100k.pdf | 83.30% | 0.99% | 0.60x | 0.89x |
| html_x_4 | 22.52% | 77.11% | 3.33x | 1.04x |
| alice29.txt | 57.88% | 2.34% | 1.03x | 1.56x |
| asyoulik.txt | 61.91% | -2.77% | 1.15x | 1.79x |
| lcet10.txt | 54.99% | 5.96% | 0.97x | 1.29x |
| plrabn12.txt | 66.26% | 0.40% | 1.11x | 1.67x |
| geo.protodata | 19.68% | 19.91% | 1.25x | 1.11x |
| kppkn.gtb | 37.72% | 6.06% | 0.88x | 1.03x |
| alice29.txt (128B) | 62.50% | -2.50% | 2.31x | 1.22x |
| alice29.txt (1000B) | 77.40% | 0.00% | 1.07x | 1.09x |
| alice29.txt (10000B) | 66.48% | -4.29% | 1.27x | 1.43x |
| alice29.txt (20000B) | 63.43% | -7.00% | 1.45x | 1.93x |
Speed is generally at or above Snappy. Small blocks gets a significant speedup, although at the expense of size.
Decompression speed is better than Snappy, except in one case.
Since payloads are very small the variance in terms of size is rather big, so they should only be seen as a general guideline.
Size is on average around Snappy, but varies on content type.
In cases where compression is worse, it usually is compensated by a speed boost.
### Better compression
Benchmarking single block performance is subject to a lot more variation since it only tests a limited number of file patterns.
So individual benchmarks should only be seen as a guideline and the overall picture is more important.
| Absolute Perf | Snappy size | Better Size | Snappy Speed | Better Speed | Snappy dec | Better dec |
|-----------------------|-------------|-------------|--------------|--------------|-------------|-------------|
| html | 22843 | 19833 | 16246 MB/s | 7731 MB/s | 40972 MB/s | 40292 MB/s |
| urls.10K | 335492 | 253529 | 7943 MB/s | 3980 MB/s | 22523 MB/s | 20981 MB/s |
| fireworks.jpeg | 123034 | 123100 | 349544 MB/s | 9760 MB/s | 718321 MB/s | 823698 MB/s |
| fireworks.jpeg (200B) | 146 | 142 | 8869 MB/s | 594 MB/s | 33691 MB/s | 30101 MB/s |
| paper-100k.pdf | 85304 | 82915 | 167546 MB/s | 7470 MB/s | 326905 MB/s | 198869 MB/s |
| html_x_4 | 92234 | 19841 | 15194 MB/s | 23403 MB/s | 30843 MB/s | 30937 MB/s |
| alice29.txt | 88034 | 73218 | 5936 MB/s | 2945 MB/s | 12882 MB/s | 16611 MB/s |
| asyoulik.txt | 77503 | 66844 | 5517 MB/s | 2739 MB/s | 12735 MB/s | 14975 MB/s |
| lcet10.txt | 234661 | 190589 | 6235 MB/s | 3099 MB/s | 14519 MB/s | 16634 MB/s |
| plrabn12.txt | 319267 | 270828 | 5159 MB/s | 2600 MB/s | 11923 MB/s | 13382 MB/s |
| geo.protodata | 23335 | 18278 | 21220 MB/s | 11208 MB/s | 56271 MB/s | 57961 MB/s |
| kppkn.gtb | 69526 | 61851 | 9732 MB/s | 4556 MB/s | 18491 MB/s | 16524 MB/s |
| alice29.txt (128B) | 80 | 81 | 6691 MB/s | 529 MB/s | 31883 MB/s | 34225 MB/s |
| alice29.txt (1000B) | 774 | 748 | 12204 MB/s | 1943 MB/s | 48056 MB/s | 42068 MB/s |
| alice29.txt (10000B) | 6648 | 6234 | 10044 MB/s | 2949 MB/s | 32378 MB/s | 28813 MB/s |
| alice29.txt (20000B) | 12686 | 11584 | 7733 MB/s | 2822 MB/s | 30566 MB/s | 27315 MB/s |
| Relative Perf | Snappy size | Better size | Better Speed | Better dec |
|-----------------------|-------------|-------------|--------------|------------|
| html | 22.31% | 13.18% | 0.48x | 0.98x |
| urls.10K | 47.78% | 24.43% | 0.50x | 0.93x |
| fireworks.jpeg | 99.95% | -0.05% | 0.03x | 1.15x |
| fireworks.jpeg (200B) | 73.00% | 2.74% | 0.07x | 0.89x |
| paper-100k.pdf | 83.30% | 2.80% | 0.07x | 0.61x |
| html_x_4 | 22.52% | 78.49% | 0.04x | 1.00x |
| alice29.txt | 57.88% | 16.83% | 1.54x | 1.29x |
| asyoulik.txt | 61.91% | 13.75% | 0.50x | 1.18x |
| lcet10.txt | 54.99% | 18.78% | 0.50x | 1.15x |
| plrabn12.txt | 66.26% | 15.17% | 0.50x | 1.12x |
| geo.protodata | 19.68% | 21.67% | 0.50x | 1.03x |
| kppkn.gtb | 37.72% | 11.04% | 0.53x | 0.89x |
| alice29.txt (128B) | 62.50% | -1.25% | 0.47x | 1.07x |
| alice29.txt (1000B) | 77.40% | 3.36% | 0.08x | 0.88x |
| alice29.txt (10000B) | 66.48% | 6.23% | 0.16x | 0.89x |
| alice29.txt (20000B) | 63.43% | 8.69% | 0.29x | 0.89x |
Except for the mostly incompressible JPEG image compression is better and usually in the
double digits in terms of percentage reduction over Snappy.
The PDF sample shows a significant slowdown compared to Snappy, as this mode tries harder
to compress the data. Very small blocks are also not favorable for better compression, so throughput is way down.
This mode aims to provide better compression at the expense of performance and achieves that
without a huge performance penalty, except on very small blocks.
Decompression speed suffers a little compared to the regular S2 mode,
but still manages to be close to Snappy in spite of increased compression.
# Best compression mode
S2 offers a "best" compression mode.
This will compress as much as possible with little regard to CPU usage.
Mainly for offline compression, but where decompression speed should still
be high and compatible with other S2 compressed data.
Some examples compared on 16 core CPU, amd64 assembly used:
```
* enwik10
Default... 10000000000 -> 4761467548 [47.61%]; 1.098s, 8685.6MB/s
Better... 10000000000 -> 4219438251 [42.19%]; 1.925s, 4954.2MB/s
Best... 10000000000 -> 3627364337 [36.27%]; 43.051s, 221.5MB/s
* github-june-2days-2019.json
Default... 6273951764 -> 1043196283 [16.63%]; 431ms, 13882.3MB/s
Better... 6273951764 -> 949146808 [15.13%]; 547ms, 10938.4MB/s
Best... 6273951764 -> 832855506 [13.27%]; 9.455s, 632.8MB/s
* nyc-taxi-data-10M.csv
Default... 3325605752 -> 1095998837 [32.96%]; 324ms, 9788.7MB/s
Better... 3325605752 -> 954776589 [28.71%]; 491ms, 6459.4MB/s
Best... 3325605752 -> 779098746 [23.43%]; 8.29s, 382.6MB/s
* 10gb.tar
Default... 10065157632 -> 5916578242 [58.78%]; 1.028s, 9337.4MB/s
Better... 10065157632 -> 5649207485 [56.13%]; 1.597s, 6010.6MB/s
Best... 10065157632 -> 5208719802 [51.75%]; 32.78s, 292.8MB/
* consensus.db.10gb
Default... 10737418240 -> 4562648848 [42.49%]; 882ms, 11610.0MB/s
Better... 10737418240 -> 4542428129 [42.30%]; 1.533s, 6679.7MB/s
Best... 10737418240 -> 4244773384 [39.53%]; 42.96s, 238.4MB/s
```
Decompression speed should be around the same as using the 'better' compression mode.
# Snappy Compatibility
S2 now offers full compatibility with Snappy.
This means that the efficient encoders of S2 can be used to generate fully Snappy compatible output.
There is a [snappy](https://github.com/klauspost/compress/tree/master/snappy) package that can be used by
simply changing imports from `github.com/golang/snappy` to `github.com/klauspost/compress/snappy`.
This uses "better" mode for all operations.
If you would like more control, you can use the s2 package as described below:
## Blocks
Snappy compatible blocks can be generated with the S2 encoder.
Compression and speed is typically a bit better `MaxEncodedLen` is also smaller for smaller memory usage. Replace
| Snappy | S2 replacement |
|----------------------------|-------------------------|
| snappy.Encode(...) | s2.EncodeSnappy(...) |
| snappy.MaxEncodedLen(...) | s2.MaxEncodedLen(...) |
`s2.EncodeSnappy` can be replaced with `s2.EncodeSnappyBetter` or `s2.EncodeSnappyBest` to get more efficiently compressed snappy compatible output.
`s2.ConcatBlocks` is compatible with snappy blocks.
Comparison of [`webdevdata.org-2015-01-07-subset`](https://files.klauspost.com/compress/webdevdata.org-2015-01-07-4GB-subset.7z),
53927 files, total input size: 4,014,735,833 bytes. amd64, single goroutine used:
| Encoder | Size | MB/s | Reduction |
|-----------------------|------------|--------|------------
| snappy.Encode | 1128706759 | 725.59 | 71.89% |
| s2.EncodeSnappy | 1093823291 | 899.16 | 72.75% |
| s2.EncodeSnappyBetter | 1001158548 | 578.49 | 75.06% |
| s2.EncodeSnappyBest | 944507998 | 66.00 | 76.47% |
## Streams
For streams, replace `enc = snappy.NewBufferedWriter(w)` with `enc = s2.NewWriter(w, s2.WriterSnappyCompat())`.
All other options are available, but note that block size limit is different for snappy.
Comparison of different streams, AMD Ryzen 3950x, 16 cores. Size and throughput:
| File | snappy.NewWriter | S2 Snappy | S2 Snappy, Better | S2 Snappy, Best |
|-----------------------------|--------------------------|---------------------------|--------------------------|-------------------------|
| nyc-taxi-data-10M.csv | 1316042016 - 517.54MB/s | 1307003093 - 8406.29MB/s | 1174534014 - 4984.35MB/s | 1115904679 - 177.81MB/s |
| enwik10 | 5088294643 - 433.45MB/s | 5175840939 - 8454.52MB/s | 4560784526 - 4403.10MB/s | 4340299103 - 159.71MB/s |
| 10gb.tar | 6056946612 - 703.25MB/s | 6208571995 - 9035.75MB/s | 5741646126 - 2402.08MB/s | 5548973895 - 171.17MB/s |
| github-june-2days-2019.json | 1525176492 - 908.11MB/s | 1476519054 - 12625.93MB/s | 1400547532 - 6163.61MB/s | 1321887137 - 200.71MB/s |
| consensus.db.10gb | 5412897703 - 1054.38MB/s | 5354073487 - 12634.82MB/s | 5335069899 - 2472.23MB/s | 5201000954 - 166.32MB/s |
# Decompression
All decompression functions map directly to equivalent s2 functions.
| Snappy | S2 replacement |
|------------------------|--------------------|
| snappy.Decode(...) | s2.Decode(...) |
| snappy.DecodedLen(...) | s2.DecodedLen(...) |
| snappy.NewReader(...) | s2.NewReader(...) |
Features like [quick forward skipping without decompression](https://pkg.go.dev/github.com/klauspost/compress/s2#Reader.Skip)
are also available for Snappy streams.
If you know you are only decompressing snappy streams, setting [`ReaderMaxBlockSize(64<<10)`](https://pkg.go.dev/github.com/klauspost/compress/s2#ReaderMaxBlockSize)
on your Reader will reduce memory consumption.
# Concatenating blocks and streams.
Concatenating streams will concatenate the output of both without recompressing them.
While this is inefficient in terms of compression it might be usable in certain scenarios.
The 10 byte 'stream identifier' of the second stream can optionally be stripped, but it is not a requirement.
Blocks can be concatenated using the `ConcatBlocks` function.
Snappy blocks/streams can safely be concatenated with S2 blocks and streams.
# Format Extensions
* Frame [Stream identifier](https://github.com/google/snappy/blob/master/framing_format.txt#L68) changed from `sNaPpY` to `S2sTwO`.
* [Framed compressed blocks](https://github.com/google/snappy/blob/master/format_description.txt) can be up to 4MB (up from 64KB).
* Compressed blocks can have an offset of `0`, which indicates to repeat the last seen offset.
Repeat offsets must be encoded as a [2.2.1. Copy with 1-byte offset (01)](https://github.com/google/snappy/blob/master/format_description.txt#L89), where the offset is 0.
The length is specified by reading the 3-bit length specified in the tag and decode using this table:
| Length | Actual Length |
|--------|----------------------|
| 0 | 4 |
| 1 | 5 |
| 2 | 6 |
| 3 | 7 |
| 4 | 8 |
| 5 | 8 + read 1 byte |
| 6 | 260 + read 2 bytes |
| 7 | 65540 + read 3 bytes |
This allows any repeat offset + length to be represented by 2 to 5 bytes.
Lengths are stored as little endian values.
The first copy of a block cannot be a repeat offset and the offset is not carried across blocks in streams.
Default streaming block size is 1MB.
# LICENSE
This code is based on the [Snappy-Go](https://github.com/golang/snappy) implementation.
Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.

565
vendor/github.com/klauspost/compress/s2/decode.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,565 @@
// Copyright 2011 The Snappy-Go Authors. All rights reserved.
// Copyright (c) 2019 Klaus Post. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package s2
import (
"encoding/binary"
"errors"
"io"
)
var (
// ErrCorrupt reports that the input is invalid.
ErrCorrupt = errors.New("s2: corrupt input")
// ErrCRC reports that the input failed CRC validation (streams only)
ErrCRC = errors.New("s2: corrupt input, crc mismatch")
// ErrTooLarge reports that the uncompressed length is too large.
ErrTooLarge = errors.New("s2: decoded block is too large")
// ErrUnsupported reports that the input isn't supported.
ErrUnsupported = errors.New("s2: unsupported input")
)
// DecodedLen returns the length of the decoded block.
func DecodedLen(src []byte) (int, error) {
v, _, err := decodedLen(src)
return v, err
}
// decodedLen returns the length of the decoded block and the number of bytes
// that the length header occupied.
func decodedLen(src []byte) (blockLen, headerLen int, err error) {
v, n := binary.Uvarint(src)
if n <= 0 || v > 0xffffffff {
return 0, 0, ErrCorrupt
}
const wordSize = 32 << (^uint(0) >> 32 & 1)
if wordSize == 32 && v > 0x7fffffff {
return 0, 0, ErrTooLarge
}
return int(v), n, nil
}
const (
decodeErrCodeCorrupt = 1
)
// Decode returns the decoded form of src. The returned slice may be a sub-
// slice of dst if dst was large enough to hold the entire decoded block.
// Otherwise, a newly allocated slice will be returned.
//
// The dst and src must not overlap. It is valid to pass a nil dst.
func Decode(dst, src []byte) ([]byte, error) {
dLen, s, err := decodedLen(src)
if err != nil {
return nil, err
}
if dLen <= cap(dst) {
dst = dst[:dLen]
} else {
dst = make([]byte, dLen)
}
if s2Decode(dst, src[s:]) != 0 {
return nil, ErrCorrupt
}
return dst, nil
}
// NewReader returns a new Reader that decompresses from r, using the framing
// format described at
// https://github.com/google/snappy/blob/master/framing_format.txt with S2 changes.
func NewReader(r io.Reader, opts ...ReaderOption) *Reader {
nr := Reader{
r: r,
maxBlock: maxBlockSize,
}
for _, opt := range opts {
if err := opt(&nr); err != nil {
nr.err = err
return &nr
}
}
nr.maxBufSize = MaxEncodedLen(nr.maxBlock) + checksumSize
if nr.lazyBuf > 0 {
nr.buf = make([]byte, MaxEncodedLen(nr.lazyBuf)+checksumSize)
} else {
nr.buf = make([]byte, MaxEncodedLen(defaultBlockSize)+checksumSize)
}
nr.paramsOK = true
return &nr
}
// ReaderOption is an option for creating a decoder.
type ReaderOption func(*Reader) error
// ReaderMaxBlockSize allows to control allocations if the stream
// has been compressed with a smaller WriterBlockSize, or with the default 1MB.
// Blocks must be this size or smaller to decompress,
// otherwise the decoder will return ErrUnsupported.
//
// For streams compressed with Snappy this can safely be set to 64KB (64 << 10).
//
// Default is the maximum limit of 4MB.
func ReaderMaxBlockSize(blockSize int) ReaderOption {
return func(r *Reader) error {
if blockSize > maxBlockSize || blockSize <= 0 {
return errors.New("s2: block size too large. Must be <= 4MB and > 0")
}
if r.lazyBuf == 0 && blockSize < defaultBlockSize {
r.lazyBuf = blockSize
}
r.maxBlock = blockSize
return nil
}
}
// ReaderAllocBlock allows to control upfront stream allocations
// and not allocate for frames bigger than this initially.
// If frames bigger than this is seen a bigger buffer will be allocated.
//
// Default is 1MB, which is default output size.
func ReaderAllocBlock(blockSize int) ReaderOption {
return func(r *Reader) error {
if blockSize > maxBlockSize || blockSize < 1024 {
return errors.New("s2: invalid ReaderAllocBlock. Must be <= 4MB and >= 1024")
}
r.lazyBuf = blockSize
return nil
}
}
// Reader is an io.Reader that can read Snappy-compressed bytes.
type Reader struct {
r io.Reader
err error
decoded []byte
buf []byte
// decoded[i:j] contains decoded bytes that have not yet been passed on.
i, j int
// maximum block size allowed.
maxBlock int
// maximum expected buffer size.
maxBufSize int
// alloc a buffer this size if > 0.
lazyBuf int
readHeader bool
paramsOK bool
snappyFrame bool
}
// ensureBufferSize will ensure that the buffer can take at least n bytes.
// If false is returned the buffer exceeds maximum allowed size.
func (r *Reader) ensureBufferSize(n int) bool {
if len(r.buf) >= n {
return true
}
if n > r.maxBufSize {
r.err = ErrCorrupt
return false
}
// Realloc buffer.
r.buf = make([]byte, n)
return true
}
// Reset discards any buffered data, resets all state, and switches the Snappy
// reader to read from r. This permits reusing a Reader rather than allocating
// a new one.
func (r *Reader) Reset(reader io.Reader) {
if !r.paramsOK {
return
}
r.r = reader
r.err = nil
r.i = 0
r.j = 0
r.readHeader = false
}
func (r *Reader) readFull(p []byte, allowEOF bool) (ok bool) {
if _, r.err = io.ReadFull(r.r, p); r.err != nil {
if r.err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) {
r.err = ErrCorrupt
}
return false
}
return true
}
// skipN will skip n bytes.
// If the supplied reader supports seeking that is used.
// tmp is used as a temporary buffer for reading.
// The supplied slice does not need to be the size of the read.
func (r *Reader) skipN(tmp []byte, n int, allowEOF bool) (ok bool) {
if rs, ok := r.r.(io.ReadSeeker); ok {
_, err := rs.Seek(int64(n), io.SeekCurrent)
if err == nil {
return true
}
if err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) {
r.err = ErrCorrupt
return false
}
}
for n > 0 {
if n < len(tmp) {
tmp = tmp[:n]
}
if _, r.err = io.ReadFull(r.r, tmp); r.err != nil {
if r.err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) {
r.err = ErrCorrupt
}
return false
}
n -= len(tmp)
}
return true
}
// Read satisfies the io.Reader interface.
func (r *Reader) Read(p []byte) (int, error) {
if r.err != nil {
return 0, r.err
}
for {
if r.i < r.j {
n := copy(p, r.decoded[r.i:r.j])
r.i += n
return n, nil
}
if !r.readFull(r.buf[:4], true) {
return 0, r.err
}
chunkType := r.buf[0]
if !r.readHeader {
if chunkType != chunkTypeStreamIdentifier {
r.err = ErrCorrupt
return 0, r.err
}
r.readHeader = true
}
chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16
// The chunk types are specified at
// https://github.com/google/snappy/blob/master/framing_format.txt
switch chunkType {
case chunkTypeCompressedData:
// Section 4.2. Compressed data (chunk type 0x00).
if chunkLen < checksumSize {
r.err = ErrCorrupt
return 0, r.err
}
if !r.ensureBufferSize(chunkLen) {
if r.err == nil {
r.err = ErrUnsupported
}
return 0, r.err
}
buf := r.buf[:chunkLen]
if !r.readFull(buf, false) {
return 0, r.err
}
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
buf = buf[checksumSize:]
n, err := DecodedLen(buf)
if err != nil {
r.err = err
return 0, r.err
}
if r.snappyFrame && n > maxSnappyBlockSize {
r.err = ErrCorrupt
return 0, r.err
}
if n > len(r.decoded) {
if n > r.maxBlock {
r.err = ErrCorrupt
return 0, r.err
}
r.decoded = make([]byte, n)
}
if _, err := Decode(r.decoded, buf); err != nil {
r.err = err
return 0, r.err
}
if crc(r.decoded[:n]) != checksum {
r.err = ErrCRC
return 0, r.err
}
r.i, r.j = 0, n
continue
case chunkTypeUncompressedData:
// Section 4.3. Uncompressed data (chunk type 0x01).
if chunkLen < checksumSize {
r.err = ErrCorrupt
return 0, r.err
}
if !r.ensureBufferSize(chunkLen) {
if r.err == nil {
r.err = ErrUnsupported
}
return 0, r.err
}
buf := r.buf[:checksumSize]
if !r.readFull(buf, false) {
return 0, r.err
}
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
// Read directly into r.decoded instead of via r.buf.
n := chunkLen - checksumSize
if r.snappyFrame && n > maxSnappyBlockSize {
r.err = ErrCorrupt
return 0, r.err
}
if n > len(r.decoded) {
if n > r.maxBlock {
r.err = ErrCorrupt
return 0, r.err
}
r.decoded = make([]byte, n)
}
if !r.readFull(r.decoded[:n], false) {
return 0, r.err
}
if crc(r.decoded[:n]) != checksum {
r.err = ErrCRC
return 0, r.err
}
r.i, r.j = 0, n
continue
case chunkTypeStreamIdentifier:
// Section 4.1. Stream identifier (chunk type 0xff).
if chunkLen != len(magicBody) {
r.err = ErrCorrupt
return 0, r.err
}
if !r.readFull(r.buf[:len(magicBody)], false) {
return 0, r.err
}
if string(r.buf[:len(magicBody)]) != magicBody {
if string(r.buf[:len(magicBody)]) != magicBodySnappy {
r.err = ErrCorrupt
return 0, r.err
} else {
r.snappyFrame = true
}
} else {
r.snappyFrame = false
}
continue
}
if chunkType <= 0x7f {
// Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f).
r.err = ErrUnsupported
return 0, r.err
}
// Section 4.4 Padding (chunk type 0xfe).
// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
if chunkLen > maxBlockSize {
r.err = ErrUnsupported
return 0, r.err
}
if !r.skipN(r.buf, chunkLen, false) {
return 0, r.err
}
}
}
// Skip will skip n bytes forward in the decompressed output.
// For larger skips this consumes less CPU and is faster than reading output and discarding it.
// CRC is not checked on skipped blocks.
// io.ErrUnexpectedEOF is returned if the stream ends before all bytes have been skipped.
// If a decoding error is encountered subsequent calls to Read will also fail.
func (r *Reader) Skip(n int64) error {
if n < 0 {
return errors.New("attempted negative skip")
}
if r.err != nil {
return r.err
}
for n > 0 {
if r.i < r.j {
// Skip in buffer.
// decoded[i:j] contains decoded bytes that have not yet been passed on.
left := int64(r.j - r.i)
if left >= n {
r.i += int(n)
return nil
}
n -= int64(r.j - r.i)
r.i, r.j = 0, 0
}
// Buffer empty; read blocks until we have content.
if !r.readFull(r.buf[:4], true) {
if r.err == io.EOF {
r.err = io.ErrUnexpectedEOF
}
return r.err
}
chunkType := r.buf[0]
if !r.readHeader {
if chunkType != chunkTypeStreamIdentifier {
r.err = ErrCorrupt
return r.err
}
r.readHeader = true
}
chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16
// The chunk types are specified at
// https://github.com/google/snappy/blob/master/framing_format.txt
switch chunkType {
case chunkTypeCompressedData:
// Section 4.2. Compressed data (chunk type 0x00).
if chunkLen < checksumSize {
r.err = ErrCorrupt
return r.err
}
if !r.ensureBufferSize(chunkLen) {
if r.err == nil {
r.err = ErrUnsupported
}
return r.err
}
buf := r.buf[:chunkLen]
if !r.readFull(buf, false) {
return r.err
}
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
buf = buf[checksumSize:]
dLen, err := DecodedLen(buf)
if err != nil {
r.err = err
return r.err
}
if dLen > r.maxBlock {
r.err = ErrCorrupt
return r.err
}
// Check if destination is within this block
if int64(dLen) > n {
if len(r.decoded) < dLen {
r.decoded = make([]byte, dLen)
}
if _, err := Decode(r.decoded, buf); err != nil {
r.err = err
return r.err
}
if crc(r.decoded[:dLen]) != checksum {
r.err = ErrCorrupt
return r.err
}
} else {
// Skip block completely
n -= int64(dLen)
dLen = 0
}
r.i, r.j = 0, dLen
continue
case chunkTypeUncompressedData:
// Section 4.3. Uncompressed data (chunk type 0x01).
if chunkLen < checksumSize {
r.err = ErrCorrupt
return r.err
}
if !r.ensureBufferSize(chunkLen) {
if r.err != nil {
r.err = ErrUnsupported
}
return r.err
}
buf := r.buf[:checksumSize]
if !r.readFull(buf, false) {
return r.err
}
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
// Read directly into r.decoded instead of via r.buf.
n2 := chunkLen - checksumSize
if n2 > len(r.decoded) {
if n2 > r.maxBlock {
r.err = ErrCorrupt
return r.err
}
r.decoded = make([]byte, n2)
}
if !r.readFull(r.decoded[:n2], false) {
return r.err
}
if int64(n2) < n {
if crc(r.decoded[:n2]) != checksum {
r.err = ErrCorrupt
return r.err
}
}
r.i, r.j = 0, n2
continue
case chunkTypeStreamIdentifier:
// Section 4.1. Stream identifier (chunk type 0xff).
if chunkLen != len(magicBody) {
r.err = ErrCorrupt
return r.err
}
if !r.readFull(r.buf[:len(magicBody)], false) {
return r.err
}
if string(r.buf[:len(magicBody)]) != magicBody {
if string(r.buf[:len(magicBody)]) != magicBodySnappy {
r.err = ErrCorrupt
return r.err
}
}
continue
}
if chunkType <= 0x7f {
// Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f).
r.err = ErrUnsupported
return r.err
}
if chunkLen > maxBlockSize {
r.err = ErrUnsupported
return r.err
}
// Section 4.4 Padding (chunk type 0xfe).
// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
if !r.skipN(r.buf, chunkLen, false) {
return r.err
}
}
return nil
}
// ReadByte satisfies the io.ByteReader interface.
func (r *Reader) ReadByte() (byte, error) {
if r.err != nil {
return 0, r.err
}
if r.i < r.j {
c := r.decoded[r.i]
r.i++
return c, nil
}
var tmp [1]byte
for i := 0; i < 10; i++ {
n, err := r.Read(tmp[:])
if err != nil {
return 0, err
}
if n == 1 {
return tmp[0], nil
}
}
return 0, io.ErrNoProgress
}

568
vendor/github.com/klauspost/compress/s2/decode_amd64.s сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,568 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Copyright (c) 2019 Klaus Post. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !appengine
// +build gc
// +build !noasm
#include "textflag.h"
#define R_TMP0 AX
#define R_TMP1 BX
#define R_LEN CX
#define R_OFF DX
#define R_SRC SI
#define R_DST DI
#define R_DBASE R8
#define R_DLEN R9
#define R_DEND R10
#define R_SBASE R11
#define R_SLEN R12
#define R_SEND R13
#define R_TMP2 R14
#define R_TMP3 R15
// The asm code generally follows the pure Go code in decode_other.go, except
// where marked with a "!!!".
// func decode(dst, src []byte) int
//
// All local variables fit into registers. The non-zero stack size is only to
// spill registers and push args when issuing a CALL. The register allocation:
// - R_TMP0 scratch
// - R_TMP1 scratch
// - R_LEN length or x (shared)
// - R_OFF offset
// - R_SRC &src[s]
// - R_DST &dst[d]
// + R_DBASE dst_base
// + R_DLEN dst_len
// + R_DEND dst_base + dst_len
// + R_SBASE src_base
// + R_SLEN src_len
// + R_SEND src_base + src_len
// - R_TMP2 used by doCopy
// - R_TMP3 used by doCopy
//
// The registers R_DBASE-R_SEND (marked with a "+") are set at the start of the
// function, and after a CALL returns, and are not otherwise modified.
//
// The d variable is implicitly R_DST - R_DBASE, and len(dst)-d is R_DEND - R_DST.
// The s variable is implicitly R_SRC - R_SBASE, and len(src)-s is R_SEND - R_SRC.
TEXT ·s2Decode(SB), NOSPLIT, $48-56
// Initialize R_SRC, R_DST and R_DBASE-R_SEND.
MOVQ dst_base+0(FP), R_DBASE
MOVQ dst_len+8(FP), R_DLEN
MOVQ R_DBASE, R_DST
MOVQ R_DBASE, R_DEND
ADDQ R_DLEN, R_DEND
MOVQ src_base+24(FP), R_SBASE
MOVQ src_len+32(FP), R_SLEN
MOVQ R_SBASE, R_SRC
MOVQ R_SBASE, R_SEND
ADDQ R_SLEN, R_SEND
XORQ R_OFF, R_OFF
loop:
// for s < len(src)
CMPQ R_SRC, R_SEND
JEQ end
// R_LEN = uint32(src[s])
//
// switch src[s] & 0x03
MOVBLZX (R_SRC), R_LEN
MOVL R_LEN, R_TMP1
ANDL $3, R_TMP1
CMPL R_TMP1, $1
JAE tagCopy
// ----------------------------------------
// The code below handles literal tags.
// case tagLiteral:
// x := uint32(src[s] >> 2)
// switch
SHRL $2, R_LEN
CMPL R_LEN, $60
JAE tagLit60Plus
// case x < 60:
// s++
INCQ R_SRC
doLit:
// This is the end of the inner "switch", when we have a literal tag.
//
// We assume that R_LEN == x and x fits in a uint32, where x is the variable
// used in the pure Go decode_other.go code.
// length = int(x) + 1
//
// Unlike the pure Go code, we don't need to check if length <= 0 because
// R_LEN can hold 64 bits, so the increment cannot overflow.
INCQ R_LEN
// Prepare to check if copying length bytes will run past the end of dst or
// src.
//
// R_TMP0 = len(dst) - d
// R_TMP1 = len(src) - s
MOVQ R_DEND, R_TMP0
SUBQ R_DST, R_TMP0
MOVQ R_SEND, R_TMP1
SUBQ R_SRC, R_TMP1
// !!! Try a faster technique for short (16 or fewer bytes) copies.
//
// if length > 16 || len(dst)-d < 16 || len(src)-s < 16 {
// goto callMemmove // Fall back on calling runtime·memmove.
// }
//
// The C++ snappy code calls this TryFastAppend. It also checks len(src)-s
// against 21 instead of 16, because it cannot assume that all of its input
// is contiguous in memory and so it needs to leave enough source bytes to
// read the next tag without refilling buffers, but Go's Decode assumes
// contiguousness (the src argument is a []byte).
CMPQ R_LEN, $16
JGT callMemmove
CMPQ R_TMP0, $16
JLT callMemmove
CMPQ R_TMP1, $16
JLT callMemmove
// !!! Implement the copy from src to dst as a 16-byte load and store.
// (Decode's documentation says that dst and src must not overlap.)
//
// This always copies 16 bytes, instead of only length bytes, but that's
// OK. If the input is a valid Snappy encoding then subsequent iterations
// will fix up the overrun. Otherwise, Decode returns a nil []byte (and a
// non-nil error), so the overrun will be ignored.
//
// Note that on amd64, it is legal and cheap to issue unaligned 8-byte or
// 16-byte loads and stores. This technique probably wouldn't be as
// effective on architectures that are fussier about alignment.
MOVOU 0(R_SRC), X0
MOVOU X0, 0(R_DST)
// d += length
// s += length
ADDQ R_LEN, R_DST
ADDQ R_LEN, R_SRC
JMP loop
callMemmove:
// if length > len(dst)-d || length > len(src)-s { etc }
CMPQ R_LEN, R_TMP0
JGT errCorrupt
CMPQ R_LEN, R_TMP1
JGT errCorrupt
// copy(dst[d:], src[s:s+length])
//
// This means calling runtime·memmove(&dst[d], &src[s], length), so we push
// R_DST, R_SRC and R_LEN as arguments. Coincidentally, we also need to spill those
// three registers to the stack, to save local variables across the CALL.
MOVQ R_DST, 0(SP)
MOVQ R_SRC, 8(SP)
MOVQ R_LEN, 16(SP)
MOVQ R_DST, 24(SP)
MOVQ R_SRC, 32(SP)
MOVQ R_LEN, 40(SP)
MOVQ R_OFF, 48(SP)
CALL runtime·memmove(SB)
// Restore local variables: unspill registers from the stack and
// re-calculate R_DBASE-R_SEND.
MOVQ 24(SP), R_DST
MOVQ 32(SP), R_SRC
MOVQ 40(SP), R_LEN
MOVQ 48(SP), R_OFF
MOVQ dst_base+0(FP), R_DBASE
MOVQ dst_len+8(FP), R_DLEN
MOVQ R_DBASE, R_DEND
ADDQ R_DLEN, R_DEND
MOVQ src_base+24(FP), R_SBASE
MOVQ src_len+32(FP), R_SLEN
MOVQ R_SBASE, R_SEND
ADDQ R_SLEN, R_SEND
// d += length
// s += length
ADDQ R_LEN, R_DST
ADDQ R_LEN, R_SRC
JMP loop
tagLit60Plus:
// !!! This fragment does the
//
// s += x - 58; if uint(s) > uint(len(src)) { etc }
//
// checks. In the asm version, we code it once instead of once per switch case.
ADDQ R_LEN, R_SRC
SUBQ $58, R_SRC
CMPQ R_SRC, R_SEND
JA errCorrupt
// case x == 60:
CMPL R_LEN, $61
JEQ tagLit61
JA tagLit62Plus
// x = uint32(src[s-1])
MOVBLZX -1(R_SRC), R_LEN
JMP doLit
tagLit61:
// case x == 61:
// x = uint32(src[s-2]) | uint32(src[s-1])<<8
MOVWLZX -2(R_SRC), R_LEN
JMP doLit
tagLit62Plus:
CMPL R_LEN, $62
JA tagLit63
// case x == 62:
// x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
// We read one byte, safe to read one back, since we are just reading tag.
// x = binary.LittleEndian.Uint32(src[s-1:]) >> 8
MOVL -4(R_SRC), R_LEN
SHRL $8, R_LEN
JMP doLit
tagLit63:
// case x == 63:
// x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
MOVL -4(R_SRC), R_LEN
JMP doLit
// The code above handles literal tags.
// ----------------------------------------
// The code below handles copy tags.
tagCopy4:
// case tagCopy4:
// s += 5
ADDQ $5, R_SRC
// if uint(s) > uint(len(src)) { etc }
CMPQ R_SRC, R_SEND
JA errCorrupt
// length = 1 + int(src[s-5])>>2
SHRQ $2, R_LEN
INCQ R_LEN
// offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
MOVLQZX -4(R_SRC), R_OFF
JMP doCopy
tagCopy2:
// case tagCopy2:
// s += 3
ADDQ $3, R_SRC
// if uint(s) > uint(len(src)) { etc }
CMPQ R_SRC, R_SEND
JA errCorrupt
// length = 1 + int(src[s-3])>>2
SHRQ $2, R_LEN
INCQ R_LEN
// offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
MOVWQZX -2(R_SRC), R_OFF
JMP doCopy
tagCopy:
// We have a copy tag. We assume that:
// - R_TMP1 == src[s] & 0x03
// - R_LEN == src[s]
CMPQ R_TMP1, $2
JEQ tagCopy2
JA tagCopy4
// case tagCopy1:
// s += 2
ADDQ $2, R_SRC
// if uint(s) > uint(len(src)) { etc }
CMPQ R_SRC, R_SEND
JA errCorrupt
// offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
// length = 4 + int(src[s-2])>>2&0x7
MOVBQZX -1(R_SRC), R_TMP1
MOVQ R_LEN, R_TMP0
SHRQ $2, R_LEN
ANDQ $0xe0, R_TMP0
ANDQ $7, R_LEN
SHLQ $3, R_TMP0
ADDQ $4, R_LEN
ORQ R_TMP1, R_TMP0
// check if repeat code, ZF set by ORQ.
JZ repeatCode
// This is a regular copy, transfer our temporary value to R_OFF (length)
MOVQ R_TMP0, R_OFF
JMP doCopy
// This is a repeat code.
repeatCode:
// If length < 9, reuse last offset, with the length already calculated.
CMPQ R_LEN, $9
JL doCopyRepeat
// Read additional bytes for length.
JE repeatLen1
// Rare, so the extra branch shouldn't hurt too much.
CMPQ R_LEN, $10
JE repeatLen2
JMP repeatLen3
// Read repeat lengths.
repeatLen1:
// s ++
ADDQ $1, R_SRC
// if uint(s) > uint(len(src)) { etc }
CMPQ R_SRC, R_SEND
JA errCorrupt
// length = src[s-1] + 8
MOVBQZX -1(R_SRC), R_LEN
ADDL $8, R_LEN
JMP doCopyRepeat
repeatLen2:
// s +=2
ADDQ $2, R_SRC
// if uint(s) > uint(len(src)) { etc }
CMPQ R_SRC, R_SEND
JA errCorrupt
// length = uint32(src[s-2]) | (uint32(src[s-1])<<8) + (1 << 8)
MOVWQZX -2(R_SRC), R_LEN
ADDL $260, R_LEN
JMP doCopyRepeat
repeatLen3:
// s +=3
ADDQ $3, R_SRC
// if uint(s) > uint(len(src)) { etc }
CMPQ R_SRC, R_SEND
JA errCorrupt
// length = uint32(src[s-3]) | (uint32(src[s-2])<<8) | (uint32(src[s-1])<<16) + (1 << 16)
// Read one byte further back (just part of the tag, shifted out)
MOVL -4(R_SRC), R_LEN
SHRL $8, R_LEN
ADDL $65540, R_LEN
JMP doCopyRepeat
doCopy:
// This is the end of the outer "switch", when we have a copy tag.
//
// We assume that:
// - R_LEN == length && R_LEN > 0
// - R_OFF == offset
// if d < offset { etc }
MOVQ R_DST, R_TMP1
SUBQ R_DBASE, R_TMP1
CMPQ R_TMP1, R_OFF
JLT errCorrupt
// Repeat values can skip the test above, since any offset > 0 will be in dst.
doCopyRepeat:
// if offset <= 0 { etc }
CMPQ R_OFF, $0
JLE errCorrupt
// if length > len(dst)-d { etc }
MOVQ R_DEND, R_TMP1
SUBQ R_DST, R_TMP1
CMPQ R_LEN, R_TMP1
JGT errCorrupt
// forwardCopy(dst[d:d+length], dst[d-offset:]); d += length
//
// Set:
// - R_TMP2 = len(dst)-d
// - R_TMP3 = &dst[d-offset]
MOVQ R_DEND, R_TMP2
SUBQ R_DST, R_TMP2
MOVQ R_DST, R_TMP3
SUBQ R_OFF, R_TMP3
// !!! Try a faster technique for short (16 or fewer bytes) forward copies.
//
// First, try using two 8-byte load/stores, similar to the doLit technique
// above. Even if dst[d:d+length] and dst[d-offset:] can overlap, this is
// still OK if offset >= 8. Note that this has to be two 8-byte load/stores
// and not one 16-byte load/store, and the first store has to be before the
// second load, due to the overlap if offset is in the range [8, 16).
//
// if length > 16 || offset < 8 || len(dst)-d < 16 {
// goto slowForwardCopy
// }
// copy 16 bytes
// d += length
CMPQ R_LEN, $16
JGT slowForwardCopy
CMPQ R_OFF, $8
JLT slowForwardCopy
CMPQ R_TMP2, $16
JLT slowForwardCopy
MOVQ 0(R_TMP3), R_TMP0
MOVQ R_TMP0, 0(R_DST)
MOVQ 8(R_TMP3), R_TMP1
MOVQ R_TMP1, 8(R_DST)
ADDQ R_LEN, R_DST
JMP loop
slowForwardCopy:
// !!! If the forward copy is longer than 16 bytes, or if offset < 8, we
// can still try 8-byte load stores, provided we can overrun up to 10 extra
// bytes. As above, the overrun will be fixed up by subsequent iterations
// of the outermost loop.
//
// The C++ snappy code calls this technique IncrementalCopyFastPath. Its
// commentary says:
//
// ----
//
// The main part of this loop is a simple copy of eight bytes at a time
// until we've copied (at least) the requested amount of bytes. However,
// if d and d-offset are less than eight bytes apart (indicating a
// repeating pattern of length < 8), we first need to expand the pattern in
// order to get the correct results. For instance, if the buffer looks like
// this, with the eight-byte <d-offset> and <d> patterns marked as
// intervals:
//
// abxxxxxxxxxxxx
// [------] d-offset
// [------] d
//
// a single eight-byte copy from <d-offset> to <d> will repeat the pattern
// once, after which we can move <d> two bytes without moving <d-offset>:
//
// ababxxxxxxxxxx
// [------] d-offset
// [------] d
//
// and repeat the exercise until the two no longer overlap.
//
// This allows us to do very well in the special case of one single byte
// repeated many times, without taking a big hit for more general cases.
//
// The worst case of extra writing past the end of the match occurs when
// offset == 1 and length == 1; the last copy will read from byte positions
// [0..7] and write to [4..11], whereas it was only supposed to write to
// position 1. Thus, ten excess bytes.
//
// ----
//
// That "10 byte overrun" worst case is confirmed by Go's
// TestSlowForwardCopyOverrun, which also tests the fixUpSlowForwardCopy
// and finishSlowForwardCopy algorithm.
//
// if length > len(dst)-d-10 {
// goto verySlowForwardCopy
// }
SUBQ $10, R_TMP2
CMPQ R_LEN, R_TMP2
JGT verySlowForwardCopy
// We want to keep the offset, so we use R_TMP2 from here.
MOVQ R_OFF, R_TMP2
makeOffsetAtLeast8:
// !!! As above, expand the pattern so that offset >= 8 and we can use
// 8-byte load/stores.
//
// for offset < 8 {
// copy 8 bytes from dst[d-offset:] to dst[d:]
// length -= offset
// d += offset
// offset += offset
// // The two previous lines together means that d-offset, and therefore
// // R_TMP3, is unchanged.
// }
CMPQ R_TMP2, $8
JGE fixUpSlowForwardCopy
MOVQ (R_TMP3), R_TMP1
MOVQ R_TMP1, (R_DST)
SUBQ R_TMP2, R_LEN
ADDQ R_TMP2, R_DST
ADDQ R_TMP2, R_TMP2
JMP makeOffsetAtLeast8
fixUpSlowForwardCopy:
// !!! Add length (which might be negative now) to d (implied by R_DST being
// &dst[d]) so that d ends up at the right place when we jump back to the
// top of the loop. Before we do that, though, we save R_DST to R_TMP0 so that, if
// length is positive, copying the remaining length bytes will write to the
// right place.
MOVQ R_DST, R_TMP0
ADDQ R_LEN, R_DST
finishSlowForwardCopy:
// !!! Repeat 8-byte load/stores until length <= 0. Ending with a negative
// length means that we overrun, but as above, that will be fixed up by
// subsequent iterations of the outermost loop.
CMPQ R_LEN, $0
JLE loop
MOVQ (R_TMP3), R_TMP1
MOVQ R_TMP1, (R_TMP0)
ADDQ $8, R_TMP3
ADDQ $8, R_TMP0
SUBQ $8, R_LEN
JMP finishSlowForwardCopy
verySlowForwardCopy:
// verySlowForwardCopy is a simple implementation of forward copy. In C
// parlance, this is a do/while loop instead of a while loop, since we know
// that length > 0. In Go syntax:
//
// for {
// dst[d] = dst[d - offset]
// d++
// length--
// if length == 0 {
// break
// }
// }
MOVB (R_TMP3), R_TMP1
MOVB R_TMP1, (R_DST)
INCQ R_TMP3
INCQ R_DST
DECQ R_LEN
JNZ verySlowForwardCopy
JMP loop
// The code above handles copy tags.
// ----------------------------------------
end:
// This is the end of the "for s < len(src)".
//
// if d != len(dst) { etc }
CMPQ R_DST, R_DEND
JNE errCorrupt
// return 0
MOVQ $0, ret+48(FP)
RET
errCorrupt:
// return decodeErrCodeCorrupt
MOVQ $1, ret+48(FP)
RET

574
vendor/github.com/klauspost/compress/s2/decode_arm64.s сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,574 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !appengine
// +build gc
// +build !noasm
#include "textflag.h"
#define R_TMP0 R2
#define R_TMP1 R3
#define R_LEN R4
#define R_OFF R5
#define R_SRC R6
#define R_DST R7
#define R_DBASE R8
#define R_DLEN R9
#define R_DEND R10
#define R_SBASE R11
#define R_SLEN R12
#define R_SEND R13
#define R_TMP2 R14
#define R_TMP3 R15
// TEST_SRC will check if R_SRC is <= SRC_END
#define TEST_SRC() \
CMP R_SEND, R_SRC \
BGT errCorrupt
// MOVD R_SRC, R_TMP1
// SUB R_SBASE, R_TMP1, R_TMP1
// CMP R_SLEN, R_TMP1
// BGT errCorrupt
// The asm code generally follows the pure Go code in decode_other.go, except
// where marked with a "!!!".
// func decode(dst, src []byte) int
//
// All local variables fit into registers. The non-zero stack size is only to
// spill registers and push args when issuing a CALL. The register allocation:
// - R_TMP0 scratch
// - R_TMP1 scratch
// - R_LEN length or x
// - R_OFF offset
// - R_SRC &src[s]
// - R_DST &dst[d]
// + R_DBASE dst_base
// + R_DLEN dst_len
// + R_DEND dst_base + dst_len
// + R_SBASE src_base
// + R_SLEN src_len
// + R_SEND src_base + src_len
// - R_TMP2 used by doCopy
// - R_TMP3 used by doCopy
//
// The registers R_DBASE-R_SEND (marked with a "+") are set at the start of the
// function, and after a CALL returns, and are not otherwise modified.
//
// The d variable is implicitly R_DST - R_DBASE, and len(dst)-d is R_DEND - R_DST.
// The s variable is implicitly R_SRC - R_SBASE, and len(src)-s is R_SEND - R_SRC.
TEXT ·s2Decode(SB), NOSPLIT, $56-64
// Initialize R_SRC, R_DST and R_DBASE-R_SEND.
MOVD dst_base+0(FP), R_DBASE
MOVD dst_len+8(FP), R_DLEN
MOVD R_DBASE, R_DST
MOVD R_DBASE, R_DEND
ADD R_DLEN, R_DEND, R_DEND
MOVD src_base+24(FP), R_SBASE
MOVD src_len+32(FP), R_SLEN
MOVD R_SBASE, R_SRC
MOVD R_SBASE, R_SEND
ADD R_SLEN, R_SEND, R_SEND
MOVD $0, R_OFF
loop:
// for s < len(src)
CMP R_SEND, R_SRC
BEQ end
// R_LEN = uint32(src[s])
//
// switch src[s] & 0x03
MOVBU (R_SRC), R_LEN
MOVW R_LEN, R_TMP1
ANDW $3, R_TMP1
MOVW $1, R1
CMPW R1, R_TMP1
BGE tagCopy
// ----------------------------------------
// The code below handles literal tags.
// case tagLiteral:
// x := uint32(src[s] >> 2)
// switch
MOVW $60, R1
LSRW $2, R_LEN, R_LEN
CMPW R_LEN, R1
BLS tagLit60Plus
// case x < 60:
// s++
ADD $1, R_SRC, R_SRC
doLit:
// This is the end of the inner "switch", when we have a literal tag.
//
// We assume that R_LEN == x and x fits in a uint32, where x is the variable
// used in the pure Go decode_other.go code.
// length = int(x) + 1
//
// Unlike the pure Go code, we don't need to check if length <= 0 because
// R_LEN can hold 64 bits, so the increment cannot overflow.
ADD $1, R_LEN, R_LEN
// Prepare to check if copying length bytes will run past the end of dst or
// src.
//
// R_TMP0 = len(dst) - d
// R_TMP1 = len(src) - s
MOVD R_DEND, R_TMP0
SUB R_DST, R_TMP0, R_TMP0
MOVD R_SEND, R_TMP1
SUB R_SRC, R_TMP1, R_TMP1
// !!! Try a faster technique for short (16 or fewer bytes) copies.
//
// if length > 16 || len(dst)-d < 16 || len(src)-s < 16 {
// goto callMemmove // Fall back on calling runtime·memmove.
// }
//
// The C++ snappy code calls this TryFastAppend. It also checks len(src)-s
// against 21 instead of 16, because it cannot assume that all of its input
// is contiguous in memory and so it needs to leave enough source bytes to
// read the next tag without refilling buffers, but Go's Decode assumes
// contiguousness (the src argument is a []byte).
CMP $16, R_LEN
BGT callMemmove
CMP $16, R_TMP0
BLT callMemmove
CMP $16, R_TMP1
BLT callMemmove
// !!! Implement the copy from src to dst as a 16-byte load and store.
// (Decode's documentation says that dst and src must not overlap.)
//
// This always copies 16 bytes, instead of only length bytes, but that's
// OK. If the input is a valid Snappy encoding then subsequent iterations
// will fix up the overrun. Otherwise, Decode returns a nil []byte (and a
// non-nil error), so the overrun will be ignored.
//
// Note that on arm64, it is legal and cheap to issue unaligned 8-byte or
// 16-byte loads and stores. This technique probably wouldn't be as
// effective on architectures that are fussier about alignment.
LDP 0(R_SRC), (R_TMP2, R_TMP3)
STP (R_TMP2, R_TMP3), 0(R_DST)
// d += length
// s += length
ADD R_LEN, R_DST, R_DST
ADD R_LEN, R_SRC, R_SRC
B loop
callMemmove:
// if length > len(dst)-d || length > len(src)-s { etc }
CMP R_TMP0, R_LEN
BGT errCorrupt
CMP R_TMP1, R_LEN
BGT errCorrupt
// copy(dst[d:], src[s:s+length])
//
// This means calling runtime·memmove(&dst[d], &src[s], length), so we push
// R_DST, R_SRC and R_LEN as arguments. Coincidentally, we also need to spill those
// three registers to the stack, to save local variables across the CALL.
MOVD R_DST, 8(RSP)
MOVD R_SRC, 16(RSP)
MOVD R_LEN, 24(RSP)
MOVD R_DST, 32(RSP)
MOVD R_SRC, 40(RSP)
MOVD R_LEN, 48(RSP)
MOVD R_OFF, 56(RSP)
CALL runtime·memmove(SB)
// Restore local variables: unspill registers from the stack and
// re-calculate R_DBASE-R_SEND.
MOVD 32(RSP), R_DST
MOVD 40(RSP), R_SRC
MOVD 48(RSP), R_LEN
MOVD 56(RSP), R_OFF
MOVD dst_base+0(FP), R_DBASE
MOVD dst_len+8(FP), R_DLEN
MOVD R_DBASE, R_DEND
ADD R_DLEN, R_DEND, R_DEND
MOVD src_base+24(FP), R_SBASE
MOVD src_len+32(FP), R_SLEN
MOVD R_SBASE, R_SEND
ADD R_SLEN, R_SEND, R_SEND
// d += length
// s += length
ADD R_LEN, R_DST, R_DST
ADD R_LEN, R_SRC, R_SRC
B loop
tagLit60Plus:
// !!! This fragment does the
//
// s += x - 58; if uint(s) > uint(len(src)) { etc }
//
// checks. In the asm version, we code it once instead of once per switch case.
ADD R_LEN, R_SRC, R_SRC
SUB $58, R_SRC, R_SRC
TEST_SRC()
// case x == 60:
MOVW $61, R1
CMPW R1, R_LEN
BEQ tagLit61
BGT tagLit62Plus
// x = uint32(src[s-1])
MOVBU -1(R_SRC), R_LEN
B doLit
tagLit61:
// case x == 61:
// x = uint32(src[s-2]) | uint32(src[s-1])<<8
MOVHU -2(R_SRC), R_LEN
B doLit
tagLit62Plus:
CMPW $62, R_LEN
BHI tagLit63
// case x == 62:
// x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
MOVHU -3(R_SRC), R_LEN
MOVBU -1(R_SRC), R_TMP1
ORR R_TMP1<<16, R_LEN
B doLit
tagLit63:
// case x == 63:
// x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
MOVWU -4(R_SRC), R_LEN
B doLit
// The code above handles literal tags.
// ----------------------------------------
// The code below handles copy tags.
tagCopy4:
// case tagCopy4:
// s += 5
ADD $5, R_SRC, R_SRC
// if uint(s) > uint(len(src)) { etc }
MOVD R_SRC, R_TMP1
SUB R_SBASE, R_TMP1, R_TMP1
CMP R_SLEN, R_TMP1
BGT errCorrupt
// length = 1 + int(src[s-5])>>2
MOVD $1, R1
ADD R_LEN>>2, R1, R_LEN
// offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
MOVWU -4(R_SRC), R_OFF
B doCopy
tagCopy2:
// case tagCopy2:
// s += 3
ADD $3, R_SRC, R_SRC
// if uint(s) > uint(len(src)) { etc }
TEST_SRC()
// length = 1 + int(src[s-3])>>2
MOVD $1, R1
ADD R_LEN>>2, R1, R_LEN
// offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
MOVHU -2(R_SRC), R_OFF
B doCopy
tagCopy:
// We have a copy tag. We assume that:
// - R_TMP1 == src[s] & 0x03
// - R_LEN == src[s]
CMP $2, R_TMP1
BEQ tagCopy2
BGT tagCopy4
// case tagCopy1:
// s += 2
ADD $2, R_SRC, R_SRC
// if uint(s) > uint(len(src)) { etc }
TEST_SRC()
// offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
// Calculate offset in R_TMP0 in case it is a repeat.
MOVD R_LEN, R_TMP0
AND $0xe0, R_TMP0
MOVBU -1(R_SRC), R_TMP1
ORR R_TMP0<<3, R_TMP1, R_TMP0
// length = 4 + int(src[s-2])>>2&0x7
MOVD $7, R1
AND R_LEN>>2, R1, R_LEN
ADD $4, R_LEN, R_LEN
// check if repeat code with offset 0.
CMP $0, R_TMP0
BEQ repeatCode
// This is a regular copy, transfer our temporary value to R_OFF (offset)
MOVD R_TMP0, R_OFF
B doCopy
// This is a repeat code.
repeatCode:
// If length < 9, reuse last offset, with the length already calculated.
CMP $9, R_LEN
BLT doCopyRepeat
BEQ repeatLen1
CMP $10, R_LEN
BEQ repeatLen2
repeatLen3:
// s +=3
ADD $3, R_SRC, R_SRC
// if uint(s) > uint(len(src)) { etc }
TEST_SRC()
// length = uint32(src[s-3]) | (uint32(src[s-2])<<8) | (uint32(src[s-1])<<16) + 65540
MOVBU -1(R_SRC), R_TMP0
MOVHU -3(R_SRC), R_LEN
ORR R_TMP0<<16, R_LEN, R_LEN
ADD $65540, R_LEN, R_LEN
B doCopyRepeat
repeatLen2:
// s +=2
ADD $2, R_SRC, R_SRC
// if uint(s) > uint(len(src)) { etc }
TEST_SRC()
// length = uint32(src[s-2]) | (uint32(src[s-1])<<8) + 260
MOVHU -2(R_SRC), R_LEN
ADD $260, R_LEN, R_LEN
B doCopyRepeat
repeatLen1:
// s +=1
ADD $1, R_SRC, R_SRC
// if uint(s) > uint(len(src)) { etc }
TEST_SRC()
// length = src[s-1] + 8
MOVBU -1(R_SRC), R_LEN
ADD $8, R_LEN, R_LEN
B doCopyRepeat
doCopy:
// This is the end of the outer "switch", when we have a copy tag.
//
// We assume that:
// - R_LEN == length && R_LEN > 0
// - R_OFF == offset
// if d < offset { etc }
MOVD R_DST, R_TMP1
SUB R_DBASE, R_TMP1, R_TMP1
CMP R_OFF, R_TMP1
BLT errCorrupt
// Repeat values can skip the test above, since any offset > 0 will be in dst.
doCopyRepeat:
// if offset <= 0 { etc }
CMP $0, R_OFF
BLE errCorrupt
// if length > len(dst)-d { etc }
MOVD R_DEND, R_TMP1
SUB R_DST, R_TMP1, R_TMP1
CMP R_TMP1, R_LEN
BGT errCorrupt
// forwardCopy(dst[d:d+length], dst[d-offset:]); d += length
//
// Set:
// - R_TMP2 = len(dst)-d
// - R_TMP3 = &dst[d-offset]
MOVD R_DEND, R_TMP2
SUB R_DST, R_TMP2, R_TMP2
MOVD R_DST, R_TMP3
SUB R_OFF, R_TMP3, R_TMP3
// !!! Try a faster technique for short (16 or fewer bytes) forward copies.
//
// First, try using two 8-byte load/stores, similar to the doLit technique
// above. Even if dst[d:d+length] and dst[d-offset:] can overlap, this is
// still OK if offset >= 8. Note that this has to be two 8-byte load/stores
// and not one 16-byte load/store, and the first store has to be before the
// second load, due to the overlap if offset is in the range [8, 16).
//
// if length > 16 || offset < 8 || len(dst)-d < 16 {
// goto slowForwardCopy
// }
// copy 16 bytes
// d += length
CMP $16, R_LEN
BGT slowForwardCopy
CMP $8, R_OFF
BLT slowForwardCopy
CMP $16, R_TMP2
BLT slowForwardCopy
MOVD 0(R_TMP3), R_TMP0
MOVD R_TMP0, 0(R_DST)
MOVD 8(R_TMP3), R_TMP1
MOVD R_TMP1, 8(R_DST)
ADD R_LEN, R_DST, R_DST
B loop
slowForwardCopy:
// !!! If the forward copy is longer than 16 bytes, or if offset < 8, we
// can still try 8-byte load stores, provided we can overrun up to 10 extra
// bytes. As above, the overrun will be fixed up by subsequent iterations
// of the outermost loop.
//
// The C++ snappy code calls this technique IncrementalCopyFastPath. Its
// commentary says:
//
// ----
//
// The main part of this loop is a simple copy of eight bytes at a time
// until we've copied (at least) the requested amount of bytes. However,
// if d and d-offset are less than eight bytes apart (indicating a
// repeating pattern of length < 8), we first need to expand the pattern in
// order to get the correct results. For instance, if the buffer looks like
// this, with the eight-byte <d-offset> and <d> patterns marked as
// intervals:
//
// abxxxxxxxxxxxx
// [------] d-offset
// [------] d
//
// a single eight-byte copy from <d-offset> to <d> will repeat the pattern
// once, after which we can move <d> two bytes without moving <d-offset>:
//
// ababxxxxxxxxxx
// [------] d-offset
// [------] d
//
// and repeat the exercise until the two no longer overlap.
//
// This allows us to do very well in the special case of one single byte
// repeated many times, without taking a big hit for more general cases.
//
// The worst case of extra writing past the end of the match occurs when
// offset == 1 and length == 1; the last copy will read from byte positions
// [0..7] and write to [4..11], whereas it was only supposed to write to
// position 1. Thus, ten excess bytes.
//
// ----
//
// That "10 byte overrun" worst case is confirmed by Go's
// TestSlowForwardCopyOverrun, which also tests the fixUpSlowForwardCopy
// and finishSlowForwardCopy algorithm.
//
// if length > len(dst)-d-10 {
// goto verySlowForwardCopy
// }
SUB $10, R_TMP2, R_TMP2
CMP R_TMP2, R_LEN
BGT verySlowForwardCopy
// We want to keep the offset, so we use R_TMP2 from here.
MOVD R_OFF, R_TMP2
makeOffsetAtLeast8:
// !!! As above, expand the pattern so that offset >= 8 and we can use
// 8-byte load/stores.
//
// for offset < 8 {
// copy 8 bytes from dst[d-offset:] to dst[d:]
// length -= offset
// d += offset
// offset += offset
// // The two previous lines together means that d-offset, and therefore
// // R_TMP3, is unchanged.
// }
CMP $8, R_TMP2
BGE fixUpSlowForwardCopy
MOVD (R_TMP3), R_TMP1
MOVD R_TMP1, (R_DST)
SUB R_TMP2, R_LEN, R_LEN
ADD R_TMP2, R_DST, R_DST
ADD R_TMP2, R_TMP2, R_TMP2
B makeOffsetAtLeast8
fixUpSlowForwardCopy:
// !!! Add length (which might be negative now) to d (implied by R_DST being
// &dst[d]) so that d ends up at the right place when we jump back to the
// top of the loop. Before we do that, though, we save R_DST to R_TMP0 so that, if
// length is positive, copying the remaining length bytes will write to the
// right place.
MOVD R_DST, R_TMP0
ADD R_LEN, R_DST, R_DST
finishSlowForwardCopy:
// !!! Repeat 8-byte load/stores until length <= 0. Ending with a negative
// length means that we overrun, but as above, that will be fixed up by
// subsequent iterations of the outermost loop.
MOVD $0, R1
CMP R1, R_LEN
BLE loop
MOVD (R_TMP3), R_TMP1
MOVD R_TMP1, (R_TMP0)
ADD $8, R_TMP3, R_TMP3
ADD $8, R_TMP0, R_TMP0
SUB $8, R_LEN, R_LEN
B finishSlowForwardCopy
verySlowForwardCopy:
// verySlowForwardCopy is a simple implementation of forward copy. In C
// parlance, this is a do/while loop instead of a while loop, since we know
// that length > 0. In Go syntax:
//
// for {
// dst[d] = dst[d - offset]
// d++
// length--
// if length == 0 {
// break
// }
// }
MOVB (R_TMP3), R_TMP1
MOVB R_TMP1, (R_DST)
ADD $1, R_TMP3, R_TMP3
ADD $1, R_DST, R_DST
SUB $1, R_LEN, R_LEN
CBNZ R_LEN, verySlowForwardCopy
B loop
// The code above handles copy tags.
// ----------------------------------------
end:
// This is the end of the "for s < len(src)".
//
// if d != len(dst) { etc }
CMP R_DEND, R_DST
BNE errCorrupt
// return 0
MOVD $0, ret+48(FP)
RET
errCorrupt:
// return decodeErrCodeCorrupt
MOVD $1, R_TMP0
MOVD R_TMP0, ret+48(FP)
RET

17
vendor/github.com/klauspost/compress/s2/decode_asm.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,17 @@
// Copyright 2016 The Snappy-Go Authors. All rights reserved.
// Copyright (c) 2019 Klaus Post. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build (amd64 || arm64) && !appengine && gc && !noasm
// +build amd64 arm64
// +build !appengine
// +build gc
// +build !noasm
package s2
// decode has the same semantics as in decode_other.go.
//
//go:noescape
func s2Decode(dst, src []byte) int

267
vendor/github.com/klauspost/compress/s2/decode_other.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,267 @@
// Copyright 2016 The Snappy-Go Authors. All rights reserved.
// Copyright (c) 2019 Klaus Post. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build (!amd64 && !arm64) || appengine || !gc || noasm
// +build !amd64,!arm64 appengine !gc noasm
package s2
import (
"fmt"
"strconv"
)
// decode writes the decoding of src to dst. It assumes that the varint-encoded
// length of the decompressed bytes has already been read, and that len(dst)
// equals that length.
//
// It returns 0 on success or a decodeErrCodeXxx error code on failure.
func s2Decode(dst, src []byte) int {
const debug = false
if debug {
fmt.Println("Starting decode, dst len:", len(dst))
}
var d, s, length int
offset := 0
// As long as we can read at least 5 bytes...
for s < len(src)-5 {
switch src[s] & 0x03 {
case tagLiteral:
x := uint32(src[s] >> 2)
switch {
case x < 60:
s++
case x == 60:
s += 2
x = uint32(src[s-1])
case x == 61:
s += 3
x = uint32(src[s-2]) | uint32(src[s-1])<<8
case x == 62:
s += 4
x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
case x == 63:
s += 5
x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
}
length = int(x) + 1
if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) {
return decodeErrCodeCorrupt
}
if debug {
fmt.Println("literals, length:", length, "d-after:", d+length)
}
copy(dst[d:], src[s:s+length])
d += length
s += length
continue
case tagCopy1:
s += 2
length = int(src[s-2]) >> 2 & 0x7
toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
if toffset == 0 {
if debug {
fmt.Print("(repeat) ")
}
// keep last offset
switch length {
case 5:
s += 1
length = int(uint32(src[s-1])) + 4
case 6:
s += 2
length = int(uint32(src[s-2])|(uint32(src[s-1])<<8)) + (1 << 8)
case 7:
s += 3
length = int(uint32(src[s-3])|(uint32(src[s-2])<<8)|(uint32(src[s-1])<<16)) + (1 << 16)
default: // 0-> 4
}
} else {
offset = toffset
}
length += 4
case tagCopy2:
s += 3
length = 1 + int(src[s-3])>>2
offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
case tagCopy4:
s += 5
length = 1 + int(src[s-5])>>2
offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
}
if offset <= 0 || d < offset || length > len(dst)-d {
return decodeErrCodeCorrupt
}
if debug {
fmt.Println("copy, length:", length, "offset:", offset, "d-after:", d+length)
}
// Copy from an earlier sub-slice of dst to a later sub-slice.
// If no overlap, use the built-in copy:
if offset > length {
copy(dst[d:d+length], dst[d-offset:])
d += length
continue
}
// Unlike the built-in copy function, this byte-by-byte copy always runs
// forwards, even if the slices overlap. Conceptually, this is:
//
// d += forwardCopy(dst[d:d+length], dst[d-offset:])
//
// We align the slices into a and b and show the compiler they are the same size.
// This allows the loop to run without bounds checks.
a := dst[d : d+length]
b := dst[d-offset:]
b = b[:len(a)]
for i := range a {
a[i] = b[i]
}
d += length
}
// Remaining with extra checks...
for s < len(src) {
switch src[s] & 0x03 {
case tagLiteral:
x := uint32(src[s] >> 2)
switch {
case x < 60:
s++
case x == 60:
s += 2
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
return decodeErrCodeCorrupt
}
x = uint32(src[s-1])
case x == 61:
s += 3
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
return decodeErrCodeCorrupt
}
x = uint32(src[s-2]) | uint32(src[s-1])<<8
case x == 62:
s += 4
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
return decodeErrCodeCorrupt
}
x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
case x == 63:
s += 5
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
return decodeErrCodeCorrupt
}
x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
}
length = int(x) + 1
if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) {
return decodeErrCodeCorrupt
}
if debug {
fmt.Println("literals, length:", length, "d-after:", d+length)
}
copy(dst[d:], src[s:s+length])
d += length
s += length
continue
case tagCopy1:
s += 2
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
return decodeErrCodeCorrupt
}
length = int(src[s-2]) >> 2 & 0x7
toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
if toffset == 0 {
if debug {
fmt.Print("(repeat) ")
}
// keep last offset
switch length {
case 5:
s += 1
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
return decodeErrCodeCorrupt
}
length = int(uint32(src[s-1])) + 4
case 6:
s += 2
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
return decodeErrCodeCorrupt
}
length = int(uint32(src[s-2])|(uint32(src[s-1])<<8)) + (1 << 8)
case 7:
s += 3
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
return decodeErrCodeCorrupt
}
length = int(uint32(src[s-3])|(uint32(src[s-2])<<8)|(uint32(src[s-1])<<16)) + (1 << 16)
default: // 0-> 4
}
} else {
offset = toffset
}
length += 4
case tagCopy2:
s += 3
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
return decodeErrCodeCorrupt
}
length = 1 + int(src[s-3])>>2
offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
case tagCopy4:
s += 5
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
return decodeErrCodeCorrupt
}
length = 1 + int(src[s-5])>>2
offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
}
if offset <= 0 || d < offset || length > len(dst)-d {
return decodeErrCodeCorrupt
}
if debug {
fmt.Println("copy, length:", length, "offset:", offset, "d-after:", d+length)
}
// Copy from an earlier sub-slice of dst to a later sub-slice.
// If no overlap, use the built-in copy:
if offset > length {
copy(dst[d:d+length], dst[d-offset:])
d += length
continue
}
// Unlike the built-in copy function, this byte-by-byte copy always runs
// forwards, even if the slices overlap. Conceptually, this is:
//
// d += forwardCopy(dst[d:d+length], dst[d-offset:])
//
// We align the slices into a and b and show the compiler they are the same size.
// This allows the loop to run without bounds checks.
a := dst[d : d+length]
b := dst[d-offset:]
b = b[:len(a)]
for i := range a {
a[i] = b[i]
}
d += length
}
if d != len(dst) {
return decodeErrCodeCorrupt
}
return 0
}

1172
vendor/github.com/klauspost/compress/s2/encode.go сгенерированный поставляемый Обычный файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

456
vendor/github.com/klauspost/compress/s2/encode_all.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,456 @@
// Copyright 2016 The Snappy-Go Authors. All rights reserved.
// Copyright (c) 2019 Klaus Post. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package s2
import (
"bytes"
"encoding/binary"
"math/bits"
)
func load32(b []byte, i int) uint32 {
return binary.LittleEndian.Uint32(b[i:])
}
func load64(b []byte, i int) uint64 {
return binary.LittleEndian.Uint64(b[i:])
}
// hash6 returns the hash of the lowest 6 bytes of u to fit in a hash table with h bits.
// Preferably h should be a constant and should always be <64.
func hash6(u uint64, h uint8) uint32 {
const prime6bytes = 227718039650203
return uint32(((u << (64 - 48)) * prime6bytes) >> ((64 - h) & 63))
}
func encodeGo(dst, src []byte) []byte {
if n := MaxEncodedLen(len(src)); n < 0 {
panic(ErrTooLarge)
} else if len(dst) < n {
dst = make([]byte, n)
}
// The block starts with the varint-encoded length of the decompressed bytes.
d := binary.PutUvarint(dst, uint64(len(src)))
if len(src) == 0 {
return dst[:d]
}
if len(src) < minNonLiteralBlockSize {
d += emitLiteral(dst[d:], src)
return dst[:d]
}
n := encodeBlockGo(dst[d:], src)
if n > 0 {
d += n
return dst[:d]
}
// Not compressible
d += emitLiteral(dst[d:], src)
return dst[:d]
}
// encodeBlockGo encodes a non-empty src to a guaranteed-large-enough dst. It
// assumes that the varint-encoded length of the decompressed bytes has already
// been written.
//
// It also assumes that:
// len(dst) >= MaxEncodedLen(len(src)) &&
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
func encodeBlockGo(dst, src []byte) (d int) {
// Initialize the hash table.
const (
tableBits = 14
maxTableSize = 1 << tableBits
debug = false
)
var table [maxTableSize]uint32
// sLimit is when to stop looking for offset/length copies. The inputMargin
// lets us use a fast path for emitLiteral in the main loop, while we are
// looking for copies.
sLimit := len(src) - inputMargin
// Bail if we can't compress to at least this.
dstLimit := len(src) - len(src)>>5 - 5
// nextEmit is where in src the next emitLiteral should start from.
nextEmit := 0
// The encoded form must start with a literal, as there are no previous
// bytes to copy, so we start looking for hash matches at s == 1.
s := 1
cv := load64(src, s)
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
repeat := 1
for {
candidate := 0
for {
// Next src position to check
nextS := s + (s-nextEmit)>>6 + 4
if nextS > sLimit {
goto emitRemainder
}
hash0 := hash6(cv, tableBits)
hash1 := hash6(cv>>8, tableBits)
candidate = int(table[hash0])
candidate2 := int(table[hash1])
table[hash0] = uint32(s)
table[hash1] = uint32(s + 1)
hash2 := hash6(cv>>16, tableBits)
// Check repeat at offset checkRep.
const checkRep = 1
if uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) {
base := s + checkRep
// Extend back
for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
i--
base--
}
d += emitLiteral(dst[d:], src[nextEmit:base])
// Extend forward
candidate := s - repeat + 4 + checkRep
s += 4 + checkRep
for s <= sLimit {
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
s += bits.TrailingZeros64(diff) >> 3
break
}
s += 8
candidate += 8
}
if debug {
// Validate match.
if s <= candidate {
panic("s <= candidate")
}
a := src[base:s]
b := src[base-repeat : base-repeat+(s-base)]
if !bytes.Equal(a, b) {
panic("mismatch")
}
}
if nextEmit > 0 {
// same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset.
d += emitRepeat(dst[d:], repeat, s-base)
} else {
// First match, cannot be repeat.
d += emitCopy(dst[d:], repeat, s-base)
}
nextEmit = s
if s >= sLimit {
goto emitRemainder
}
cv = load64(src, s)
continue
}
if uint32(cv) == load32(src, candidate) {
break
}
candidate = int(table[hash2])
if uint32(cv>>8) == load32(src, candidate2) {
table[hash2] = uint32(s + 2)
candidate = candidate2
s++
break
}
table[hash2] = uint32(s + 2)
if uint32(cv>>16) == load32(src, candidate) {
s += 2
break
}
cv = load64(src, nextS)
s = nextS
}
// Extend backwards.
// The top bytes will be rechecked to get the full match.
for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] {
candidate--
s--
}
// Bail if we exceed the maximum size.
if d+(s-nextEmit) > dstLimit {
return 0
}
// A 4-byte match has been found. We'll later see if more than 4 bytes
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
// them as literal bytes.
d += emitLiteral(dst[d:], src[nextEmit:s])
// Call emitCopy, and then see if another emitCopy could be our next
// move. Repeat until we find no match for the input immediately after
// what was consumed by the last emitCopy call.
//
// If we exit this loop normally then we need to call emitLiteral next,
// though we don't yet know how big the literal will be. We handle that
// by proceeding to the next iteration of the main loop. We also can
// exit this loop via goto if we get close to exhausting the input.
for {
// Invariant: we have a 4-byte match at s, and no need to emit any
// literal bytes prior to s.
base := s
repeat = base - candidate
// Extend the 4-byte match as long as possible.
s += 4
candidate += 4
for s <= len(src)-8 {
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
s += bits.TrailingZeros64(diff) >> 3
break
}
s += 8
candidate += 8
}
d += emitCopy(dst[d:], repeat, s-base)
if debug {
// Validate match.
if s <= candidate {
panic("s <= candidate")
}
a := src[base:s]
b := src[base-repeat : base-repeat+(s-base)]
if !bytes.Equal(a, b) {
panic("mismatch")
}
}
nextEmit = s
if s >= sLimit {
goto emitRemainder
}
if d > dstLimit {
// Do we have space for more, if not bail.
return 0
}
// Check for an immediate match, otherwise start search at s+1
x := load64(src, s-2)
m2Hash := hash6(x, tableBits)
currHash := hash6(x>>16, tableBits)
candidate = int(table[currHash])
table[m2Hash] = uint32(s - 2)
table[currHash] = uint32(s)
if debug && s == candidate {
panic("s == candidate")
}
if uint32(x>>16) != load32(src, candidate) {
cv = load64(src, s+1)
s++
break
}
}
}
emitRemainder:
if nextEmit < len(src) {
// Bail if we exceed the maximum size.
if d+len(src)-nextEmit > dstLimit {
return 0
}
d += emitLiteral(dst[d:], src[nextEmit:])
}
return d
}
func encodeBlockSnappyGo(dst, src []byte) (d int) {
// Initialize the hash table.
const (
tableBits = 14
maxTableSize = 1 << tableBits
)
var table [maxTableSize]uint32
// sLimit is when to stop looking for offset/length copies. The inputMargin
// lets us use a fast path for emitLiteral in the main loop, while we are
// looking for copies.
sLimit := len(src) - inputMargin
// Bail if we can't compress to at least this.
dstLimit := len(src) - len(src)>>5 - 5
// nextEmit is where in src the next emitLiteral should start from.
nextEmit := 0
// The encoded form must start with a literal, as there are no previous
// bytes to copy, so we start looking for hash matches at s == 1.
s := 1
cv := load64(src, s)
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
repeat := 1
for {
candidate := 0
for {
// Next src position to check
nextS := s + (s-nextEmit)>>6 + 4
if nextS > sLimit {
goto emitRemainder
}
hash0 := hash6(cv, tableBits)
hash1 := hash6(cv>>8, tableBits)
candidate = int(table[hash0])
candidate2 := int(table[hash1])
table[hash0] = uint32(s)
table[hash1] = uint32(s + 1)
hash2 := hash6(cv>>16, tableBits)
// Check repeat at offset checkRep.
const checkRep = 1
if uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) {
base := s + checkRep
// Extend back
for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
i--
base--
}
d += emitLiteral(dst[d:], src[nextEmit:base])
// Extend forward
candidate := s - repeat + 4 + checkRep
s += 4 + checkRep
for s <= sLimit {
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
s += bits.TrailingZeros64(diff) >> 3
break
}
s += 8
candidate += 8
}
d += emitCopyNoRepeat(dst[d:], repeat, s-base)
nextEmit = s
if s >= sLimit {
goto emitRemainder
}
cv = load64(src, s)
continue
}
if uint32(cv) == load32(src, candidate) {
break
}
candidate = int(table[hash2])
if uint32(cv>>8) == load32(src, candidate2) {
table[hash2] = uint32(s + 2)
candidate = candidate2
s++
break
}
table[hash2] = uint32(s + 2)
if uint32(cv>>16) == load32(src, candidate) {
s += 2
break
}
cv = load64(src, nextS)
s = nextS
}
// Extend backwards
for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] {
candidate--
s--
}
// Bail if we exceed the maximum size.
if d+(s-nextEmit) > dstLimit {
return 0
}
// A 4-byte match has been found. We'll later see if more than 4 bytes
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
// them as literal bytes.
d += emitLiteral(dst[d:], src[nextEmit:s])
// Call emitCopy, and then see if another emitCopy could be our next
// move. Repeat until we find no match for the input immediately after
// what was consumed by the last emitCopy call.
//
// If we exit this loop normally then we need to call emitLiteral next,
// though we don't yet know how big the literal will be. We handle that
// by proceeding to the next iteration of the main loop. We also can
// exit this loop via goto if we get close to exhausting the input.
for {
// Invariant: we have a 4-byte match at s, and no need to emit any
// literal bytes prior to s.
base := s
repeat = base - candidate
// Extend the 4-byte match as long as possible.
s += 4
candidate += 4
for s <= len(src)-8 {
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
s += bits.TrailingZeros64(diff) >> 3
break
}
s += 8
candidate += 8
}
d += emitCopyNoRepeat(dst[d:], repeat, s-base)
if false {
// Validate match.
a := src[base:s]
b := src[base-repeat : base-repeat+(s-base)]
if !bytes.Equal(a, b) {
panic("mismatch")
}
}
nextEmit = s
if s >= sLimit {
goto emitRemainder
}
if d > dstLimit {
// Do we have space for more, if not bail.
return 0
}
// Check for an immediate match, otherwise start search at s+1
x := load64(src, s-2)
m2Hash := hash6(x, tableBits)
currHash := hash6(x>>16, tableBits)
candidate = int(table[currHash])
table[m2Hash] = uint32(s - 2)
table[currHash] = uint32(s)
if uint32(x>>16) != load32(src, candidate) {
cv = load64(src, s+1)
s++
break
}
}
}
emitRemainder:
if nextEmit < len(src) {
// Bail if we exceed the maximum size.
if d+len(src)-nextEmit > dstLimit {
return 0
}
d += emitLiteral(dst[d:], src[nextEmit:])
}
return d
}

142
vendor/github.com/klauspost/compress/s2/encode_amd64.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,142 @@
//go:build !appengine && !noasm && gc
// +build !appengine,!noasm,gc
package s2
// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It
// assumes that the varint-encoded length of the decompressed bytes has already
// been written.
//
// It also assumes that:
// len(dst) >= MaxEncodedLen(len(src)) &&
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
func encodeBlock(dst, src []byte) (d int) {
const (
// Use 12 bit table when less than...
limit12B = 16 << 10
// Use 10 bit table when less than...
limit10B = 4 << 10
// Use 8 bit table when less than...
limit8B = 512
)
if len(src) >= 4<<20 {
return encodeBlockAsm(dst, src)
}
if len(src) >= limit12B {
return encodeBlockAsm4MB(dst, src)
}
if len(src) >= limit10B {
return encodeBlockAsm12B(dst, src)
}
if len(src) >= limit8B {
return encodeBlockAsm10B(dst, src)
}
if len(src) < minNonLiteralBlockSize {
return 0
}
return encodeBlockAsm8B(dst, src)
}
// encodeBlockBetter encodes a non-empty src to a guaranteed-large-enough dst. It
// assumes that the varint-encoded length of the decompressed bytes has already
// been written.
//
// It also assumes that:
// len(dst) >= MaxEncodedLen(len(src)) &&
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
func encodeBlockBetter(dst, src []byte) (d int) {
const (
// Use 12 bit table when less than...
limit12B = 16 << 10
// Use 10 bit table when less than...
limit10B = 4 << 10
// Use 8 bit table when less than...
limit8B = 512
)
if len(src) > 4<<20 {
return encodeBetterBlockAsm(dst, src)
}
if len(src) >= limit12B {
return encodeBetterBlockAsm4MB(dst, src)
}
if len(src) >= limit10B {
return encodeBetterBlockAsm12B(dst, src)
}
if len(src) >= limit8B {
return encodeBetterBlockAsm10B(dst, src)
}
if len(src) < minNonLiteralBlockSize {
return 0
}
return encodeBetterBlockAsm8B(dst, src)
}
// encodeBlockSnappy encodes a non-empty src to a guaranteed-large-enough dst. It
// assumes that the varint-encoded length of the decompressed bytes has already
// been written.
//
// It also assumes that:
// len(dst) >= MaxEncodedLen(len(src)) &&
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
func encodeBlockSnappy(dst, src []byte) (d int) {
const (
// Use 12 bit table when less than...
limit12B = 16 << 10
// Use 10 bit table when less than...
limit10B = 4 << 10
// Use 8 bit table when less than...
limit8B = 512
)
if len(src) >= 64<<10 {
return encodeSnappyBlockAsm(dst, src)
}
if len(src) >= limit12B {
return encodeSnappyBlockAsm64K(dst, src)
}
if len(src) >= limit10B {
return encodeSnappyBlockAsm12B(dst, src)
}
if len(src) >= limit8B {
return encodeSnappyBlockAsm10B(dst, src)
}
if len(src) < minNonLiteralBlockSize {
return 0
}
return encodeSnappyBlockAsm8B(dst, src)
}
// encodeBlockSnappy encodes a non-empty src to a guaranteed-large-enough dst. It
// assumes that the varint-encoded length of the decompressed bytes has already
// been written.
//
// It also assumes that:
// len(dst) >= MaxEncodedLen(len(src)) &&
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
func encodeBlockBetterSnappy(dst, src []byte) (d int) {
const (
// Use 12 bit table when less than...
limit12B = 16 << 10
// Use 10 bit table when less than...
limit10B = 4 << 10
// Use 8 bit table when less than...
limit8B = 512
)
if len(src) >= 64<<10 {
return encodeSnappyBetterBlockAsm(dst, src)
}
if len(src) >= limit12B {
return encodeSnappyBetterBlockAsm64K(dst, src)
}
if len(src) >= limit10B {
return encodeSnappyBetterBlockAsm12B(dst, src)
}
if len(src) >= limit8B {
return encodeSnappyBetterBlockAsm10B(dst, src)
}
if len(src) < minNonLiteralBlockSize {
return 0
}
return encodeSnappyBetterBlockAsm8B(dst, src)
}

604
vendor/github.com/klauspost/compress/s2/encode_best.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,604 @@
// Copyright 2016 The Snappy-Go Authors. All rights reserved.
// Copyright (c) 2019 Klaus Post. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package s2
import (
"fmt"
"math/bits"
)
// encodeBlockBest encodes a non-empty src to a guaranteed-large-enough dst. It
// assumes that the varint-encoded length of the decompressed bytes has already
// been written.
//
// It also assumes that:
// len(dst) >= MaxEncodedLen(len(src)) &&
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
func encodeBlockBest(dst, src []byte) (d int) {
// Initialize the hash tables.
const (
// Long hash matches.
lTableBits = 19
maxLTableSize = 1 << lTableBits
// Short hash matches.
sTableBits = 16
maxSTableSize = 1 << sTableBits
inputMargin = 8 + 2
)
// sLimit is when to stop looking for offset/length copies. The inputMargin
// lets us use a fast path for emitLiteral in the main loop, while we are
// looking for copies.
sLimit := len(src) - inputMargin
if len(src) < minNonLiteralBlockSize {
return 0
}
var lTable [maxLTableSize]uint64
var sTable [maxSTableSize]uint64
// Bail if we can't compress to at least this.
dstLimit := len(src) - 5
// nextEmit is where in src the next emitLiteral should start from.
nextEmit := 0
// The encoded form must start with a literal, as there are no previous
// bytes to copy, so we start looking for hash matches at s == 1.
s := 1
cv := load64(src, s)
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
repeat := 1
const lowbitMask = 0xffffffff
getCur := func(x uint64) int {
return int(x & lowbitMask)
}
getPrev := func(x uint64) int {
return int(x >> 32)
}
const maxSkip = 64
for {
type match struct {
offset int
s int
length int
score int
rep bool
}
var best match
for {
// Next src position to check
nextS := (s-nextEmit)>>8 + 1
if nextS > maxSkip {
nextS = s + maxSkip
} else {
nextS += s
}
if nextS > sLimit {
goto emitRemainder
}
hashL := hash8(cv, lTableBits)
hashS := hash4(cv, sTableBits)
candidateL := lTable[hashL]
candidateS := sTable[hashS]
score := func(m match) int {
// Matches that are longer forward are penalized since we must emit it as a literal.
score := m.length - m.s
if nextEmit == m.s {
// If we do not have to emit literals, we save 1 byte
score++
}
offset := m.s - m.offset
if m.rep {
return score - emitRepeatSize(offset, m.length)
}
return score - emitCopySize(offset, m.length)
}
matchAt := func(offset, s int, first uint32, rep bool) match {
if best.length != 0 && best.s-best.offset == s-offset {
// Don't retest if we have the same offset.
return match{offset: offset, s: s}
}
if load32(src, offset) != first {
return match{offset: offset, s: s}
}
m := match{offset: offset, s: s, length: 4 + offset, rep: rep}
s += 4
for s <= sLimit {
if diff := load64(src, s) ^ load64(src, m.length); diff != 0 {
m.length += bits.TrailingZeros64(diff) >> 3
break
}
s += 8
m.length += 8
}
m.length -= offset
m.score = score(m)
if m.score <= -m.s {
// Eliminate if no savings, we might find a better one.
m.length = 0
}
return m
}
bestOf := func(a, b match) match {
if b.length == 0 {
return a
}
if a.length == 0 {
return b
}
as := a.score + b.s
bs := b.score + a.s
if as >= bs {
return a
}
return b
}
best = bestOf(matchAt(getCur(candidateL), s, uint32(cv), false), matchAt(getPrev(candidateL), s, uint32(cv), false))
best = bestOf(best, matchAt(getCur(candidateS), s, uint32(cv), false))
best = bestOf(best, matchAt(getPrev(candidateS), s, uint32(cv), false))
{
best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8), true))
if best.length > 0 {
// s+1
nextShort := sTable[hash4(cv>>8, sTableBits)]
s := s + 1
cv := load64(src, s)
nextLong := lTable[hash8(cv, lTableBits)]
best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv), false))
best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv), false))
best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv), false))
best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv), false))
// Repeat at + 2
best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8), true))
// s+2
if true {
nextShort = sTable[hash4(cv>>8, sTableBits)]
s++
cv = load64(src, s)
nextLong = lTable[hash8(cv, lTableBits)]
best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv), false))
best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv), false))
best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv), false))
best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv), false))
}
// Search for a match at best match end, see if that is better.
if sAt := best.s + best.length; sAt < sLimit {
sBack := best.s
backL := best.length
// Load initial values
cv = load64(src, sBack)
// Search for mismatch
next := lTable[hash8(load64(src, sAt), lTableBits)]
//next := sTable[hash4(load64(src, sAt), sTableBits)]
if checkAt := getCur(next) - backL; checkAt > 0 {
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
}
if checkAt := getPrev(next) - backL; checkAt > 0 {
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
}
}
}
}
// Update table
lTable[hashL] = uint64(s) | candidateL<<32
sTable[hashS] = uint64(s) | candidateS<<32
if best.length > 0 {
break
}
cv = load64(src, nextS)
s = nextS
}
// Extend backwards, not needed for repeats...
s = best.s
if !best.rep {
for best.offset > 0 && s > nextEmit && src[best.offset-1] == src[s-1] {
best.offset--
best.length++
s--
}
}
if false && best.offset >= s {
panic(fmt.Errorf("t %d >= s %d", best.offset, s))
}
// Bail if we exceed the maximum size.
if d+(s-nextEmit) > dstLimit {
return 0
}
base := s
offset := s - best.offset
s += best.length
if offset > 65535 && s-base <= 5 && !best.rep {
// Bail if the match is equal or worse to the encoding.
s = best.s + 1
if s >= sLimit {
goto emitRemainder
}
cv = load64(src, s)
continue
}
d += emitLiteral(dst[d:], src[nextEmit:base])
if best.rep {
if nextEmit > 0 {
// same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset.
d += emitRepeat(dst[d:], offset, best.length)
} else {
// First match, cannot be repeat.
d += emitCopy(dst[d:], offset, best.length)
}
} else {
d += emitCopy(dst[d:], offset, best.length)
}
repeat = offset
nextEmit = s
if s >= sLimit {
goto emitRemainder
}
if d > dstLimit {
// Do we have space for more, if not bail.
return 0
}
// Fill tables...
for i := best.s + 1; i < s; i++ {
cv0 := load64(src, i)
long0 := hash8(cv0, lTableBits)
short0 := hash4(cv0, sTableBits)
lTable[long0] = uint64(i) | lTable[long0]<<32
sTable[short0] = uint64(i) | sTable[short0]<<32
}
cv = load64(src, s)
}
emitRemainder:
if nextEmit < len(src) {
// Bail if we exceed the maximum size.
if d+len(src)-nextEmit > dstLimit {
return 0
}
d += emitLiteral(dst[d:], src[nextEmit:])
}
return d
}
// encodeBlockBestSnappy encodes a non-empty src to a guaranteed-large-enough dst. It
// assumes that the varint-encoded length of the decompressed bytes has already
// been written.
//
// It also assumes that:
// len(dst) >= MaxEncodedLen(len(src)) &&
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
func encodeBlockBestSnappy(dst, src []byte) (d int) {
// Initialize the hash tables.
const (
// Long hash matches.
lTableBits = 19
maxLTableSize = 1 << lTableBits
// Short hash matches.
sTableBits = 16
maxSTableSize = 1 << sTableBits
inputMargin = 8 + 2
)
// sLimit is when to stop looking for offset/length copies. The inputMargin
// lets us use a fast path for emitLiteral in the main loop, while we are
// looking for copies.
sLimit := len(src) - inputMargin
if len(src) < minNonLiteralBlockSize {
return 0
}
var lTable [maxLTableSize]uint64
var sTable [maxSTableSize]uint64
// Bail if we can't compress to at least this.
dstLimit := len(src) - 5
// nextEmit is where in src the next emitLiteral should start from.
nextEmit := 0
// The encoded form must start with a literal, as there are no previous
// bytes to copy, so we start looking for hash matches at s == 1.
s := 1
cv := load64(src, s)
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
repeat := 1
const lowbitMask = 0xffffffff
getCur := func(x uint64) int {
return int(x & lowbitMask)
}
getPrev := func(x uint64) int {
return int(x >> 32)
}
const maxSkip = 64
for {
type match struct {
offset int
s int
length int
score int
}
var best match
for {
// Next src position to check
nextS := (s-nextEmit)>>8 + 1
if nextS > maxSkip {
nextS = s + maxSkip
} else {
nextS += s
}
if nextS > sLimit {
goto emitRemainder
}
hashL := hash8(cv, lTableBits)
hashS := hash4(cv, sTableBits)
candidateL := lTable[hashL]
candidateS := sTable[hashS]
score := func(m match) int {
// Matches that are longer forward are penalized since we must emit it as a literal.
score := m.length - m.s
if nextEmit == m.s {
// If we do not have to emit literals, we save 1 byte
score++
}
offset := m.s - m.offset
return score - emitCopySize(offset, m.length)
}
matchAt := func(offset, s int, first uint32) match {
if best.length != 0 && best.s-best.offset == s-offset {
// Don't retest if we have the same offset.
return match{offset: offset, s: s}
}
if load32(src, offset) != first {
return match{offset: offset, s: s}
}
m := match{offset: offset, s: s, length: 4 + offset}
s += 4
for s <= sLimit {
if diff := load64(src, s) ^ load64(src, m.length); diff != 0 {
m.length += bits.TrailingZeros64(diff) >> 3
break
}
s += 8
m.length += 8
}
m.length -= offset
m.score = score(m)
if m.score <= -m.s {
// Eliminate if no savings, we might find a better one.
m.length = 0
}
return m
}
bestOf := func(a, b match) match {
if b.length == 0 {
return a
}
if a.length == 0 {
return b
}
as := a.score + b.s
bs := b.score + a.s
if as >= bs {
return a
}
return b
}
best = bestOf(matchAt(getCur(candidateL), s, uint32(cv)), matchAt(getPrev(candidateL), s, uint32(cv)))
best = bestOf(best, matchAt(getCur(candidateS), s, uint32(cv)))
best = bestOf(best, matchAt(getPrev(candidateS), s, uint32(cv)))
{
best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8)))
if best.length > 0 {
// s+1
nextShort := sTable[hash4(cv>>8, sTableBits)]
s := s + 1
cv := load64(src, s)
nextLong := lTable[hash8(cv, lTableBits)]
best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv)))
best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv)))
best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv)))
best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv)))
// Repeat at + 2
best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8)))
// s+2
if true {
nextShort = sTable[hash4(cv>>8, sTableBits)]
s++
cv = load64(src, s)
nextLong = lTable[hash8(cv, lTableBits)]
best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv)))
best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv)))
best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv)))
best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv)))
}
// Search for a match at best match end, see if that is better.
if sAt := best.s + best.length; sAt < sLimit {
sBack := best.s
backL := best.length
// Load initial values
cv = load64(src, sBack)
// Search for mismatch
next := lTable[hash8(load64(src, sAt), lTableBits)]
//next := sTable[hash4(load64(src, sAt), sTableBits)]
if checkAt := getCur(next) - backL; checkAt > 0 {
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv)))
}
if checkAt := getPrev(next) - backL; checkAt > 0 {
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv)))
}
}
}
}
// Update table
lTable[hashL] = uint64(s) | candidateL<<32
sTable[hashS] = uint64(s) | candidateS<<32
if best.length > 0 {
break
}
cv = load64(src, nextS)
s = nextS
}
// Extend backwards, not needed for repeats...
s = best.s
if true {
for best.offset > 0 && s > nextEmit && src[best.offset-1] == src[s-1] {
best.offset--
best.length++
s--
}
}
if false && best.offset >= s {
panic(fmt.Errorf("t %d >= s %d", best.offset, s))
}
// Bail if we exceed the maximum size.
if d+(s-nextEmit) > dstLimit {
return 0
}
base := s
offset := s - best.offset
s += best.length
if offset > 65535 && s-base <= 5 {
// Bail if the match is equal or worse to the encoding.
s = best.s + 1
if s >= sLimit {
goto emitRemainder
}
cv = load64(src, s)
continue
}
d += emitLiteral(dst[d:], src[nextEmit:base])
d += emitCopyNoRepeat(dst[d:], offset, best.length)
repeat = offset
nextEmit = s
if s >= sLimit {
goto emitRemainder
}
if d > dstLimit {
// Do we have space for more, if not bail.
return 0
}
// Fill tables...
for i := best.s + 1; i < s; i++ {
cv0 := load64(src, i)
long0 := hash8(cv0, lTableBits)
short0 := hash4(cv0, sTableBits)
lTable[long0] = uint64(i) | lTable[long0]<<32
sTable[short0] = uint64(i) | sTable[short0]<<32
}
cv = load64(src, s)
}
emitRemainder:
if nextEmit < len(src) {
// Bail if we exceed the maximum size.
if d+len(src)-nextEmit > dstLimit {
return 0
}
d += emitLiteral(dst[d:], src[nextEmit:])
}
return d
}
// emitCopySize returns the size to encode the offset+length
//
// It assumes that:
// 1 <= offset && offset <= math.MaxUint32
// 4 <= length && length <= 1 << 24
func emitCopySize(offset, length int) int {
if offset >= 65536 {
i := 0
if length > 64 {
length -= 64
if length >= 4 {
// Emit remaining as repeats
return 5 + emitRepeatSize(offset, length)
}
i = 5
}
if length == 0 {
return i
}
return i + 5
}
// Offset no more than 2 bytes.
if length > 64 {
// Emit remaining as repeats, at least 4 bytes remain.
return 3 + emitRepeatSize(offset, length-60)
}
if length >= 12 || offset >= 2048 {
return 3
}
// Emit the remaining copy, encoded as 2 bytes.
return 2
}
// emitRepeatSize returns the number of bytes required to encode a repeat.
// Length must be at least 4 and < 1<<24
func emitRepeatSize(offset, length int) int {
// Repeat offset, make length cheaper
if length <= 4+4 || (length < 8+4 && offset < 2048) {
return 2
}
if length < (1<<8)+4+4 {
return 3
}
if length < (1<<16)+(1<<8)+4 {
return 4
}
const maxRepeat = (1 << 24) - 1
length -= (1 << 16) - 4
left := 0
if length > maxRepeat {
left = length - maxRepeat + 4
length = maxRepeat - 4
}
if left > 0 {
return 5 + emitRepeatSize(offset, left)
}
return 5
}

Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше