MM-39315: Bump dependencies (#19039)
Note: We keep splitio/go-client untouched because the dependency is broken. See https://github.com/mattermost/mattermost-server/pull/18604 ```release-note NONE ```
Этот коммит содержится в:
коммит произвёл
GitHub
родитель
fd8fea804b
Коммит
189d447591
65
go.mod
65
go.mod
@@ -6,18 +6,17 @@ require (
|
||||
code.sajari.com/docconv v1.2.0
|
||||
github.com/HdrHistogram/hdrhistogram-go v0.9.0 // indirect
|
||||
github.com/Masterminds/semver/v3 v3.1.1
|
||||
github.com/Masterminds/squirrel v1.5.0
|
||||
github.com/PuerkitoBio/goquery v1.7.1 // indirect
|
||||
github.com/Masterminds/squirrel v1.5.1
|
||||
github.com/PuerkitoBio/goquery v1.8.0 // indirect
|
||||
github.com/advancedlogic/GoOse v0.0.0-20210820140952-9d5822d4a625 // indirect
|
||||
github.com/andybalholm/brotli v1.0.3 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.1 // indirect
|
||||
github.com/andybalholm/brotli v1.0.4 // indirect
|
||||
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect
|
||||
github.com/armon/go-metrics v0.3.9 // indirect
|
||||
github.com/armon/go-metrics v0.3.10 // indirect
|
||||
github.com/avct/uasurfer v0.0.0-20191028135549-26b5daa857f1
|
||||
github.com/aws/aws-sdk-go v1.40.57
|
||||
github.com/aws/aws-sdk-go v1.42.11
|
||||
github.com/bits-and-blooms/bitset v1.2.1 // indirect
|
||||
github.com/blang/semver v3.5.1+incompatible
|
||||
github.com/blevesearch/bleve/v2 v2.2.1
|
||||
github.com/blevesearch/bleve/v2 v2.2.2
|
||||
github.com/cespare/xxhash/v2 v2.1.2
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible
|
||||
github.com/dgryski/dgoogauth v0.0.0-20190221195224-5a805980a5f3
|
||||
@@ -31,19 +30,16 @@ require (
|
||||
github.com/getsentry/sentry-go v0.11.0
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.3 // indirect
|
||||
github.com/go-redis/redis/v8 v8.11.4 // indirect
|
||||
github.com/go-resty/resty/v2 v2.6.0 // indirect
|
||||
github.com/go-resty/resty/v2 v2.7.0 // indirect
|
||||
github.com/go-sql-driver/mysql v1.6.0
|
||||
github.com/golang-migrate/migrate/v4 v4.15.0
|
||||
github.com/golang-migrate/migrate/v4 v4.15.1
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/btree v1.0.1 // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/gopherjs/gopherjs v0.0.0-20211004101933-6b77bd30416d // indirect
|
||||
github.com/gopherjs/gopherjs v0.0.0-20211111143520-d0d5ecc1a356 // indirect
|
||||
github.com/gorilla/handlers v1.5.1
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/gorilla/schema v1.2.0
|
||||
github.com/gorilla/websocket v1.4.2
|
||||
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
|
||||
github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c
|
||||
github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
@@ -54,17 +50,15 @@ require (
|
||||
github.com/hashicorp/go-plugin v1.4.3
|
||||
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.4 // indirect
|
||||
github.com/hashicorp/memberlist v0.2.4
|
||||
github.com/hashicorp/yamux v0.0.0-20210826001029-26ff87cf9493 // indirect
|
||||
github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7
|
||||
github.com/hashicorp/memberlist v0.3.0
|
||||
github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 // indirect
|
||||
github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba
|
||||
github.com/jmoiron/sqlx v1.3.4
|
||||
github.com/jonboulle/clockwork v0.2.2
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.13.6 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
|
||||
github.com/klauspost/pgzip v1.2.5 // indirect
|
||||
github.com/ledongthuc/pdf v0.0.0-20210621053716-e28cb8259002
|
||||
github.com/lib/pq v1.10.3
|
||||
github.com/lib/pq v1.10.4
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattermost/go-i18n v1.11.1-0.20211013152124-5c415071e404
|
||||
github.com/mattermost/gorp v1.6.2-0.20210714143452-8b50f5209a7f
|
||||
@@ -73,15 +67,14 @@ require (
|
||||
github.com/mattermost/ldap v0.0.0-20201202150706-ee0e6284187d
|
||||
github.com/mattermost/logr/v2 v2.0.15
|
||||
github.com/mattermost/rsc v0.0.0-20160330161541-bbaefb05eaa0
|
||||
github.com/mattn/go-colorable v0.1.11 // indirect
|
||||
github.com/mattn/go-colorable v0.1.12 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.13 // indirect
|
||||
github.com/mholt/archiver/v3 v3.5.0
|
||||
github.com/mholt/archiver/v3 v3.5.1
|
||||
github.com/miekg/dns v1.1.43 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/minio/minio-go/v7 v7.0.14
|
||||
github.com/minio/minio-go/v7 v7.0.16
|
||||
github.com/minio/sha256-simd v1.0.0 // indirect
|
||||
github.com/mitchellh/go-testing-interface v1.14.1 // indirect
|
||||
github.com/muesli/smartcrop v0.3.0 // indirect
|
||||
github.com/nwaples/rardecode v1.1.2 // indirect
|
||||
github.com/oklog/run v1.1.0 // indirect
|
||||
github.com/olekukonko/tablewriter v0.0.5 // indirect
|
||||
@@ -91,11 +84,11 @@ require (
|
||||
github.com/otiai10/gosseract/v2 v2.3.1 // indirect
|
||||
github.com/pborman/uuid v1.2.1
|
||||
github.com/pelletier/go-toml v1.9.4 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.8 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.11 // indirect
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/prometheus/client_golang v1.11.0
|
||||
github.com/prometheus/client_model v0.2.0
|
||||
github.com/prometheus/common v0.31.1 // indirect
|
||||
github.com/prometheus/common v0.32.1 // indirect
|
||||
github.com/prometheus/procfs v0.7.3 // indirect
|
||||
github.com/reflog/dateconstraints v0.2.1
|
||||
github.com/rs/cors v1.8.0
|
||||
@@ -109,34 +102,34 @@ require (
|
||||
github.com/stretchr/objx v0.3.0 // indirect
|
||||
github.com/stretchr/testify v1.7.0
|
||||
github.com/throttled/throttled v2.2.5+incompatible
|
||||
github.com/tidwall/gjson v1.9.2 // indirect
|
||||
github.com/tidwall/gjson v1.11.0 // indirect
|
||||
github.com/tinylib/msgp v1.1.6
|
||||
github.com/tylerb/graceful v1.2.15
|
||||
github.com/uber/jaeger-client-go v2.29.1+incompatible
|
||||
github.com/uber/jaeger-lib v2.4.1+incompatible
|
||||
github.com/ulikunitz/xz v0.5.10 // indirect
|
||||
github.com/vmihailenco/msgpack/v5 v5.3.4
|
||||
github.com/vmihailenco/msgpack/v5 v5.3.5
|
||||
github.com/wiggin77/merror v1.0.3
|
||||
github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c
|
||||
github.com/yuin/goldmark v1.4.1
|
||||
github.com/yuin/goldmark v1.4.4
|
||||
go.etcd.io/bbolt v1.3.6 // indirect
|
||||
go.uber.org/atomic v1.9.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
|
||||
golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d
|
||||
golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410
|
||||
golang.org/x/mod v0.5.1 // indirect
|
||||
golang.org/x/net v0.0.0-20211006190231-62292e806868
|
||||
golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
|
||||
golang.org/x/sys v0.0.0-20211006225509-1a26e0398eed // indirect
|
||||
golang.org/x/sys v0.0.0-20211123173158-ef496fb156ab // indirect
|
||||
golang.org/x/text v0.3.7
|
||||
golang.org/x/tools v0.1.7
|
||||
google.golang.org/genproto v0.0.0-20211005153810-c76a74d43a8e // indirect
|
||||
google.golang.org/grpc v1.41.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1 // indirect
|
||||
google.golang.org/grpc v1.42.0 // indirect
|
||||
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
|
||||
gopkg.in/ini.v1 v1.63.2 // indirect
|
||||
gopkg.in/ini.v1 v1.64.0 // indirect
|
||||
gopkg.in/mail.v2 v2.3.1
|
||||
gopkg.in/olivere/elastic.v6 v6.2.37
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
willnorris.com/go/imageproxy v0.10.0
|
||||
willnorris.com/go/imageproxy v0.11.2
|
||||
)
|
||||
|
||||
// Hack to prevent the willf/bitset module from being upgraded to 1.2.0.
|
||||
|
||||
774
go.sum
774
go.sum
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
4
vendor/github.com/Masterminds/squirrel/README.md
сгенерированный
поставляемый
4
vendor/github.com/Masterminds/squirrel/README.md
сгенерированный
поставляемый
@@ -1,4 +1,4 @@
|
||||
[](https://www.repostatus.org/#inactive)
|
||||
[](https://masterminds.github.io/stability/maintenance.html)
|
||||
### Squirrel is "complete".
|
||||
Bug fixes will still be merged (slowly). Bug reports are welcome, but I will not necessarily respond to them. If another fork (or substantially similar project) actively improves on what Squirrel does, let me know and I may link to it here.
|
||||
|
||||
@@ -64,7 +64,7 @@ Squirrel wants to make your life easier:
|
||||
|
||||
```go
|
||||
// StmtCache caches Prepared Stmts for you
|
||||
dbCache := sq.NewStmtCacher(db)
|
||||
dbCache := sq.NewStmtCache(db)
|
||||
|
||||
// StatementBuilder keeps your syntax neat
|
||||
mydb := sq.StatementBuilder.RunWith(dbCache)
|
||||
|
||||
12
vendor/github.com/Masterminds/squirrel/case.go
сгенерированный
поставляемый
12
vendor/github.com/Masterminds/squirrel/case.go
сгенерированный
поставляемый
@@ -27,7 +27,7 @@ func (b *sqlizerBuffer) WriteSql(item Sqlizer) {
|
||||
|
||||
var str string
|
||||
var args []interface{}
|
||||
str, args, b.err = item.ToSql()
|
||||
str, args, b.err = nestedToSql(item)
|
||||
|
||||
if b.err != nil {
|
||||
return
|
||||
@@ -100,6 +100,16 @@ func (b CaseBuilder) ToSql() (string, []interface{}, error) {
|
||||
return data.ToSql()
|
||||
}
|
||||
|
||||
// MustSql builds the query into a SQL string and bound args.
|
||||
// It panics if there are any errors.
|
||||
func (b CaseBuilder) MustSql() (string, []interface{}) {
|
||||
sql, args, err := b.ToSql()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return sql, args
|
||||
}
|
||||
|
||||
// what sets optional value for CASE construct "CASE [value] ..."
|
||||
func (b CaseBuilder) what(expr interface{}) CaseBuilder {
|
||||
return builder.Set(b, "What", newPart(expr)).(CaseBuilder)
|
||||
|
||||
10
vendor/github.com/Masterminds/squirrel/delete.go
сгенерированный
поставляемый
10
vendor/github.com/Masterminds/squirrel/delete.go
сгенерированный
поставляемый
@@ -121,6 +121,16 @@ func (b DeleteBuilder) ToSql() (string, []interface{}, error) {
|
||||
return data.ToSql()
|
||||
}
|
||||
|
||||
// MustSql builds the query into a SQL string and bound args.
|
||||
// It panics if there are any errors.
|
||||
func (b DeleteBuilder) MustSql() (string, []interface{}) {
|
||||
sql, args, err := b.ToSql()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return sql, args
|
||||
}
|
||||
|
||||
// Prefix adds an expression to the beginning of the query
|
||||
func (b DeleteBuilder) Prefix(sql string, args ...interface{}) DeleteBuilder {
|
||||
return b.PrefixExpr(Expr(sql, args...))
|
||||
|
||||
10
vendor/github.com/Masterminds/squirrel/insert.go
сгенерированный
поставляемый
10
vendor/github.com/Masterminds/squirrel/insert.go
сгенерированный
поставляемый
@@ -216,6 +216,16 @@ func (b InsertBuilder) ToSql() (string, []interface{}, error) {
|
||||
return data.ToSql()
|
||||
}
|
||||
|
||||
// MustSql builds the query into a SQL string and bound args.
|
||||
// It panics if there are any errors.
|
||||
func (b InsertBuilder) MustSql() (string, []interface{}) {
|
||||
sql, args, err := b.ToSql()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return sql, args
|
||||
}
|
||||
|
||||
// Prefix adds an expression to the beginning of the query
|
||||
func (b InsertBuilder) Prefix(sql string, args ...interface{}) InsertBuilder {
|
||||
return b.PrefixExpr(Expr(sql, args...))
|
||||
|
||||
10
vendor/github.com/Masterminds/squirrel/part.go
сгенерированный
поставляемый
10
vendor/github.com/Masterminds/squirrel/part.go
сгенерированный
поставляемый
@@ -29,9 +29,17 @@ func (p part) ToSql() (sql string, args []interface{}, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
func nestedToSql(s Sqlizer) (string, []interface{}, error) {
|
||||
if raw, ok := s.(rawSqlizer); ok {
|
||||
return raw.toSqlRaw()
|
||||
} else {
|
||||
return s.ToSql()
|
||||
}
|
||||
}
|
||||
|
||||
func appendToSql(parts []Sqlizer, w io.Writer, sep string, args []interface{}) ([]interface{}, error) {
|
||||
for i, p := range parts {
|
||||
partSql, partArgs, err := p.ToSql()
|
||||
partSql, partArgs, err := nestedToSql(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
} else if len(partSql) == 0 {
|
||||
|
||||
18
vendor/github.com/Masterminds/squirrel/select.go
сгенерированный
поставляемый
18
vendor/github.com/Masterminds/squirrel/select.go
сгенерированный
поставляемый
@@ -52,7 +52,7 @@ func (d *selectData) QueryRow() RowScanner {
|
||||
}
|
||||
|
||||
func (d *selectData) ToSql() (sqlStr string, args []interface{}, err error) {
|
||||
sqlStr, args, err = d.toSql()
|
||||
sqlStr, args, err = d.toSqlRaw()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -62,10 +62,6 @@ func (d *selectData) ToSql() (sqlStr string, args []interface{}, err error) {
|
||||
}
|
||||
|
||||
func (d *selectData) toSqlRaw() (sqlStr string, args []interface{}, err error) {
|
||||
return d.toSql()
|
||||
}
|
||||
|
||||
func (d *selectData) toSql() (sqlStr string, args []interface{}, err error) {
|
||||
if len(d.Columns) == 0 {
|
||||
err = fmt.Errorf("select statements must have at least one result column")
|
||||
return
|
||||
@@ -222,6 +218,13 @@ func (b SelectBuilder) ToSql() (string, []interface{}, error) {
|
||||
return data.ToSql()
|
||||
}
|
||||
|
||||
func (b SelectBuilder) toSqlRaw() (string, []interface{}, error) {
|
||||
data := builder.GetStruct(b).(selectData)
|
||||
return data.toSqlRaw()
|
||||
}
|
||||
|
||||
// MustSql builds the query into a SQL string and bound args.
|
||||
// It panics if there are any errors.
|
||||
func (b SelectBuilder) MustSql() (string, []interface{}) {
|
||||
sql, args, err := b.ToSql()
|
||||
if err != nil {
|
||||
@@ -230,11 +233,6 @@ func (b SelectBuilder) MustSql() (string, []interface{}) {
|
||||
return sql, args
|
||||
}
|
||||
|
||||
func (b SelectBuilder) toSqlRaw() (string, []interface{}, error) {
|
||||
data := builder.GetStruct(b).(selectData)
|
||||
return data.toSqlRaw()
|
||||
}
|
||||
|
||||
// Prefix adds an expression to the beginning of the query
|
||||
func (b SelectBuilder) Prefix(sql string, args ...interface{}) SelectBuilder {
|
||||
return b.PrefixExpr(Expr(sql, args...))
|
||||
|
||||
10
vendor/github.com/Masterminds/squirrel/update.go
сгенерированный
поставляемый
10
vendor/github.com/Masterminds/squirrel/update.go
сгенерированный
поставляемый
@@ -187,6 +187,16 @@ func (b UpdateBuilder) ToSql() (string, []interface{}, error) {
|
||||
return data.ToSql()
|
||||
}
|
||||
|
||||
// MustSql builds the query into a SQL string and bound args.
|
||||
// It panics if there are any errors.
|
||||
func (b UpdateBuilder) MustSql() (string, []interface{}) {
|
||||
sql, args, err := b.ToSql()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return sql, args
|
||||
}
|
||||
|
||||
// Prefix adds an expression to the beginning of the query
|
||||
func (b UpdateBuilder) Prefix(sql string, args ...interface{}) UpdateBuilder {
|
||||
return b.PrefixExpr(Expr(sql, args...))
|
||||
|
||||
31
vendor/github.com/PuerkitoBio/goquery/.travis.yml
сгенерированный
поставляемый
31
vendor/github.com/PuerkitoBio/goquery/.travis.yml
сгенерированный
поставляемый
@@ -1,31 +0,0 @@
|
||||
arch:
|
||||
- amd64
|
||||
- ppc64le
|
||||
language: go
|
||||
|
||||
go:
|
||||
- 1.7.x
|
||||
- 1.8.x
|
||||
- 1.9.x
|
||||
- 1.10.x
|
||||
- 1.11.x
|
||||
- 1.12.x
|
||||
- 1.13.x
|
||||
- 1.14.x
|
||||
- 1.15.x
|
||||
- tip
|
||||
|
||||
jobs:
|
||||
exclude:
|
||||
- arch: ppc64le
|
||||
go: 1.7.x
|
||||
- arch: ppc64le
|
||||
go: 1.8.x
|
||||
- arch: ppc64le
|
||||
go: 1.9.x
|
||||
- arch: ppc64le
|
||||
go: 1.10.x
|
||||
- arch: ppc64le
|
||||
go: 1.11.x
|
||||
- arch: ppc64le
|
||||
go: 1.12.x
|
||||
4
vendor/github.com/PuerkitoBio/goquery/README.md
сгенерированный
поставляемый
4
vendor/github.com/PuerkitoBio/goquery/README.md
сгенерированный
поставляемый
@@ -1,7 +1,6 @@
|
||||
# goquery - a little like that j-thing, only in Go
|
||||
|
||||
[](https://builds.sr.ht/~mna/goquery/commits/fedora.yml?)
|
||||
[](http://travis-ci.org/PuerkitoBio/goquery)
|
||||
[](https://github.com/PuerkitoBio/goquery/actions)
|
||||
[](https://pkg.go.dev/github.com/PuerkitoBio/goquery)
|
||||
[](https://sourcegraph.com/github.com/PuerkitoBio/goquery?badge)
|
||||
|
||||
@@ -41,6 +40,7 @@ Please note that because of the net/html dependency, goquery requires Go1.1+ and
|
||||
|
||||
**Note that goquery's API is now stable, and will not break.**
|
||||
|
||||
* **2021-10-25 (v1.8.0)** : Add `Render` function to render a `Selection` to an `io.Writer` (thanks [@anthonygedeon](https://github.com/anthonygedeon)).
|
||||
* **2021-07-11 (v1.7.1)** : Update go.mod dependencies and add dependabot config (thanks [@jauderho](https://github.com/jauderho)).
|
||||
* **2021-06-14 (v1.7.0)** : Add `Single` and `SingleMatcher` functions to optimize first-match selection (thanks [@gdollardollar](https://github.com/gdollardollar)).
|
||||
* **2021-01-11 (v1.6.1)** : Fix panic when calling `{Prepend,Append,Set}Html` on a `Selection` that contains non-Element nodes.
|
||||
|
||||
4
vendor/github.com/PuerkitoBio/goquery/go.mod
сгенерированный
поставляемый
4
vendor/github.com/PuerkitoBio/goquery/go.mod
сгенерированный
поставляемый
@@ -1,8 +1,8 @@
|
||||
module github.com/PuerkitoBio/goquery
|
||||
|
||||
require (
|
||||
github.com/andybalholm/cascadia v1.2.0
|
||||
golang.org/x/net v0.0.0-20210614182718-04defd469f4e
|
||||
github.com/andybalholm/cascadia v1.3.1
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8
|
||||
)
|
||||
|
||||
go 1.13
|
||||
|
||||
9
vendor/github.com/PuerkitoBio/goquery/go.sum
сгенерированный
поставляемый
9
vendor/github.com/PuerkitoBio/goquery/go.sum
сгенерированный
поставляемый
@@ -1,8 +1,7 @@
|
||||
github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE=
|
||||
github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
|
||||
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
|
||||
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
|
||||
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk=
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
|
||||
21
vendor/github.com/PuerkitoBio/goquery/utilities.go
сгенерированный
поставляемый
21
vendor/github.com/PuerkitoBio/goquery/utilities.go
сгенерированный
поставляемый
@@ -2,6 +2,7 @@ package goquery
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
@@ -50,13 +51,24 @@ func nodeName(node *html.Node) string {
|
||||
case html.ElementNode, html.DoctypeNode:
|
||||
return node.Data
|
||||
default:
|
||||
if node.Type >= 0 && int(node.Type) < len(nodeNames) {
|
||||
if int(node.Type) < len(nodeNames) {
|
||||
return nodeNames[node.Type]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// Render renders the html of the first element from selector and writes it to
|
||||
// the writer. It behaves the same as OuterHtml but writes to w instead of
|
||||
// returning the string.
|
||||
func Render(w io.Writer, s *Selection) error {
|
||||
if s.Length() == 0 {
|
||||
return nil
|
||||
}
|
||||
n := s.Get(0)
|
||||
return html.Render(w, n)
|
||||
}
|
||||
|
||||
// OuterHtml returns the outer HTML rendering of the first item in
|
||||
// the selection - that is, the HTML including the first element's
|
||||
// tag and attributes.
|
||||
@@ -66,12 +78,7 @@ func nodeName(node *html.Node) string {
|
||||
// a property provided by the DOM).
|
||||
func OuterHtml(s *Selection) (string, error) {
|
||||
var buf bytes.Buffer
|
||||
|
||||
if s.Length() == 0 {
|
||||
return "", nil
|
||||
}
|
||||
n := s.Get(0)
|
||||
if err := html.Render(&buf, n); err != nil {
|
||||
if err := Render(&buf, s); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return buf.String(), nil
|
||||
|
||||
162
vendor/github.com/andybalholm/brotli/cluster_command.go
сгенерированный
поставляемый
162
vendor/github.com/andybalholm/brotli/cluster_command.go
сгенерированный
поставляемый
@@ -1,7 +1,5 @@
|
||||
package brotli
|
||||
|
||||
import "math"
|
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Distributed under MIT license.
|
||||
@@ -164,163 +162,3 @@ func histogramBitCostDistanceCommand(histogram *histogramCommand, candidate *his
|
||||
return populationCostCommand(&tmp) - candidate.bit_cost_
|
||||
}
|
||||
}
|
||||
|
||||
/* Find the best 'out' histogram for each of the 'in' histograms.
|
||||
When called, clusters[0..num_clusters) contains the unique values from
|
||||
symbols[0..in_size), but this property is not preserved in this function.
|
||||
Note: we assume that out[]->bit_cost_ is already up-to-date. */
|
||||
func histogramRemapCommand(in []histogramCommand, in_size uint, clusters []uint32, num_clusters uint, out []histogramCommand, symbols []uint32) {
|
||||
var i uint
|
||||
for i = 0; i < in_size; i++ {
|
||||
var best_out uint32
|
||||
if i == 0 {
|
||||
best_out = symbols[0]
|
||||
} else {
|
||||
best_out = symbols[i-1]
|
||||
}
|
||||
var best_bits float64 = histogramBitCostDistanceCommand(&in[i], &out[best_out])
|
||||
var j uint
|
||||
for j = 0; j < num_clusters; j++ {
|
||||
var cur_bits float64 = histogramBitCostDistanceCommand(&in[i], &out[clusters[j]])
|
||||
if cur_bits < best_bits {
|
||||
best_bits = cur_bits
|
||||
best_out = clusters[j]
|
||||
}
|
||||
}
|
||||
|
||||
symbols[i] = best_out
|
||||
}
|
||||
|
||||
/* Recompute each out based on raw and symbols. */
|
||||
for i = 0; i < num_clusters; i++ {
|
||||
histogramClearCommand(&out[clusters[i]])
|
||||
}
|
||||
|
||||
for i = 0; i < in_size; i++ {
|
||||
histogramAddHistogramCommand(&out[symbols[i]], &in[i])
|
||||
}
|
||||
}
|
||||
|
||||
/* Reorders elements of the out[0..length) array and changes values in
|
||||
symbols[0..length) array in the following way:
|
||||
* when called, symbols[] contains indexes into out[], and has N unique
|
||||
values (possibly N < length)
|
||||
* on return, symbols'[i] = f(symbols[i]) and
|
||||
out'[symbols'[i]] = out[symbols[i]], for each 0 <= i < length,
|
||||
where f is a bijection between the range of symbols[] and [0..N), and
|
||||
the first occurrences of values in symbols'[i] come in consecutive
|
||||
increasing order.
|
||||
Returns N, the number of unique values in symbols[]. */
|
||||
|
||||
var histogramReindexCommand_kInvalidIndex uint32 = math.MaxUint32
|
||||
|
||||
func histogramReindexCommand(out []histogramCommand, symbols []uint32, length uint) uint {
|
||||
var new_index []uint32 = make([]uint32, length)
|
||||
var next_index uint32
|
||||
var tmp []histogramCommand
|
||||
var i uint
|
||||
for i = 0; i < length; i++ {
|
||||
new_index[i] = histogramReindexCommand_kInvalidIndex
|
||||
}
|
||||
|
||||
next_index = 0
|
||||
for i = 0; i < length; i++ {
|
||||
if new_index[symbols[i]] == histogramReindexCommand_kInvalidIndex {
|
||||
new_index[symbols[i]] = next_index
|
||||
next_index++
|
||||
}
|
||||
}
|
||||
|
||||
/* TODO: by using idea of "cycle-sort" we can avoid allocation of
|
||||
tmp and reduce the number of copying by the factor of 2. */
|
||||
tmp = make([]histogramCommand, next_index)
|
||||
|
||||
next_index = 0
|
||||
for i = 0; i < length; i++ {
|
||||
if new_index[symbols[i]] == next_index {
|
||||
tmp[next_index] = out[symbols[i]]
|
||||
next_index++
|
||||
}
|
||||
|
||||
symbols[i] = new_index[symbols[i]]
|
||||
}
|
||||
|
||||
new_index = nil
|
||||
for i = 0; uint32(i) < next_index; i++ {
|
||||
out[i] = tmp[i]
|
||||
}
|
||||
|
||||
tmp = nil
|
||||
return uint(next_index)
|
||||
}
|
||||
|
||||
func clusterHistogramsCommand(in []histogramCommand, in_size uint, max_histograms uint, out []histogramCommand, out_size *uint, histogram_symbols []uint32) {
|
||||
var cluster_size []uint32 = make([]uint32, in_size)
|
||||
var clusters []uint32 = make([]uint32, in_size)
|
||||
var num_clusters uint = 0
|
||||
var max_input_histograms uint = 64
|
||||
var pairs_capacity uint = max_input_histograms * max_input_histograms / 2
|
||||
var pairs []histogramPair = make([]histogramPair, (pairs_capacity + 1))
|
||||
var i uint
|
||||
|
||||
/* For the first pass of clustering, we allow all pairs. */
|
||||
for i = 0; i < in_size; i++ {
|
||||
cluster_size[i] = 1
|
||||
}
|
||||
|
||||
for i = 0; i < in_size; i++ {
|
||||
out[i] = in[i]
|
||||
out[i].bit_cost_ = populationCostCommand(&in[i])
|
||||
histogram_symbols[i] = uint32(i)
|
||||
}
|
||||
|
||||
for i = 0; i < in_size; i += max_input_histograms {
|
||||
var num_to_combine uint = brotli_min_size_t(in_size-i, max_input_histograms)
|
||||
var num_new_clusters uint
|
||||
var j uint
|
||||
for j = 0; j < num_to_combine; j++ {
|
||||
clusters[num_clusters+j] = uint32(i + j)
|
||||
}
|
||||
|
||||
num_new_clusters = histogramCombineCommand(out, cluster_size, histogram_symbols[i:], clusters[num_clusters:], pairs, num_to_combine, num_to_combine, max_histograms, pairs_capacity)
|
||||
num_clusters += num_new_clusters
|
||||
}
|
||||
{
|
||||
/* For the second pass, we limit the total number of histogram pairs.
|
||||
After this limit is reached, we only keep searching for the best pair. */
|
||||
var max_num_pairs uint = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
|
||||
if pairs_capacity < (max_num_pairs + 1) {
|
||||
var _new_size uint
|
||||
if pairs_capacity == 0 {
|
||||
_new_size = max_num_pairs + 1
|
||||
} else {
|
||||
_new_size = pairs_capacity
|
||||
}
|
||||
var new_array []histogramPair
|
||||
for _new_size < (max_num_pairs + 1) {
|
||||
_new_size *= 2
|
||||
}
|
||||
new_array = make([]histogramPair, _new_size)
|
||||
if pairs_capacity != 0 {
|
||||
copy(new_array, pairs[:pairs_capacity])
|
||||
}
|
||||
|
||||
pairs = new_array
|
||||
pairs_capacity = _new_size
|
||||
}
|
||||
|
||||
/* Collapse similar histograms. */
|
||||
num_clusters = histogramCombineCommand(out, cluster_size, histogram_symbols, clusters, pairs, num_clusters, in_size, max_histograms, max_num_pairs)
|
||||
}
|
||||
|
||||
pairs = nil
|
||||
cluster_size = nil
|
||||
|
||||
/* Find the optimal map from original histograms to the final ones. */
|
||||
histogramRemapCommand(in, in_size, clusters, num_clusters, out, histogram_symbols)
|
||||
|
||||
clusters = nil
|
||||
|
||||
/* Convert the context map to a canonical form. */
|
||||
*out_size = histogramReindexCommand(out, histogram_symbols, in_size)
|
||||
}
|
||||
|
||||
2
vendor/github.com/andybalholm/brotli/compress_fragment.go
сгенерированный
поставляемый
2
vendor/github.com/andybalholm/brotli/compress_fragment.go
сгенерированный
поставляемый
@@ -604,7 +604,7 @@ emit_commands:
|
||||
assert(candidate < ip)
|
||||
|
||||
table[hash] = int(ip - base_ip)
|
||||
if !(!isMatch5(in[ip:], in[candidate:])) {
|
||||
if isMatch5(in[ip:], in[candidate:]) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
60
vendor/github.com/andybalholm/brotli/decode.go
сгенерированный
поставляемый
60
vendor/github.com/andybalholm/brotli/decode.go
сгенерированный
поставляемый
@@ -50,21 +50,6 @@ const (
|
||||
decoderErrorUnreachable = -31
|
||||
)
|
||||
|
||||
/**
|
||||
* The value of the last error code, negative integer.
|
||||
*
|
||||
* All other error code values are in the range from ::lastErrorCode
|
||||
* to @c -1. There are also 4 other possible non-error codes @c 0 .. @c 3 in
|
||||
* ::BrotliDecoderErrorCode enumeration.
|
||||
*/
|
||||
const lastErrorCode = decoderErrorUnreachable
|
||||
|
||||
/** Options to be used with ::BrotliDecoderSetParameter. */
|
||||
const (
|
||||
decoderParamDisableRingBufferReallocation = 0
|
||||
decoderParamLargeWindow = 1
|
||||
)
|
||||
|
||||
const huffmanTableBits = 8
|
||||
|
||||
const huffmanTableMask = 0xFF
|
||||
@@ -81,28 +66,6 @@ var kCodeLengthPrefixLength = [16]byte{2, 2, 2, 3, 2, 2, 2, 4, 2, 2, 2, 3, 2, 2,
|
||||
|
||||
var kCodeLengthPrefixValue = [16]byte{0, 4, 3, 2, 0, 4, 3, 1, 0, 4, 3, 2, 0, 4, 3, 5}
|
||||
|
||||
func decoderSetParameter(state *Reader, p int, value uint32) bool {
|
||||
if state.state != stateUninited {
|
||||
return false
|
||||
}
|
||||
switch p {
|
||||
case decoderParamDisableRingBufferReallocation:
|
||||
if !(value == 0) {
|
||||
state.canny_ringbuffer_allocation = 0
|
||||
} else {
|
||||
state.canny_ringbuffer_allocation = 1
|
||||
}
|
||||
return true
|
||||
|
||||
case decoderParamLargeWindow:
|
||||
state.large_window = (!(value == 0))
|
||||
return true
|
||||
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/* Saves error code and converts it to BrotliDecoderResult. */
|
||||
func saveErrorCode(s *Reader, e int) int {
|
||||
s.error_code = int(e)
|
||||
@@ -1125,10 +1088,8 @@ func decodeContextMap(context_map_size uint32, num_htrees *uint32, context_map_a
|
||||
Reads 3..54 bits. */
|
||||
func decodeBlockTypeAndLength(safe int, s *Reader, tree_type int) bool {
|
||||
var max_block_type uint32 = s.num_block_types[tree_type]
|
||||
var type_tree []huffmanCode
|
||||
type_tree = s.block_type_trees[tree_type*huffmanMaxSize258:]
|
||||
var len_tree []huffmanCode
|
||||
len_tree = s.block_len_trees[tree_type*huffmanMaxSize26:]
|
||||
type_tree := s.block_type_trees[tree_type*huffmanMaxSize258:]
|
||||
len_tree := s.block_len_trees[tree_type*huffmanMaxSize26:]
|
||||
var br *bitReader = &s.br
|
||||
var ringbuffer []uint32 = s.block_type_rb[tree_type*2:]
|
||||
var block_type uint32
|
||||
@@ -1280,8 +1241,7 @@ func unwrittenBytes(s *Reader, wrap bool) uint {
|
||||
Returns BROTLI_DECODER_NEEDS_MORE_OUTPUT only if there is more output to push
|
||||
and either ring-buffer is as big as window size, or |force| is true. */
|
||||
func writeRingBuffer(s *Reader, available_out *uint, next_out *[]byte, total_out *uint, force bool) int {
|
||||
var start []byte
|
||||
start = s.ringbuffer[s.partial_pos_out&uint(s.ringbuffer_mask):]
|
||||
start := s.ringbuffer[s.partial_pos_out&uint(s.ringbuffer_mask):]
|
||||
var to_write uint = unwrittenBytes(s, true)
|
||||
var num_written uint = *available_out
|
||||
if num_written > to_write {
|
||||
@@ -1412,8 +1372,7 @@ func copyUncompressedBlockToOutput(available_out *uint, next_out *[]byte, total_
|
||||
|
||||
case stateUncompressedWrite:
|
||||
{
|
||||
var result int
|
||||
result = writeRingBuffer(s, available_out, next_out, total_out, false)
|
||||
result := writeRingBuffer(s, available_out, next_out, total_out, false)
|
||||
if result != decoderSuccess {
|
||||
return result
|
||||
}
|
||||
@@ -1931,8 +1890,7 @@ CommandPostDecodeLiterals:
|
||||
}
|
||||
|
||||
if transform_idx < int(trans.num_transforms) {
|
||||
var word []byte
|
||||
word = words.data[offset:]
|
||||
word := words.data[offset:]
|
||||
var len int = i
|
||||
if transform_idx == int(trans.cutOffTransforms[0]) {
|
||||
copy(s.ringbuffer[pos:], word[:uint(len)])
|
||||
@@ -1954,10 +1912,8 @@ CommandPostDecodeLiterals:
|
||||
}
|
||||
} else {
|
||||
var src_start int = (pos - s.distance_code) & s.ringbuffer_mask
|
||||
var copy_dst []byte
|
||||
copy_dst = s.ringbuffer[pos:]
|
||||
var copy_src []byte
|
||||
copy_src = s.ringbuffer[src_start:]
|
||||
copy_dst := s.ringbuffer[pos:]
|
||||
copy_src := s.ringbuffer[src_start:]
|
||||
var dst_end int = pos + i
|
||||
var src_end int = src_start + i
|
||||
|
||||
@@ -2494,8 +2450,6 @@ func decoderDecompressStream(s *Reader, available_in *uint, next_in *[]byte, ava
|
||||
} else {
|
||||
s.state = stateCommandBegin
|
||||
}
|
||||
|
||||
break
|
||||
} else if s.state == stateCommandPostWrite2 {
|
||||
s.state = stateCommandPostWrapCopy /* BROTLI_STATE_COMMAND_INNER_WRITE */
|
||||
} else {
|
||||
|
||||
3
vendor/github.com/andybalholm/brotli/encode.go
сгенерированный
поставляемый
3
vendor/github.com/andybalholm/brotli/encode.go
сгенерированный
поставляемый
@@ -920,8 +920,7 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
|
||||
REQUIRED: |header| should be 8-byte aligned and at least 16 bytes long.
|
||||
REQUIRED: |block_size| <= (1 << 24). */
|
||||
func writeMetadataHeader(s *Writer, block_size uint, header []byte) uint {
|
||||
var storage_ix uint
|
||||
storage_ix = uint(s.last_bytes_bits_)
|
||||
storage_ix := uint(s.last_bytes_bits_)
|
||||
header[0] = byte(s.last_bytes_)
|
||||
header[1] = byte(s.last_bytes_ >> 8)
|
||||
s.last_bytes_ = 0
|
||||
|
||||
16
vendor/github.com/andybalholm/brotli/fast_log.go
сгенерированный
поставляемый
16
vendor/github.com/andybalholm/brotli/fast_log.go
сгенерированный
поставляемый
@@ -1,6 +1,9 @@
|
||||
package brotli
|
||||
|
||||
import "math"
|
||||
import (
|
||||
"math"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
@@ -11,16 +14,7 @@ import "math"
|
||||
/* Utilities for fast computation of logarithms. */
|
||||
|
||||
func log2FloorNonZero(n uint) uint32 {
|
||||
/* TODO: generalize and move to platform.h */
|
||||
var result uint32 = 0
|
||||
for {
|
||||
n >>= 1
|
||||
if n == 0 {
|
||||
break
|
||||
}
|
||||
result++
|
||||
}
|
||||
return result
|
||||
return uint32(bits.Len(n)) - 1
|
||||
}
|
||||
|
||||
/* A lookup table for small values of log2(int) to be used in entropy
|
||||
|
||||
2
vendor/github.com/andybalholm/brotli/hash.go
сгенерированный
поставляемый
2
vendor/github.com/andybalholm/brotli/hash.go
сгенерированный
поставляемый
@@ -29,8 +29,6 @@ type hasherHandle interface {
|
||||
Store(data []byte, mask uint, ix uint)
|
||||
}
|
||||
|
||||
type score_t uint
|
||||
|
||||
const kCutoffTransformsCount uint32 = 10
|
||||
|
||||
/* 0, 12, 27, 23, 42, 63, 56, 48, 59, 64 */
|
||||
|
||||
3
vendor/github.com/andybalholm/brotli/hash_forgetful_chain.go
сгенерированный
поставляемый
3
vendor/github.com/andybalholm/brotli/hash_forgetful_chain.go
сгенерированный
поставляемый
@@ -110,8 +110,7 @@ func (h *hashForgetfulChain) Prepare(one_shot bool, input_size uint, data []byte
|
||||
func (h *hashForgetfulChain) Store(data []byte, mask uint, ix uint) {
|
||||
var key uint = h.HashBytes(data[ix&mask:])
|
||||
var bank uint = key & (h.numBanks - 1)
|
||||
var idx uint
|
||||
idx = uint(h.free_slot_idx[bank]) & ((1 << h.bankBits) - 1)
|
||||
idx := uint(h.free_slot_idx[bank]) & ((1 << h.bankBits) - 1)
|
||||
h.free_slot_idx[bank]++
|
||||
var delta uint = ix - uint(h.addr[key])
|
||||
h.tiny_hash[uint16(ix)] = byte(key)
|
||||
|
||||
1
vendor/github.com/andybalholm/brotli/hash_rolling.go
сгенерированный
поставляемый
1
vendor/github.com/andybalholm/brotli/hash_rolling.go
сгенерированный
поставляемый
@@ -48,7 +48,6 @@ type hashRolling struct {
|
||||
state uint32
|
||||
table []uint32
|
||||
next_ix uint
|
||||
chunk_len uint32
|
||||
factor uint32
|
||||
factor_remove uint32
|
||||
}
|
||||
|
||||
4
vendor/github.com/andybalholm/brotli/http.go
сгенерированный
поставляемый
4
vendor/github.com/andybalholm/brotli/http.go
сгенерированный
поставляемый
@@ -180,8 +180,8 @@ func init() {
|
||||
var t octetType
|
||||
isCtl := c <= 31 || c == 127
|
||||
isChar := 0 <= c && c <= 127
|
||||
isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0
|
||||
if strings.IndexRune(" \t\r\n", rune(c)) >= 0 {
|
||||
isSeparator := strings.ContainsRune(" \t\"(),/:;<=>?@[]\\{}", rune(c))
|
||||
if strings.ContainsRune(" \t\r\n", rune(c)) {
|
||||
t |= isSpace
|
||||
}
|
||||
if isChar && !isCtl && !isSeparator {
|
||||
|
||||
12
vendor/github.com/andybalholm/brotli/static_dict.go
сгенерированный
поставляемый
12
vendor/github.com/andybalholm/brotli/static_dict.go
сгенерированный
поставляемый
@@ -77,8 +77,7 @@ func findAllStaticDictionaryMatches(dict *encoderDictionary, data []byte, min_le
|
||||
var offset uint = uint(dict.buckets[hash(data)])
|
||||
var end bool = offset == 0
|
||||
for !end {
|
||||
var w dictWord
|
||||
w = dict.dict_words[offset]
|
||||
w := dict.dict_words[offset]
|
||||
offset++
|
||||
var l uint = uint(w.len) & 0x1F
|
||||
var n uint = uint(1) << dict.words.size_bits_by_length[l]
|
||||
@@ -431,8 +430,7 @@ func findAllStaticDictionaryMatches(dict *encoderDictionary, data []byte, min_le
|
||||
var offset uint = uint(dict.buckets[hash(data[1:])])
|
||||
var end bool = offset == 0
|
||||
for !end {
|
||||
var w dictWord
|
||||
w = dict.dict_words[offset]
|
||||
w := dict.dict_words[offset]
|
||||
offset++
|
||||
var l uint = uint(w.len) & 0x1F
|
||||
var n uint = uint(1) << dict.words.size_bits_by_length[l]
|
||||
@@ -596,8 +594,7 @@ func findAllStaticDictionaryMatches(dict *encoderDictionary, data []byte, min_le
|
||||
var offset uint = uint(dict.buckets[hash(data[2:])])
|
||||
var end bool = offset == 0
|
||||
for !end {
|
||||
var w dictWord
|
||||
w = dict.dict_words[offset]
|
||||
w := dict.dict_words[offset]
|
||||
offset++
|
||||
var l uint = uint(w.len) & 0x1F
|
||||
var n uint = uint(1) << dict.words.size_bits_by_length[l]
|
||||
@@ -629,8 +626,7 @@ func findAllStaticDictionaryMatches(dict *encoderDictionary, data []byte, min_le
|
||||
var offset uint = uint(dict.buckets[hash(data[5:])])
|
||||
var end bool = offset == 0
|
||||
for !end {
|
||||
var w dictWord
|
||||
w = dict.dict_words[offset]
|
||||
w := dict.dict_words[offset]
|
||||
offset++
|
||||
var l uint = uint(w.len) & 0x1F
|
||||
var n uint = uint(1) << dict.words.size_bits_by_length[l]
|
||||
|
||||
3
vendor/github.com/andybalholm/brotli/utf8_util.go
сгенерированный
поставляемый
3
vendor/github.com/andybalholm/brotli/utf8_util.go
сгенерированный
поставляемый
@@ -58,8 +58,7 @@ func isMostlyUTF8(data []byte, pos uint, mask uint, length uint, min_fraction fl
|
||||
var i uint = 0
|
||||
for i < length {
|
||||
var symbol int
|
||||
var current_data []byte
|
||||
current_data = data[(pos+i)&mask:]
|
||||
current_data := data[(pos+i)&mask:]
|
||||
var bytes_read uint = parseAsUTF8(&symbol, current_data, length-i)
|
||||
i += bytes_read
|
||||
if symbol < 0x110000 {
|
||||
|
||||
1
vendor/github.com/andybalholm/brotli/writer.go
сгенерированный
поставляемый
1
vendor/github.com/andybalholm/brotli/writer.go
сгенерированный
поставляемый
@@ -61,6 +61,7 @@ func (w *Writer) Reset(dst io.Writer) {
|
||||
w.params.lgwin = uint(w.options.LGWin)
|
||||
}
|
||||
w.dst = dst
|
||||
w.err = nil
|
||||
}
|
||||
|
||||
func (w *Writer) writeChunk(p []byte, op int) (n int, err error) {
|
||||
|
||||
13
vendor/github.com/aws/aws-sdk-go/aws/client/client.go
сгенерированный
поставляемый
13
vendor/github.com/aws/aws-sdk-go/aws/client/client.go
сгенерированный
поставляемый
@@ -10,12 +10,13 @@ import (
|
||||
|
||||
// A Config provides configuration to a service client instance.
|
||||
type Config struct {
|
||||
Config *aws.Config
|
||||
Handlers request.Handlers
|
||||
PartitionID string
|
||||
Endpoint string
|
||||
SigningRegion string
|
||||
SigningName string
|
||||
Config *aws.Config
|
||||
Handlers request.Handlers
|
||||
PartitionID string
|
||||
Endpoint string
|
||||
SigningRegion string
|
||||
SigningName string
|
||||
ResolvedRegion string
|
||||
|
||||
// States that the signing name did not come from a modeled source but
|
||||
// was derived based on other data. Used by service client constructors
|
||||
|
||||
19
vendor/github.com/aws/aws-sdk-go/aws/client/metadata/client_info.go
сгенерированный
поставляемый
19
vendor/github.com/aws/aws-sdk-go/aws/client/metadata/client_info.go
сгенерированный
поставляемый
@@ -2,13 +2,14 @@ package metadata
|
||||
|
||||
// ClientInfo wraps immutable data from the client.Client structure.
|
||||
type ClientInfo struct {
|
||||
ServiceName string
|
||||
ServiceID string
|
||||
APIVersion string
|
||||
PartitionID string
|
||||
Endpoint string
|
||||
SigningName string
|
||||
SigningRegion string
|
||||
JSONVersion string
|
||||
TargetPrefix string
|
||||
ServiceName string
|
||||
ServiceID string
|
||||
APIVersion string
|
||||
PartitionID string
|
||||
Endpoint string
|
||||
SigningName string
|
||||
SigningRegion string
|
||||
JSONVersion string
|
||||
TargetPrefix string
|
||||
ResolvedRegion string
|
||||
}
|
||||
|
||||
23
vendor/github.com/aws/aws-sdk-go/aws/config.go
сгенерированный
поставляемый
23
vendor/github.com/aws/aws-sdk-go/aws/config.go
сгенерированный
поставляемый
@@ -208,8 +208,19 @@ type Config struct {
|
||||
// svc := s3.New(sess, &aws.Config{
|
||||
// UseDualStack: aws.Bool(true),
|
||||
// })
|
||||
//
|
||||
// Deprecated: This option will continue to function for S3 and S3 Control for backwards compatibility.
|
||||
// UseDualStackEndpoint should be used to enable usage of a service's dual-stack endpoint for all service clients
|
||||
// moving forward. For S3 and S3 Control, when UseDualStackEndpoint is set to a non-zero value it takes higher
|
||||
// precedence then this option.
|
||||
UseDualStack *bool
|
||||
|
||||
// Sets the resolver to resolve a dual-stack endpoint for the service.
|
||||
UseDualStackEndpoint endpoints.DualStackEndpointState
|
||||
|
||||
// UseFIPSEndpoint specifies the resolver must resolve a FIPS endpoint.
|
||||
UseFIPSEndpoint endpoints.FIPSEndpointState
|
||||
|
||||
// SleepDelay is an override for the func the SDK will call when sleeping
|
||||
// during the lifecycle of a request. Specifically this will be used for
|
||||
// request delays. This value should only be used for testing. To adjust
|
||||
@@ -554,6 +565,10 @@ func mergeInConfig(dst *Config, other *Config) {
|
||||
dst.UseDualStack = other.UseDualStack
|
||||
}
|
||||
|
||||
if other.UseDualStackEndpoint != endpoints.DualStackEndpointStateUnset {
|
||||
dst.UseDualStackEndpoint = other.UseDualStackEndpoint
|
||||
}
|
||||
|
||||
if other.EC2MetadataDisableTimeoutOverride != nil {
|
||||
dst.EC2MetadataDisableTimeoutOverride = other.EC2MetadataDisableTimeoutOverride
|
||||
}
|
||||
@@ -589,6 +604,14 @@ func mergeInConfig(dst *Config, other *Config) {
|
||||
if other.LowerCaseHeaderMaps != nil {
|
||||
dst.LowerCaseHeaderMaps = other.LowerCaseHeaderMaps
|
||||
}
|
||||
|
||||
if other.UseDualStackEndpoint != endpoints.DualStackEndpointStateUnset {
|
||||
dst.UseDualStackEndpoint = other.UseDualStackEndpoint
|
||||
}
|
||||
|
||||
if other.UseFIPSEndpoint != endpoints.FIPSEndpointStateUnset {
|
||||
dst.UseFIPSEndpoint = other.UseFIPSEndpoint
|
||||
}
|
||||
}
|
||||
|
||||
// Copy will return a shallow copy of the Config object. If any additional
|
||||
|
||||
59
vendor/github.com/aws/aws-sdk-go/aws/endpoints/decode.go
сгенерированный
поставляемый
59
vendor/github.com/aws/aws-sdk-go/aws/endpoints/decode.go
сгенерированный
поставляемый
@@ -81,7 +81,6 @@ func decodeV3Endpoints(modelDef modelDefinition, opts DecodeModelOptions) (Resol
|
||||
// Customization
|
||||
for i := 0; i < len(ps); i++ {
|
||||
p := &ps[i]
|
||||
custAddS3DualStack(p)
|
||||
custRegionalS3(p)
|
||||
custRmIotDataService(p)
|
||||
custFixAppAutoscalingChina(p)
|
||||
@@ -91,15 +90,6 @@ func decodeV3Endpoints(modelDef modelDefinition, opts DecodeModelOptions) (Resol
|
||||
return ps, nil
|
||||
}
|
||||
|
||||
func custAddS3DualStack(p *partition) {
|
||||
if !(p.ID == "aws" || p.ID == "aws-cn" || p.ID == "aws-us-gov") {
|
||||
return
|
||||
}
|
||||
|
||||
custAddDualstack(p, "s3")
|
||||
custAddDualstack(p, "s3-control")
|
||||
}
|
||||
|
||||
func custRegionalS3(p *partition) {
|
||||
if p.ID != "aws" {
|
||||
return
|
||||
@@ -110,35 +100,28 @@ func custRegionalS3(p *partition) {
|
||||
return
|
||||
}
|
||||
|
||||
const awsGlobal = "aws-global"
|
||||
const usEast1 = "us-east-1"
|
||||
|
||||
// If global endpoint already exists no customization needed.
|
||||
if _, ok := service.Endpoints["aws-global"]; ok {
|
||||
if _, ok := service.Endpoints[endpointKey{Region: awsGlobal}]; ok {
|
||||
return
|
||||
}
|
||||
|
||||
service.PartitionEndpoint = "aws-global"
|
||||
service.Endpoints["us-east-1"] = endpoint{}
|
||||
service.Endpoints["aws-global"] = endpoint{
|
||||
service.PartitionEndpoint = awsGlobal
|
||||
if _, ok := service.Endpoints[endpointKey{Region: usEast1}]; !ok {
|
||||
service.Endpoints[endpointKey{Region: usEast1}] = endpoint{}
|
||||
}
|
||||
service.Endpoints[endpointKey{Region: awsGlobal}] = endpoint{
|
||||
Hostname: "s3.amazonaws.com",
|
||||
CredentialScope: credentialScope{
|
||||
Region: "us-east-1",
|
||||
Region: usEast1,
|
||||
},
|
||||
}
|
||||
|
||||
p.Services["s3"] = service
|
||||
}
|
||||
|
||||
func custAddDualstack(p *partition, svcName string) {
|
||||
s, ok := p.Services[svcName]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
s.Defaults.HasDualStack = boxedTrue
|
||||
s.Defaults.DualStackHostname = "{service}.dualstack.{region}.{dnsSuffix}"
|
||||
|
||||
p.Services[svcName] = s
|
||||
}
|
||||
|
||||
func custRmIotDataService(p *partition) {
|
||||
delete(p.Services, "data.iot")
|
||||
}
|
||||
@@ -155,12 +138,13 @@ func custFixAppAutoscalingChina(p *partition) {
|
||||
}
|
||||
|
||||
const expectHostname = `autoscaling.{region}.amazonaws.com`
|
||||
if e, a := s.Defaults.Hostname, expectHostname; e != a {
|
||||
serviceDefault := s.Defaults[defaultKey{}]
|
||||
if e, a := expectHostname, serviceDefault.Hostname; e != a {
|
||||
fmt.Printf("custFixAppAutoscalingChina: ignoring customization, expected %s, got %s\n", e, a)
|
||||
return
|
||||
}
|
||||
|
||||
s.Defaults.Hostname = expectHostname + ".cn"
|
||||
serviceDefault.Hostname = expectHostname + ".cn"
|
||||
s.Defaults[defaultKey{}] = serviceDefault
|
||||
p.Services[serviceName] = s
|
||||
}
|
||||
|
||||
@@ -175,18 +159,25 @@ func custFixAppAutoscalingUsGov(p *partition) {
|
||||
return
|
||||
}
|
||||
|
||||
if a := s.Defaults.CredentialScope.Service; a != "" {
|
||||
serviceDefault := s.Defaults[defaultKey{}]
|
||||
if a := serviceDefault.CredentialScope.Service; a != "" {
|
||||
fmt.Printf("custFixAppAutoscalingUsGov: ignoring customization, expected empty credential scope service, got %s\n", a)
|
||||
return
|
||||
}
|
||||
|
||||
if a := s.Defaults.Hostname; a != "" {
|
||||
if a := serviceDefault.Hostname; a != "" {
|
||||
fmt.Printf("custFixAppAutoscalingUsGov: ignoring customization, expected empty hostname, got %s\n", a)
|
||||
return
|
||||
}
|
||||
|
||||
s.Defaults.CredentialScope.Service = "application-autoscaling"
|
||||
s.Defaults.Hostname = "autoscaling.{region}.amazonaws.com"
|
||||
serviceDefault.CredentialScope.Service = "application-autoscaling"
|
||||
serviceDefault.Hostname = "autoscaling.{region}.amazonaws.com"
|
||||
|
||||
if s.Defaults == nil {
|
||||
s.Defaults = make(endpointDefaults)
|
||||
}
|
||||
|
||||
s.Defaults[defaultKey{}] = serviceDefault
|
||||
|
||||
p.Services[serviceName] = s
|
||||
}
|
||||
|
||||
27409
vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go
сгенерированный
поставляемый
27409
vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go
сгенерированный
поставляемый
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
103
vendor/github.com/aws/aws-sdk-go/aws/endpoints/endpoints.go
сгенерированный
поставляемый
103
vendor/github.com/aws/aws-sdk-go/aws/endpoints/endpoints.go
сгенерированный
поставляемый
@@ -8,6 +8,41 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
)
|
||||
|
||||
// A Logger is a minimalistic interface for the SDK to log messages to.
|
||||
type Logger interface {
|
||||
Log(...interface{})
|
||||
}
|
||||
|
||||
// DualStackEndpointState is a constant to describe the dual-stack endpoint resolution
|
||||
// behavior.
|
||||
type DualStackEndpointState uint
|
||||
|
||||
const (
|
||||
// DualStackEndpointStateUnset is the default value behavior for dual-stack endpoint
|
||||
// resolution.
|
||||
DualStackEndpointStateUnset DualStackEndpointState = iota
|
||||
|
||||
// DualStackEndpointStateEnabled enable dual-stack endpoint resolution for endpoints.
|
||||
DualStackEndpointStateEnabled
|
||||
|
||||
// DualStackEndpointStateDisabled disables dual-stack endpoint resolution for endpoints.
|
||||
DualStackEndpointStateDisabled
|
||||
)
|
||||
|
||||
// FIPSEndpointState is a constant to describe the FIPS endpoint resolution behavior.
|
||||
type FIPSEndpointState uint
|
||||
|
||||
const (
|
||||
// FIPSEndpointStateUnset is the default value behavior for FIPS endpoint resolution.
|
||||
FIPSEndpointStateUnset FIPSEndpointState = iota
|
||||
|
||||
// FIPSEndpointStateEnabled enables FIPS endpoint resolution for service endpoints.
|
||||
FIPSEndpointStateEnabled
|
||||
|
||||
// FIPSEndpointStateDisabled disables FIPS endpoint resolution for endpoints.
|
||||
FIPSEndpointStateDisabled
|
||||
)
|
||||
|
||||
// Options provide the configuration needed to direct how the
|
||||
// endpoints will be resolved.
|
||||
type Options struct {
|
||||
@@ -21,8 +56,19 @@ type Options struct {
|
||||
// be returned. This endpoint may not be valid. If StrictMatching is
|
||||
// enabled only services that are known to support dualstack will return
|
||||
// dualstack endpoints.
|
||||
//
|
||||
// Deprecated: This option will continue to function for S3 and S3 Control for backwards compatibility.
|
||||
// UseDualStackEndpoint should be used to enable usage of a service's dual-stack endpoint for all service clients
|
||||
// moving forward. For S3 and S3 Control, when UseDualStackEndpoint is set to a non-zero value it takes higher
|
||||
// precedence then this option.
|
||||
UseDualStack bool
|
||||
|
||||
// Sets the resolver to resolve a dual-stack endpoint for the service.
|
||||
UseDualStackEndpoint DualStackEndpointState
|
||||
|
||||
// UseFIPSEndpoint specifies the resolver must resolve a FIPS endpoint.
|
||||
UseFIPSEndpoint FIPSEndpointState
|
||||
|
||||
// Enables strict matching of services and regions resolved endpoints.
|
||||
// If the partition doesn't enumerate the exact service and region an
|
||||
// error will be returned. This option will prevent returning endpoints
|
||||
@@ -56,6 +102,30 @@ type Options struct {
|
||||
|
||||
// S3 Regional Endpoint flag helps with resolving the S3 endpoint
|
||||
S3UsEast1RegionalEndpoint S3UsEast1RegionalEndpoint
|
||||
|
||||
// ResolvedRegion is the resolved region string. If provided (non-zero length) it takes priority
|
||||
// over the region name passed to the ResolveEndpoint call.
|
||||
ResolvedRegion string
|
||||
|
||||
// Logger is the logger that will be used to log messages.
|
||||
Logger Logger
|
||||
|
||||
// Determines whether logging of deprecated endpoints usage is enabled.
|
||||
LogDeprecated bool
|
||||
}
|
||||
|
||||
func (o Options) getEndpointVariant(service string) (v endpointVariant) {
|
||||
const s3 = "s3"
|
||||
const s3Control = "s3-control"
|
||||
|
||||
if (o.UseDualStackEndpoint == DualStackEndpointStateEnabled) ||
|
||||
((service == s3 || service == s3Control) && (o.UseDualStackEndpoint == DualStackEndpointStateUnset && o.UseDualStack)) {
|
||||
v |= dualStackVariant
|
||||
}
|
||||
if o.UseFIPSEndpoint == FIPSEndpointStateEnabled {
|
||||
v |= fipsVariant
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// EC2IMDSEndpointModeState is an enum configuration variable describing the client endpoint mode.
|
||||
@@ -196,10 +266,25 @@ func DisableSSLOption(o *Options) {
|
||||
|
||||
// UseDualStackOption sets the UseDualStack option. Can be used as a functional
|
||||
// option when resolving endpoints.
|
||||
//
|
||||
// Deprecated: UseDualStackEndpointOption should be used to enable usage of a service's dual-stack endpoint.
|
||||
// When DualStackEndpointState is set to a non-zero value it takes higher precedence then this option.
|
||||
func UseDualStackOption(o *Options) {
|
||||
o.UseDualStack = true
|
||||
}
|
||||
|
||||
// UseDualStackEndpointOption sets the UseDualStackEndpoint option to enabled. Can be used as a functional
|
||||
// option when resolving endpoints.
|
||||
func UseDualStackEndpointOption(o *Options) {
|
||||
o.UseDualStackEndpoint = DualStackEndpointStateEnabled
|
||||
}
|
||||
|
||||
// UseFIPSEndpointOption sets the UseFIPSEndpoint option to enabled. Can be used as a functional
|
||||
// option when resolving endpoints.
|
||||
func UseFIPSEndpointOption(o *Options) {
|
||||
o.UseFIPSEndpoint = FIPSEndpointStateEnabled
|
||||
}
|
||||
|
||||
// StrictMatchingOption sets the StrictMatching option. Can be used as a functional
|
||||
// option when resolving endpoints.
|
||||
func StrictMatchingOption(o *Options) {
|
||||
@@ -407,7 +492,7 @@ func (r Region) ResolveEndpoint(service string, opts ...func(*Options)) (Resolve
|
||||
func (r Region) Services() map[string]Service {
|
||||
ss := map[string]Service{}
|
||||
for id, s := range r.p.Services {
|
||||
if _, ok := s.Endpoints[r.id]; ok {
|
||||
if _, ok := s.Endpoints[endpointKey{Region: r.id}]; ok {
|
||||
ss[id] = Service{
|
||||
id: id,
|
||||
p: r.p,
|
||||
@@ -452,9 +537,12 @@ func (s Service) Regions() map[string]Region {
|
||||
}
|
||||
|
||||
for id := range service.Endpoints {
|
||||
if r, ok := s.p.Regions[id]; ok {
|
||||
rs[id] = Region{
|
||||
id: id,
|
||||
if id.Variant != 0 {
|
||||
continue
|
||||
}
|
||||
if r, ok := s.p.Regions[id.Region]; ok {
|
||||
rs[id.Region] = Region{
|
||||
id: id.Region,
|
||||
desc: r.Description,
|
||||
p: s.p,
|
||||
}
|
||||
@@ -472,8 +560,11 @@ func (s Service) Regions() map[string]Region {
|
||||
func (s Service) Endpoints() map[string]Endpoint {
|
||||
es := make(map[string]Endpoint, len(s.p.Services[s.id].Endpoints))
|
||||
for id := range s.p.Services[s.id].Endpoints {
|
||||
es[id] = Endpoint{
|
||||
id: id,
|
||||
if id.Variant != 0 {
|
||||
continue
|
||||
}
|
||||
es[id.Region] = Endpoint{
|
||||
id: id.Region,
|
||||
serviceID: s.id,
|
||||
p: s.p,
|
||||
}
|
||||
|
||||
297
vendor/github.com/aws/aws-sdk-go/aws/endpoints/v3model.go
сгенерированный
поставляемый
297
vendor/github.com/aws/aws-sdk-go/aws/endpoints/v3model.go
сгенерированный
поставляемый
@@ -1,6 +1,7 @@
|
||||
package endpoints
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
@@ -12,6 +13,34 @@ const (
|
||||
ec2MetadataEndpointIPv4 = "http://169.254.169.254/latest"
|
||||
)
|
||||
|
||||
const dnsSuffixTemplateKey = "{dnsSuffix}"
|
||||
|
||||
// defaultKey is a compound map key of a variant and other values.
|
||||
type defaultKey struct {
|
||||
Variant endpointVariant
|
||||
ServiceVariant serviceVariant
|
||||
}
|
||||
|
||||
// endpointKey is a compound map key of a region and associated variant value.
|
||||
type endpointKey struct {
|
||||
Region string
|
||||
Variant endpointVariant
|
||||
}
|
||||
|
||||
// endpointVariant is a bit field to describe the endpoints attributes.
|
||||
type endpointVariant uint64
|
||||
|
||||
// serviceVariant is a bit field to describe the service endpoint attributes.
|
||||
type serviceVariant uint64
|
||||
|
||||
const (
|
||||
// fipsVariant indicates that the endpoint is FIPS capable.
|
||||
fipsVariant endpointVariant = 1 << (64 - 1 - iota)
|
||||
|
||||
// dualStackVariant indicates that the endpoint is DualStack capable.
|
||||
dualStackVariant
|
||||
)
|
||||
|
||||
var regionValidationRegex = regexp.MustCompile(`^[[:alnum:]]([[:alnum:]\-]*[[:alnum:]])?$`)
|
||||
|
||||
type partitions []partition
|
||||
@@ -20,8 +49,12 @@ func (ps partitions) EndpointFor(service, region string, opts ...func(*Options))
|
||||
var opt Options
|
||||
opt.Set(opts...)
|
||||
|
||||
if len(opt.ResolvedRegion) > 0 {
|
||||
region = opt.ResolvedRegion
|
||||
}
|
||||
|
||||
for i := 0; i < len(ps); i++ {
|
||||
if !ps[i].canResolveEndpoint(service, region, opt.StrictMatching) {
|
||||
if !ps[i].canResolveEndpoint(service, region, opt) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -49,14 +82,76 @@ func (ps partitions) Partitions() []Partition {
|
||||
return parts
|
||||
}
|
||||
|
||||
type endpointWithVariants struct {
|
||||
endpoint
|
||||
Variants []endpointWithTags `json:"variants"`
|
||||
}
|
||||
|
||||
type endpointWithTags struct {
|
||||
endpoint
|
||||
Tags []string `json:"tags"`
|
||||
}
|
||||
|
||||
type endpointDefaults map[defaultKey]endpoint
|
||||
|
||||
func (p *endpointDefaults) UnmarshalJSON(data []byte) error {
|
||||
if *p == nil {
|
||||
*p = make(endpointDefaults)
|
||||
}
|
||||
|
||||
var e endpointWithVariants
|
||||
if err := json.Unmarshal(data, &e); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
(*p)[defaultKey{Variant: 0}] = e.endpoint
|
||||
|
||||
e.Hostname = ""
|
||||
e.DNSSuffix = ""
|
||||
|
||||
for _, variant := range e.Variants {
|
||||
endpointVariant, unknown := parseVariantTags(variant.Tags)
|
||||
if unknown {
|
||||
continue
|
||||
}
|
||||
|
||||
var ve endpoint
|
||||
ve.mergeIn(e.endpoint)
|
||||
ve.mergeIn(variant.endpoint)
|
||||
|
||||
(*p)[defaultKey{Variant: endpointVariant}] = ve
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseVariantTags(tags []string) (ev endpointVariant, unknown bool) {
|
||||
if len(tags) == 0 {
|
||||
unknown = true
|
||||
return
|
||||
}
|
||||
|
||||
for _, tag := range tags {
|
||||
switch {
|
||||
case strings.EqualFold("fips", tag):
|
||||
ev |= fipsVariant
|
||||
case strings.EqualFold("dualstack", tag):
|
||||
ev |= dualStackVariant
|
||||
default:
|
||||
unknown = true
|
||||
}
|
||||
}
|
||||
return ev, unknown
|
||||
}
|
||||
|
||||
type partition struct {
|
||||
ID string `json:"partition"`
|
||||
Name string `json:"partitionName"`
|
||||
DNSSuffix string `json:"dnsSuffix"`
|
||||
RegionRegex regionRegex `json:"regionRegex"`
|
||||
Defaults endpoint `json:"defaults"`
|
||||
Regions regions `json:"regions"`
|
||||
Services services `json:"services"`
|
||||
ID string `json:"partition"`
|
||||
Name string `json:"partitionName"`
|
||||
DNSSuffix string `json:"dnsSuffix"`
|
||||
RegionRegex regionRegex `json:"regionRegex"`
|
||||
Defaults endpointDefaults `json:"defaults"`
|
||||
Regions regions `json:"regions"`
|
||||
Services services `json:"services"`
|
||||
}
|
||||
|
||||
func (p partition) Partition() Partition {
|
||||
@@ -67,15 +162,18 @@ func (p partition) Partition() Partition {
|
||||
}
|
||||
}
|
||||
|
||||
func (p partition) canResolveEndpoint(service, region string, strictMatch bool) bool {
|
||||
func (p partition) canResolveEndpoint(service, region string, options Options) bool {
|
||||
s, hasService := p.Services[service]
|
||||
_, hasEndpoint := s.Endpoints[region]
|
||||
_, hasEndpoint := s.Endpoints[endpointKey{
|
||||
Region: region,
|
||||
Variant: options.getEndpointVariant(service),
|
||||
}]
|
||||
|
||||
if hasEndpoint && hasService {
|
||||
return true
|
||||
}
|
||||
|
||||
if strictMatch {
|
||||
if options.StrictMatching {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -106,6 +204,10 @@ func (p partition) EndpointFor(service, region string, opts ...func(*Options)) (
|
||||
var opt Options
|
||||
opt.Set(opts...)
|
||||
|
||||
if len(opt.ResolvedRegion) > 0 {
|
||||
region = opt.ResolvedRegion
|
||||
}
|
||||
|
||||
s, hasService := p.Services[service]
|
||||
|
||||
if service == Ec2metadataServiceID && !hasService {
|
||||
@@ -123,21 +225,44 @@ func (p partition) EndpointFor(service, region string, opts ...func(*Options)) (
|
||||
region = s.PartitionEndpoint
|
||||
}
|
||||
|
||||
if (service == "sts" && opt.STSRegionalEndpoint != RegionalSTSEndpoint) ||
|
||||
(service == "s3" && opt.S3UsEast1RegionalEndpoint != RegionalS3UsEast1Endpoint) {
|
||||
if _, ok := legacyGlobalRegions[service][region]; ok {
|
||||
region = "aws-global"
|
||||
}
|
||||
if r, ok := isLegacyGlobalRegion(service, region, opt); ok {
|
||||
region = r
|
||||
}
|
||||
|
||||
e, hasEndpoint := s.endpointForRegion(region)
|
||||
if len(region) == 0 || (!hasEndpoint && opt.StrictMatching) {
|
||||
return resolved, NewUnknownEndpointError(p.ID, service, region, endpointList(s.Endpoints))
|
||||
variant := opt.getEndpointVariant(service)
|
||||
|
||||
endpoints := s.Endpoints
|
||||
|
||||
serviceDefaults, hasServiceDefault := s.Defaults[defaultKey{Variant: variant}]
|
||||
// If we searched for a variant which may have no explicit service defaults,
|
||||
// then we need to inherit the standard service defaults except the hostname and dnsSuffix
|
||||
if variant != 0 && !hasServiceDefault {
|
||||
serviceDefaults = s.Defaults[defaultKey{}]
|
||||
serviceDefaults.Hostname = ""
|
||||
serviceDefaults.DNSSuffix = ""
|
||||
}
|
||||
|
||||
defs := []endpoint{p.Defaults, s.Defaults}
|
||||
partitionDefaults, hasPartitionDefault := p.Defaults[defaultKey{Variant: variant}]
|
||||
|
||||
return e.resolve(service, p.ID, region, p.DNSSuffix, defs, opt)
|
||||
var dnsSuffix string
|
||||
if len(serviceDefaults.DNSSuffix) > 0 {
|
||||
dnsSuffix = serviceDefaults.DNSSuffix
|
||||
} else if variant == 0 {
|
||||
// For legacy reasons the partition dnsSuffix is not in the defaults, so if we looked for
|
||||
// a non-variant endpoint then we need to set the dnsSuffix.
|
||||
dnsSuffix = p.DNSSuffix
|
||||
}
|
||||
|
||||
noDefaults := !hasServiceDefault && !hasPartitionDefault
|
||||
|
||||
e, hasEndpoint := s.endpointForRegion(region, endpoints, variant)
|
||||
if len(region) == 0 || (!hasEndpoint && (opt.StrictMatching || noDefaults)) {
|
||||
return resolved, NewUnknownEndpointError(p.ID, service, region, endpointList(endpoints, variant))
|
||||
}
|
||||
|
||||
defs := []endpoint{partitionDefaults, serviceDefaults}
|
||||
|
||||
return e.resolve(service, p.ID, region, dnsSuffixTemplateKey, dnsSuffix, defs, opt)
|
||||
}
|
||||
|
||||
func getEC2MetadataEndpoint(partitionID, service string, mode EC2IMDSEndpointModeState) ResolvedEndpoint {
|
||||
@@ -165,6 +290,31 @@ func getEC2MetadataEndpoint(partitionID, service string, mode EC2IMDSEndpointMod
|
||||
}
|
||||
}
|
||||
|
||||
func isLegacyGlobalRegion(service string, region string, opt Options) (string, bool) {
|
||||
if opt.getEndpointVariant(service) != 0 {
|
||||
return "", false
|
||||
}
|
||||
|
||||
const (
|
||||
sts = "sts"
|
||||
s3 = "s3"
|
||||
awsGlobal = "aws-global"
|
||||
)
|
||||
|
||||
switch {
|
||||
case service == sts && opt.STSRegionalEndpoint == RegionalSTSEndpoint:
|
||||
return region, false
|
||||
case service == s3 && opt.S3UsEast1RegionalEndpoint == RegionalS3UsEast1Endpoint:
|
||||
return region, false
|
||||
default:
|
||||
if _, ok := legacyGlobalRegions[service][region]; ok {
|
||||
return awsGlobal, true
|
||||
}
|
||||
}
|
||||
|
||||
return region, false
|
||||
}
|
||||
|
||||
func serviceList(ss services) []string {
|
||||
list := make([]string, 0, len(ss))
|
||||
for k := range ss {
|
||||
@@ -172,10 +322,13 @@ func serviceList(ss services) []string {
|
||||
}
|
||||
return list
|
||||
}
|
||||
func endpointList(es endpoints) []string {
|
||||
func endpointList(es serviceEndpoints, variant endpointVariant) []string {
|
||||
list := make([]string, 0, len(es))
|
||||
for k := range es {
|
||||
list = append(list, k)
|
||||
if k.Variant != variant {
|
||||
continue
|
||||
}
|
||||
list = append(list, k.Region)
|
||||
}
|
||||
return list
|
||||
}
|
||||
@@ -207,19 +360,19 @@ type region struct {
|
||||
type services map[string]service
|
||||
|
||||
type service struct {
|
||||
PartitionEndpoint string `json:"partitionEndpoint"`
|
||||
IsRegionalized boxedBool `json:"isRegionalized,omitempty"`
|
||||
Defaults endpoint `json:"defaults"`
|
||||
Endpoints endpoints `json:"endpoints"`
|
||||
PartitionEndpoint string `json:"partitionEndpoint"`
|
||||
IsRegionalized boxedBool `json:"isRegionalized,omitempty"`
|
||||
Defaults endpointDefaults `json:"defaults"`
|
||||
Endpoints serviceEndpoints `json:"endpoints"`
|
||||
}
|
||||
|
||||
func (s *service) endpointForRegion(region string) (endpoint, bool) {
|
||||
if e, ok := s.Endpoints[region]; ok {
|
||||
func (s *service) endpointForRegion(region string, endpoints serviceEndpoints, variant endpointVariant) (endpoint, bool) {
|
||||
if e, ok := endpoints[endpointKey{Region: region, Variant: variant}]; ok {
|
||||
return e, true
|
||||
}
|
||||
|
||||
if s.IsRegionalized == boxedFalse {
|
||||
return s.Endpoints[s.PartitionEndpoint], region == s.PartitionEndpoint
|
||||
return endpoints[endpointKey{Region: s.PartitionEndpoint, Variant: variant}], region == s.PartitionEndpoint
|
||||
}
|
||||
|
||||
// Unable to find any matching endpoint, return
|
||||
@@ -227,22 +380,73 @@ func (s *service) endpointForRegion(region string) (endpoint, bool) {
|
||||
return endpoint{}, false
|
||||
}
|
||||
|
||||
type endpoints map[string]endpoint
|
||||
type serviceEndpoints map[endpointKey]endpoint
|
||||
|
||||
func (s *serviceEndpoints) UnmarshalJSON(data []byte) error {
|
||||
if *s == nil {
|
||||
*s = make(serviceEndpoints)
|
||||
}
|
||||
|
||||
var regionToEndpoint map[string]endpointWithVariants
|
||||
|
||||
if err := json.Unmarshal(data, ®ionToEndpoint); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for region, e := range regionToEndpoint {
|
||||
(*s)[endpointKey{Region: region}] = e.endpoint
|
||||
|
||||
e.Hostname = ""
|
||||
e.DNSSuffix = ""
|
||||
|
||||
for _, variant := range e.Variants {
|
||||
endpointVariant, unknown := parseVariantTags(variant.Tags)
|
||||
if unknown {
|
||||
continue
|
||||
}
|
||||
|
||||
var ve endpoint
|
||||
ve.mergeIn(e.endpoint)
|
||||
ve.mergeIn(variant.endpoint)
|
||||
|
||||
(*s)[endpointKey{Region: region, Variant: endpointVariant}] = ve
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type endpoint struct {
|
||||
Hostname string `json:"hostname"`
|
||||
Protocols []string `json:"protocols"`
|
||||
CredentialScope credentialScope `json:"credentialScope"`
|
||||
|
||||
// Custom fields not modeled
|
||||
HasDualStack boxedBool `json:"-"`
|
||||
DualStackHostname string `json:"-"`
|
||||
DNSSuffix string `json:"dnsSuffix"`
|
||||
|
||||
// Signature Version not used
|
||||
SignatureVersions []string `json:"signatureVersions"`
|
||||
|
||||
// SSLCommonName not used.
|
||||
SSLCommonName string `json:"sslCommonName"`
|
||||
|
||||
Deprecated boxedBool `json:"deprecated"`
|
||||
}
|
||||
|
||||
// isZero returns whether the endpoint structure is an empty (zero) value.
|
||||
func (e endpoint) isZero() bool {
|
||||
switch {
|
||||
case len(e.Hostname) != 0:
|
||||
return false
|
||||
case len(e.Protocols) != 0:
|
||||
return false
|
||||
case e.CredentialScope != (credentialScope{}):
|
||||
return false
|
||||
case len(e.SignatureVersions) != 0:
|
||||
return false
|
||||
case len(e.SSLCommonName) != 0:
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -271,7 +475,7 @@ func getByPriority(s []string, p []string, def string) string {
|
||||
return s[0]
|
||||
}
|
||||
|
||||
func (e endpoint) resolve(service, partitionID, region, dnsSuffix string, defs []endpoint, opts Options) (ResolvedEndpoint, error) {
|
||||
func (e endpoint) resolve(service, partitionID, region, dnsSuffixTemplateVariable, dnsSuffix string, defs []endpoint, opts Options) (ResolvedEndpoint, error) {
|
||||
var merged endpoint
|
||||
for _, def := range defs {
|
||||
merged.mergeIn(def)
|
||||
@@ -292,23 +496,26 @@ func (e endpoint) resolve(service, partitionID, region, dnsSuffix string, defs [
|
||||
}
|
||||
|
||||
hostname := e.Hostname
|
||||
// Offset the hostname for dualstack if enabled
|
||||
if opts.UseDualStack && e.HasDualStack == boxedTrue {
|
||||
hostname = e.DualStackHostname
|
||||
region = signingRegion
|
||||
}
|
||||
|
||||
if !validateInputRegion(region) {
|
||||
return ResolvedEndpoint{}, fmt.Errorf("invalid region identifier format provided")
|
||||
}
|
||||
|
||||
if len(merged.DNSSuffix) > 0 {
|
||||
dnsSuffix = merged.DNSSuffix
|
||||
}
|
||||
|
||||
u := strings.Replace(hostname, "{service}", service, 1)
|
||||
u = strings.Replace(u, "{region}", region, 1)
|
||||
u = strings.Replace(u, "{dnsSuffix}", dnsSuffix, 1)
|
||||
u = strings.Replace(u, dnsSuffixTemplateVariable, dnsSuffix, 1)
|
||||
|
||||
scheme := getEndpointScheme(e.Protocols, opts.DisableSSL)
|
||||
u = fmt.Sprintf("%s://%s", scheme, u)
|
||||
|
||||
if e.Deprecated == boxedTrue && opts.LogDeprecated && opts.Logger != nil {
|
||||
opts.Logger.Log(fmt.Sprintf("endpoint identifier %q, url %q marked as deprecated", region, u))
|
||||
}
|
||||
|
||||
return ResolvedEndpoint{
|
||||
URL: u,
|
||||
PartitionID: partitionID,
|
||||
@@ -346,11 +553,11 @@ func (e *endpoint) mergeIn(other endpoint) {
|
||||
if len(other.SSLCommonName) > 0 {
|
||||
e.SSLCommonName = other.SSLCommonName
|
||||
}
|
||||
if other.HasDualStack != boxedBoolUnset {
|
||||
e.HasDualStack = other.HasDualStack
|
||||
if len(other.DNSSuffix) > 0 {
|
||||
e.DNSSuffix = other.DNSSuffix
|
||||
}
|
||||
if len(other.DualStackHostname) > 0 {
|
||||
e.DualStackHostname = other.DualStackHostname
|
||||
if other.Deprecated != boxedBoolUnset {
|
||||
e.Deprecated = other.Deprecated
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
106
vendor/github.com/aws/aws-sdk-go/aws/endpoints/v3model_codegen.go
сгенерированный
поставляемый
106
vendor/github.com/aws/aws-sdk-go/aws/endpoints/v3model_codegen.go
сгенерированный
поставляемый
@@ -155,18 +155,71 @@ func serviceSet(ps partitions) map[string]struct{} {
|
||||
return set
|
||||
}
|
||||
|
||||
func endpointVariantSetter(variant endpointVariant) (string, error) {
|
||||
if variant == 0 {
|
||||
return "0", nil
|
||||
}
|
||||
|
||||
if variant > (fipsVariant | dualStackVariant) {
|
||||
return "", fmt.Errorf("unknown endpoint variant")
|
||||
}
|
||||
|
||||
var symbols []string
|
||||
if variant&fipsVariant != 0 {
|
||||
symbols = append(symbols, "fipsVariant")
|
||||
}
|
||||
if variant&dualStackVariant != 0 {
|
||||
symbols = append(symbols, "dualStackVariant")
|
||||
}
|
||||
v := strings.Join(symbols, "|")
|
||||
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func endpointKeySetter(e endpointKey) (string, error) {
|
||||
var sb strings.Builder
|
||||
sb.WriteString("endpointKey{\n")
|
||||
sb.WriteString(fmt.Sprintf("Region: %q,\n", e.Region))
|
||||
if e.Variant != 0 {
|
||||
variantSetter, err := endpointVariantSetter(e.Variant)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sb.WriteString(fmt.Sprintf("Variant: %s,\n", variantSetter))
|
||||
}
|
||||
sb.WriteString("}")
|
||||
return sb.String(), nil
|
||||
}
|
||||
|
||||
func defaultKeySetter(e defaultKey) (string, error) {
|
||||
var sb strings.Builder
|
||||
sb.WriteString("defaultKey{\n")
|
||||
if e.Variant != 0 {
|
||||
variantSetter, err := endpointVariantSetter(e.Variant)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sb.WriteString(fmt.Sprintf("Variant: %s,\n", variantSetter))
|
||||
}
|
||||
sb.WriteString("}")
|
||||
return sb.String(), nil
|
||||
}
|
||||
|
||||
var funcMap = template.FuncMap{
|
||||
"ToSymbol": toSymbol,
|
||||
"QuoteString": quoteString,
|
||||
"RegionConst": regionConstName,
|
||||
"PartitionGetter": partitionGetter,
|
||||
"PartitionVarName": partitionVarName,
|
||||
"ListPartitionNames": listPartitionNames,
|
||||
"BoxedBoolIfSet": boxedBoolIfSet,
|
||||
"StringIfSet": stringIfSet,
|
||||
"StringSliceIfSet": stringSliceIfSet,
|
||||
"EndpointIsSet": endpointIsSet,
|
||||
"ServicesSet": serviceSet,
|
||||
"ToSymbol": toSymbol,
|
||||
"QuoteString": quoteString,
|
||||
"RegionConst": regionConstName,
|
||||
"PartitionGetter": partitionGetter,
|
||||
"PartitionVarName": partitionVarName,
|
||||
"ListPartitionNames": listPartitionNames,
|
||||
"BoxedBoolIfSet": boxedBoolIfSet,
|
||||
"StringIfSet": stringIfSet,
|
||||
"StringSliceIfSet": stringSliceIfSet,
|
||||
"EndpointIsSet": endpointIsSet,
|
||||
"ServicesSet": serviceSet,
|
||||
"EndpointVariantSetter": endpointVariantSetter,
|
||||
"EndpointKeySetter": endpointKeySetter,
|
||||
"DefaultKeySetter": defaultKeySetter,
|
||||
}
|
||||
|
||||
const v3Tmpl = `
|
||||
@@ -272,9 +325,9 @@ partition{
|
||||
{{ StringIfSet "Name: %q,\n" .Name -}}
|
||||
{{ StringIfSet "DNSSuffix: %q,\n" .DNSSuffix -}}
|
||||
RegionRegex: {{ template "gocode RegionRegex" .RegionRegex }},
|
||||
{{ if EndpointIsSet .Defaults -}}
|
||||
Defaults: {{ template "gocode Endpoint" .Defaults }},
|
||||
{{- end }}
|
||||
{{ if (gt (len .Defaults) 0) -}}
|
||||
Defaults: {{ template "gocode Defaults" .Defaults -}},
|
||||
{{ end -}}
|
||||
Regions: {{ template "gocode Regions" .Regions }},
|
||||
Services: {{ template "gocode Services" .Services }},
|
||||
}
|
||||
@@ -315,19 +368,27 @@ services{
|
||||
service{
|
||||
{{ StringIfSet "PartitionEndpoint: %q,\n" .PartitionEndpoint -}}
|
||||
{{ BoxedBoolIfSet "IsRegionalized: %s,\n" .IsRegionalized -}}
|
||||
{{ if EndpointIsSet .Defaults -}}
|
||||
Defaults: {{ template "gocode Endpoint" .Defaults -}},
|
||||
{{- end }}
|
||||
{{ if (gt (len .Defaults) 0) -}}
|
||||
Defaults: {{ template "gocode Defaults" .Defaults -}},
|
||||
{{ end -}}
|
||||
{{ if .Endpoints -}}
|
||||
Endpoints: {{ template "gocode Endpoints" .Endpoints }},
|
||||
{{- end }}
|
||||
}
|
||||
{{- end }}
|
||||
|
||||
{{ define "gocode Endpoints" -}}
|
||||
endpoints{
|
||||
{{ define "gocode Defaults" -}}
|
||||
endpointDefaults{
|
||||
{{ range $id, $endpoint := . -}}
|
||||
"{{ $id }}": {{ template "gocode Endpoint" $endpoint }},
|
||||
{{ DefaultKeySetter $id }}: {{ template "gocode Endpoint" $endpoint }},
|
||||
{{ end }}
|
||||
}
|
||||
{{- end }}
|
||||
|
||||
{{ define "gocode Endpoints" -}}
|
||||
serviceEndpoints{
|
||||
{{ range $id, $endpoint := . -}}
|
||||
{{ EndpointKeySetter $id }}: {{ template "gocode Endpoint" $endpoint }},
|
||||
{{ end }}
|
||||
}
|
||||
{{- end }}
|
||||
@@ -335,6 +396,7 @@ endpoints{
|
||||
{{ define "gocode Endpoint" -}}
|
||||
endpoint{
|
||||
{{ StringIfSet "Hostname: %q,\n" .Hostname -}}
|
||||
{{ StringIfSet "DNSSuffix: %q,\n" .DNSSuffix -}}
|
||||
{{ StringIfSet "SSLCommonName: %q,\n" .SSLCommonName -}}
|
||||
{{ StringSliceIfSet "Protocols: []string{%s},\n" .Protocols -}}
|
||||
{{ StringSliceIfSet "SignatureVersions: []string{%s},\n" .SignatureVersions -}}
|
||||
@@ -344,9 +406,7 @@ endpoint{
|
||||
{{ StringIfSet "Service: %q,\n" .CredentialScope.Service -}}
|
||||
},
|
||||
{{- end }}
|
||||
{{ BoxedBoolIfSet "HasDualStack: %s,\n" .HasDualStack -}}
|
||||
{{ StringIfSet "DualStackHostname: %q,\n" .DualStackHostname -}}
|
||||
|
||||
{{ BoxedBoolIfSet "Deprecated: %s,\n" .Deprecated -}}
|
||||
}
|
||||
{{- end }}
|
||||
`
|
||||
|
||||
3
vendor/github.com/aws/aws-sdk-go/aws/logger.go
сгенерированный
поставляемый
3
vendor/github.com/aws/aws-sdk-go/aws/logger.go
сгенерированный
поставляемый
@@ -77,6 +77,9 @@ const (
|
||||
// wire unmarshaled message content of requests and responses made while
|
||||
// using the SDK Will also enable LogDebug.
|
||||
LogDebugWithEventStreamBody
|
||||
|
||||
// LogDebugWithDeprecated states the SDK should log details about deprecated functionality.
|
||||
LogDebugWithDeprecated
|
||||
)
|
||||
|
||||
// A Logger is a minimalistic interface for the SDK to log messages to. Should
|
||||
|
||||
78
vendor/github.com/aws/aws-sdk-go/aws/session/doc.go
сгенерированный
поставляемый
78
vendor/github.com/aws/aws-sdk-go/aws/session/doc.go
сгенерированный
поставляемый
@@ -285,5 +285,83 @@ The custom EC2 IMDS endpoint can also be specified via the Session options.
|
||||
sess, err := session.NewSessionWithOptions(session.Options{
|
||||
EC2MetadataEndpoint: "http://[::1]",
|
||||
})
|
||||
|
||||
FIPS and DualStack Endpoints
|
||||
|
||||
The SDK can be configured to resolve an endpoint with certain capabilities such as FIPS and DualStack.
|
||||
|
||||
You can configure a FIPS endpoint using an environment variable, shared config ($HOME/.aws/config),
|
||||
or programmatically.
|
||||
|
||||
To configure a FIPS endpoint set the environment variable set the AWS_USE_FIPS_ENDPOINT to true or false to enable
|
||||
or disable FIPS endpoint resolution.
|
||||
|
||||
AWS_USE_FIPS_ENDPOINT=true
|
||||
|
||||
To configure a FIPS endpoint using shared config, set use_fips_endpoint to true or false to enable
|
||||
or disable FIPS endpoint resolution.
|
||||
|
||||
[profile myprofile]
|
||||
region=us-west-2
|
||||
use_fips_endpoint=true
|
||||
|
||||
To configure a FIPS endpoint programmatically
|
||||
|
||||
// Option 1: Configure it on a session for all clients
|
||||
sess, err := session.NewSessionWithOptions(session.Options{
|
||||
UseFIPSEndpoint: endpoints.FIPSEndpointStateEnabled,
|
||||
})
|
||||
if err != nil {
|
||||
// handle error
|
||||
}
|
||||
|
||||
client := s3.New(sess)
|
||||
|
||||
// Option 2: Configure it per client
|
||||
sess, err := session.NewSession()
|
||||
if err != nil {
|
||||
// handle error
|
||||
}
|
||||
|
||||
client := s3.New(sess, &aws.Config{
|
||||
UseFIPSEndpoint: endpoints.FIPSEndpointStateEnabled,
|
||||
})
|
||||
|
||||
You can configure a DualStack endpoint using an environment variable, shared config ($HOME/.aws/config),
|
||||
or programmatically.
|
||||
|
||||
To configure a DualStack endpoint set the environment variable set the AWS_USE_DUALSTACK_ENDPOINT to true or false to
|
||||
enable or disable DualStack endpoint resolution.
|
||||
|
||||
AWS_USE_DUALSTACK_ENDPOINT=true
|
||||
|
||||
To configure a DualStack endpoint using shared config, set use_dualstack_endpoint to true or false to enable
|
||||
or disable DualStack endpoint resolution.
|
||||
|
||||
[profile myprofile]
|
||||
region=us-west-2
|
||||
use_dualstack_endpoint=true
|
||||
|
||||
To configure a DualStack endpoint programmatically
|
||||
|
||||
// Option 1: Configure it on a session for all clients
|
||||
sess, err := session.NewSessionWithOptions(session.Options{
|
||||
UseDualStackEndpoint: endpoints.DualStackEndpointStateEnabled,
|
||||
})
|
||||
if err != nil {
|
||||
// handle error
|
||||
}
|
||||
|
||||
client := s3.New(sess)
|
||||
|
||||
// Option 2: Configure it per client
|
||||
sess, err := session.NewSession()
|
||||
if err != nil {
|
||||
// handle error
|
||||
}
|
||||
|
||||
client := s3.New(sess, &aws.Config{
|
||||
UseDualStackEndpoint: endpoints.DualStackEndpointStateEnabled,
|
||||
})
|
||||
*/
|
||||
package session
|
||||
|
||||
68
vendor/github.com/aws/aws-sdk-go/aws/session/env_config.go
сгенерированный
поставляемый
68
vendor/github.com/aws/aws-sdk-go/aws/session/env_config.go
сгенерированный
поставляемый
@@ -170,6 +170,18 @@ type envConfig struct {
|
||||
//
|
||||
// AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE=IPv6
|
||||
EC2IMDSEndpointMode endpoints.EC2IMDSEndpointModeState
|
||||
|
||||
// Specifies that SDK clients must resolve a dual-stack endpoint for
|
||||
// services.
|
||||
//
|
||||
// AWS_USE_DUALSTACK_ENDPOINT=true
|
||||
UseDualStackEndpoint endpoints.DualStackEndpointState
|
||||
|
||||
// Specifies that SDK clients must resolve a FIPS endpoint for
|
||||
// services.
|
||||
//
|
||||
// AWS_USE_FIPS_ENDPOINT=true
|
||||
UseFIPSEndpoint endpoints.FIPSEndpointState
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -248,6 +260,12 @@ var (
|
||||
useClientTLSKey = []string{
|
||||
"AWS_SDK_GO_CLIENT_TLS_KEY",
|
||||
}
|
||||
awsUseDualStackEndpoint = []string{
|
||||
"AWS_USE_DUALSTACK_ENDPOINT",
|
||||
}
|
||||
awsUseFIPSEndpoint = []string{
|
||||
"AWS_USE_FIPS_ENDPOINT",
|
||||
}
|
||||
)
|
||||
|
||||
// loadEnvConfig retrieves the SDK's environment configuration.
|
||||
@@ -376,6 +394,14 @@ func envConfigLoad(enableSharedConfig bool) (envConfig, error) {
|
||||
return envConfig{}, err
|
||||
}
|
||||
|
||||
if err := setUseDualStackEndpointFromEnvVal(&cfg.UseDualStackEndpoint, awsUseDualStackEndpoint); err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
|
||||
if err := setUseFIPSEndpointFromEnvVal(&cfg.UseFIPSEndpoint, awsUseFIPSEndpoint); err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
@@ -401,3 +427,45 @@ func setEC2IMDSEndpointMode(mode *endpoints.EC2IMDSEndpointModeState, keys []str
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func setUseDualStackEndpointFromEnvVal(dst *endpoints.DualStackEndpointState, keys []string) error {
|
||||
for _, k := range keys {
|
||||
value := os.Getenv(k)
|
||||
if len(value) == 0 {
|
||||
continue // skip if empty
|
||||
}
|
||||
|
||||
switch {
|
||||
case strings.EqualFold(value, "true"):
|
||||
*dst = endpoints.DualStackEndpointStateEnabled
|
||||
case strings.EqualFold(value, "false"):
|
||||
*dst = endpoints.DualStackEndpointStateDisabled
|
||||
default:
|
||||
return fmt.Errorf(
|
||||
"invalid value for environment variable, %s=%s, need true, false",
|
||||
k, value)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func setUseFIPSEndpointFromEnvVal(dst *endpoints.FIPSEndpointState, keys []string) error {
|
||||
for _, k := range keys {
|
||||
value := os.Getenv(k)
|
||||
if len(value) == 0 {
|
||||
continue // skip if empty
|
||||
}
|
||||
|
||||
switch {
|
||||
case strings.EqualFold(value, "true"):
|
||||
*dst = endpoints.FIPSEndpointStateEnabled
|
||||
case strings.EqualFold(value, "false"):
|
||||
*dst = endpoints.FIPSEndpointStateDisabled
|
||||
default:
|
||||
return fmt.Errorf(
|
||||
"invalid value for environment variable, %s=%s, need true, false",
|
||||
k, value)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
55
vendor/github.com/aws/aws-sdk-go/aws/session/session.go
сгенерированный
поставляемый
55
vendor/github.com/aws/aws-sdk-go/aws/session/session.go
сгенерированный
поставляемый
@@ -8,6 +8,7 @@ import (
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
@@ -792,6 +793,20 @@ func mergeConfigSrcs(cfg, userCfg *aws.Config,
|
||||
cfg.S3UseARNRegion = &sharedCfg.S3UseARNRegion
|
||||
}
|
||||
|
||||
for _, v := range []endpoints.DualStackEndpointState{userCfg.UseDualStackEndpoint, envCfg.UseDualStackEndpoint, sharedCfg.UseDualStackEndpoint} {
|
||||
if v != endpoints.DualStackEndpointStateUnset {
|
||||
cfg.UseDualStackEndpoint = v
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range []endpoints.FIPSEndpointState{userCfg.UseFIPSEndpoint, envCfg.UseFIPSEndpoint, sharedCfg.UseFIPSEndpoint} {
|
||||
if v != endpoints.FIPSEndpointStateUnset {
|
||||
cfg.UseFIPSEndpoint = v
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -845,8 +860,10 @@ func (s *Session) Copy(cfgs ...*aws.Config) *Session {
|
||||
func (s *Session) ClientConfig(service string, cfgs ...*aws.Config) client.Config {
|
||||
s = s.Copy(cfgs...)
|
||||
|
||||
resolvedRegion := normalizeRegion(s.Config)
|
||||
|
||||
region := aws.StringValue(s.Config.Region)
|
||||
resolved, err := s.resolveEndpoint(service, region, s.Config)
|
||||
resolved, err := s.resolveEndpoint(service, region, resolvedRegion, s.Config)
|
||||
if err != nil {
|
||||
s.Handlers.Validate.PushBack(func(r *request.Request) {
|
||||
if len(r.ClientInfo.Endpoint) != 0 {
|
||||
@@ -867,12 +884,13 @@ func (s *Session) ClientConfig(service string, cfgs ...*aws.Config) client.Confi
|
||||
SigningRegion: resolved.SigningRegion,
|
||||
SigningNameDerived: resolved.SigningNameDerived,
|
||||
SigningName: resolved.SigningName,
|
||||
ResolvedRegion: resolvedRegion,
|
||||
}
|
||||
}
|
||||
|
||||
const ec2MetadataServiceID = "ec2metadata"
|
||||
|
||||
func (s *Session) resolveEndpoint(service, region string, cfg *aws.Config) (endpoints.ResolvedEndpoint, error) {
|
||||
func (s *Session) resolveEndpoint(service, region, resolvedRegion string, cfg *aws.Config) (endpoints.ResolvedEndpoint, error) {
|
||||
|
||||
if ep := aws.StringValue(cfg.Endpoint); len(ep) != 0 {
|
||||
return endpoints.ResolvedEndpoint{
|
||||
@@ -884,7 +902,12 @@ func (s *Session) resolveEndpoint(service, region string, cfg *aws.Config) (endp
|
||||
resolved, err := cfg.EndpointResolver.EndpointFor(service, region,
|
||||
func(opt *endpoints.Options) {
|
||||
opt.DisableSSL = aws.BoolValue(cfg.DisableSSL)
|
||||
|
||||
opt.UseDualStack = aws.BoolValue(cfg.UseDualStack)
|
||||
opt.UseDualStackEndpoint = cfg.UseDualStackEndpoint
|
||||
|
||||
opt.UseFIPSEndpoint = cfg.UseFIPSEndpoint
|
||||
|
||||
// Support for STSRegionalEndpoint where the STSRegionalEndpoint is
|
||||
// provided in envConfig or sharedConfig with envConfig getting
|
||||
// precedence.
|
||||
@@ -898,6 +921,11 @@ func (s *Session) resolveEndpoint(service, region string, cfg *aws.Config) (endp
|
||||
// Support the condition where the service is modeled but its
|
||||
// endpoint metadata is not available.
|
||||
opt.ResolveUnknownService = true
|
||||
|
||||
opt.ResolvedRegion = resolvedRegion
|
||||
|
||||
opt.Logger = cfg.Logger
|
||||
opt.LogDeprecated = cfg.LogLevel.Matches(aws.LogDebugWithDeprecated)
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
@@ -913,6 +941,8 @@ func (s *Session) resolveEndpoint(service, region string, cfg *aws.Config) (endp
|
||||
func (s *Session) ClientConfigNoResolveEndpoint(cfgs ...*aws.Config) client.Config {
|
||||
s = s.Copy(cfgs...)
|
||||
|
||||
resolvedRegion := normalizeRegion(s.Config)
|
||||
|
||||
var resolved endpoints.ResolvedEndpoint
|
||||
if ep := aws.StringValue(s.Config.Endpoint); len(ep) > 0 {
|
||||
resolved.URL = endpoints.AddScheme(ep, aws.BoolValue(s.Config.DisableSSL))
|
||||
@@ -926,6 +956,7 @@ func (s *Session) ClientConfigNoResolveEndpoint(cfgs ...*aws.Config) client.Conf
|
||||
SigningRegion: resolved.SigningRegion,
|
||||
SigningNameDerived: resolved.SigningNameDerived,
|
||||
SigningName: resolved.SigningName,
|
||||
ResolvedRegion: resolvedRegion,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -939,3 +970,23 @@ func (s *Session) logDeprecatedNewSessionError(msg string, err error, cfgs []*aw
|
||||
r.Error = err
|
||||
})
|
||||
}
|
||||
|
||||
// normalizeRegion resolves / normalizes the configured region (converts pseudo fips regions), and modifies the provided
|
||||
// config to have the equivalent options for resolution and returns the resolved region name.
|
||||
func normalizeRegion(cfg *aws.Config) (resolved string) {
|
||||
const fipsInfix = "-fips-"
|
||||
const fipsPrefix = "-fips"
|
||||
const fipsSuffix = "fips-"
|
||||
|
||||
region := aws.StringValue(cfg.Region)
|
||||
|
||||
if strings.Contains(region, fipsInfix) ||
|
||||
strings.Contains(region, fipsPrefix) ||
|
||||
strings.Contains(region, fipsSuffix) {
|
||||
resolved = strings.Replace(strings.Replace(strings.Replace(
|
||||
region, fipsInfix, "-", -1), fipsPrefix, "", -1), fipsSuffix, "", -1)
|
||||
cfg.UseFIPSEndpoint = endpoints.FIPSEndpointStateEnabled
|
||||
}
|
||||
|
||||
return resolved
|
||||
}
|
||||
|
||||
54
vendor/github.com/aws/aws-sdk-go/aws/session/shared_config.go
сгенерированный
поставляемый
54
vendor/github.com/aws/aws-sdk-go/aws/session/shared_config.go
сгенерированный
поставляемый
@@ -72,6 +72,12 @@ const (
|
||||
|
||||
// EC2 IMDS Endpoint
|
||||
ec2MetadataServiceEndpointKey = "ec2_metadata_service_endpoint"
|
||||
|
||||
// Use DualStack Endpoint Resolution
|
||||
useDualStackEndpoint = "use_dualstack_endpoint"
|
||||
|
||||
// Use FIPS Endpoint Resolution
|
||||
useFIPSEndpointKey = "use_fips_endpoint"
|
||||
)
|
||||
|
||||
// sharedConfig represents the configuration fields of the SDK config files.
|
||||
@@ -161,6 +167,18 @@ type sharedConfig struct {
|
||||
//
|
||||
// ec2_metadata_service_endpoint=http://fd00:ec2::254
|
||||
EC2IMDSEndpoint string
|
||||
|
||||
// Specifies that SDK clients must resolve a dual-stack endpoint for
|
||||
// services.
|
||||
//
|
||||
// use_dualstack_endpoint=true
|
||||
UseDualStackEndpoint endpoints.DualStackEndpointState
|
||||
|
||||
// Specifies that SDK clients must resolve a FIPS endpoint for
|
||||
// services.
|
||||
//
|
||||
// use_fips_endpoint=true
|
||||
UseFIPSEndpoint endpoints.FIPSEndpointState
|
||||
}
|
||||
|
||||
type sharedConfigFile struct {
|
||||
@@ -356,6 +374,10 @@ func (cfg *sharedConfig) setFromIniFile(profile string, file sharedConfigFile, e
|
||||
ec2MetadataServiceEndpointModeKey, file.Filename, err)
|
||||
}
|
||||
updateString(&cfg.EC2IMDSEndpoint, section, ec2MetadataServiceEndpointKey)
|
||||
|
||||
updateUseDualStackEndpoint(&cfg.UseDualStackEndpoint, section, useDualStackEndpoint)
|
||||
|
||||
updateUseFIPSEndpoint(&cfg.UseFIPSEndpoint, section, useFIPSEndpointKey)
|
||||
}
|
||||
|
||||
updateString(&cfg.CredentialProcess, section, credentialProcessKey)
|
||||
@@ -673,3 +695,35 @@ func (e CredentialRequiresARNError) OrigErr() error {
|
||||
func (e CredentialRequiresARNError) Error() string {
|
||||
return awserr.SprintError(e.Code(), e.Message(), "", nil)
|
||||
}
|
||||
|
||||
// updateEndpointDiscoveryType will only update the dst with the value in the section, if
|
||||
// a valid key and corresponding EndpointDiscoveryType is found.
|
||||
func updateUseDualStackEndpoint(dst *endpoints.DualStackEndpointState, section ini.Section, key string) {
|
||||
if !section.Has(key) {
|
||||
return
|
||||
}
|
||||
|
||||
if section.Bool(key) {
|
||||
*dst = endpoints.DualStackEndpointStateEnabled
|
||||
} else {
|
||||
*dst = endpoints.DualStackEndpointStateDisabled
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// updateEndpointDiscoveryType will only update the dst with the value in the section, if
|
||||
// a valid key and corresponding EndpointDiscoveryType is found.
|
||||
func updateUseFIPSEndpoint(dst *endpoints.FIPSEndpointState, section ini.Section, key string) {
|
||||
if !section.Has(key) {
|
||||
return
|
||||
}
|
||||
|
||||
if section.Bool(key) {
|
||||
*dst = endpoints.FIPSEndpointStateEnabled
|
||||
} else {
|
||||
*dst = endpoints.FIPSEndpointStateDisabled
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
18
vendor/github.com/aws/aws-sdk-go/aws/signer/v4/v4.go
сгенерированный
поставляемый
18
vendor/github.com/aws/aws-sdk-go/aws/signer/v4/v4.go
сгенерированный
поставляемый
@@ -634,21 +634,25 @@ func (ctx *signingCtx) buildCanonicalHeaders(r rule, header http.Header) {
|
||||
ctx.Query.Set("X-Amz-SignedHeaders", ctx.signedHeaders)
|
||||
}
|
||||
|
||||
headerValues := make([]string, len(headers))
|
||||
headerItems := make([]string, len(headers))
|
||||
for i, k := range headers {
|
||||
if k == "host" {
|
||||
if ctx.Request.Host != "" {
|
||||
headerValues[i] = "host:" + ctx.Request.Host
|
||||
headerItems[i] = "host:" + ctx.Request.Host
|
||||
} else {
|
||||
headerValues[i] = "host:" + ctx.Request.URL.Host
|
||||
headerItems[i] = "host:" + ctx.Request.URL.Host
|
||||
}
|
||||
} else {
|
||||
headerValues[i] = k + ":" +
|
||||
strings.Join(ctx.SignedHeaderVals[k], ",")
|
||||
headerValues := make([]string, len(ctx.SignedHeaderVals[k]))
|
||||
for i, v := range ctx.SignedHeaderVals[k] {
|
||||
headerValues[i] = strings.TrimSpace(v)
|
||||
}
|
||||
headerItems[i] = k + ":" +
|
||||
strings.Join(headerValues, ",")
|
||||
}
|
||||
}
|
||||
stripExcessSpaces(headerValues)
|
||||
ctx.canonicalHeaders = strings.Join(headerValues, "\n")
|
||||
stripExcessSpaces(headerItems)
|
||||
ctx.canonicalHeaders = strings.Join(headerItems, "\n")
|
||||
}
|
||||
|
||||
func (ctx *signingCtx) buildCanonicalString() {
|
||||
|
||||
2
vendor/github.com/aws/aws-sdk-go/aws/version.go
сгенерированный
поставляемый
2
vendor/github.com/aws/aws-sdk-go/aws/version.go
сгенерированный
поставляемый
@@ -5,4 +5,4 @@ package aws
|
||||
const SDKName = "aws-sdk-go"
|
||||
|
||||
// SDKVersion is the version of this SDK
|
||||
const SDKVersion = "1.40.57"
|
||||
const SDKVersion = "1.42.11"
|
||||
|
||||
20
vendor/github.com/aws/aws-sdk-go/internal/ini/literal_tokens.go
сгенерированный
поставляемый
20
vendor/github.com/aws/aws-sdk-go/internal/ini/literal_tokens.go
сгенерированный
поставляемый
@@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -18,7 +19,7 @@ var literalValues = [][]rune{
|
||||
|
||||
func isBoolValue(b []rune) bool {
|
||||
for _, lv := range literalValues {
|
||||
if isLitValue(lv, b) {
|
||||
if isCaselessLitValue(lv, b) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -39,6 +40,21 @@ func isLitValue(want, have []rune) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// isCaselessLitValue is a caseless value comparison, assumes want is already lower-cased for efficiency.
|
||||
func isCaselessLitValue(want, have []rune) bool {
|
||||
if len(have) < len(want) {
|
||||
return false
|
||||
}
|
||||
|
||||
for i := 0; i < len(want); i++ {
|
||||
if want[i] != unicode.ToLower(have[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// isNumberValue will return whether not the leading characters in
|
||||
// a byte slice is a number. A number is delimited by whitespace or
|
||||
// the newline token.
|
||||
@@ -177,7 +193,7 @@ func newValue(t ValueType, base int, raw []rune) (Value, error) {
|
||||
case QuotedStringType:
|
||||
v.str = string(raw[1 : len(raw)-1])
|
||||
case BoolType:
|
||||
v.boolean = runeCompare(v.raw, runesTrue)
|
||||
v.boolean = isCaselessLitValue(runesTrue, v.raw)
|
||||
}
|
||||
|
||||
// issue 2253
|
||||
|
||||
2
vendor/github.com/aws/aws-sdk-go/internal/ini/value_util.go
сгенерированный
поставляемый
2
vendor/github.com/aws/aws-sdk-go/internal/ini/value_util.go
сгенерированный
поставляемый
@@ -57,7 +57,7 @@ func getBoolValue(b []rune) (int, error) {
|
||||
continue
|
||||
}
|
||||
|
||||
if isLitValue(lv, b) {
|
||||
if isCaselessLitValue(lv, b) {
|
||||
n = len(lv)
|
||||
}
|
||||
}
|
||||
|
||||
10
vendor/github.com/aws/aws-sdk-go/private/protocol/json/jsonutil/build.go
сгенерированный
поставляемый
10
vendor/github.com/aws/aws-sdk-go/private/protocol/json/jsonutil/build.go
сгенерированный
поставляемый
@@ -82,13 +82,17 @@ func buildStruct(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag)
|
||||
field, _ := value.Type().FieldByName(payload)
|
||||
tag = field.Tag
|
||||
value = elemOf(value.FieldByName(payload))
|
||||
|
||||
if !value.IsValid() {
|
||||
if !value.IsValid() && tag.Get("type") != "structure" {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
buf.WriteByte('{')
|
||||
defer buf.WriteString("}")
|
||||
|
||||
if !value.IsValid() {
|
||||
return nil
|
||||
}
|
||||
|
||||
t := value.Type()
|
||||
first := true
|
||||
@@ -144,8 +148,6 @@ func buildStruct(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag)
|
||||
|
||||
}
|
||||
|
||||
buf.WriteString("}")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
5
vendor/github.com/aws/aws-sdk-go/private/protocol/jsonrpc/jsonrpc.go
сгенерированный
поставляемый
5
vendor/github.com/aws/aws-sdk-go/private/protocol/jsonrpc/jsonrpc.go
сгенерированный
поставляемый
@@ -49,9 +49,8 @@ func Build(req *request.Request) {
|
||||
buf = emptyJSON
|
||||
}
|
||||
|
||||
if req.ClientInfo.TargetPrefix != "" || string(buf) != "{}" {
|
||||
req.SetBufferBody(buf)
|
||||
}
|
||||
// Always serialize the body, don't suppress it.
|
||||
req.SetBufferBody(buf)
|
||||
|
||||
if req.ClientInfo.TargetPrefix != "" {
|
||||
target := req.ClientInfo.TargetPrefix + "." + req.Operation.Name
|
||||
|
||||
11
vendor/github.com/aws/aws-sdk-go/private/protocol/rest/payload.go
сгенерированный
поставляемый
11
vendor/github.com/aws/aws-sdk-go/private/protocol/rest/payload.go
сгенерированный
поставляемый
@@ -28,18 +28,27 @@ func PayloadMember(i interface{}) interface{} {
|
||||
return nil
|
||||
}
|
||||
|
||||
// PayloadType returns the type of a payload field member of i if there is one, or "".
|
||||
const nopayloadPayloadType = "nopayload"
|
||||
|
||||
// PayloadType returns the type of a payload field member of i if there is one,
|
||||
// or "".
|
||||
func PayloadType(i interface{}) string {
|
||||
v := reflect.Indirect(reflect.ValueOf(i))
|
||||
if !v.IsValid() {
|
||||
return ""
|
||||
}
|
||||
|
||||
if field, ok := v.Type().FieldByName("_"); ok {
|
||||
if noPayload := field.Tag.Get(nopayloadPayloadType); noPayload != "" {
|
||||
return nopayloadPayloadType
|
||||
}
|
||||
|
||||
if payloadName := field.Tag.Get("payload"); payloadName != "" {
|
||||
if member, ok := v.Type().FieldByName(payloadName); ok {
|
||||
return member.Tag.Get("type")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
23
vendor/github.com/aws/aws-sdk-go/service/marketplacemetering/service.go
сгенерированный
поставляемый
23
vendor/github.com/aws/aws-sdk-go/service/marketplacemetering/service.go
сгенерированный
поставляемый
@@ -52,24 +52,25 @@ func New(p client.ConfigProvider, cfgs ...*aws.Config) *MarketplaceMetering {
|
||||
if c.SigningNameDerived || len(c.SigningName) == 0 {
|
||||
c.SigningName = "aws-marketplace"
|
||||
}
|
||||
return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName)
|
||||
return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName, c.ResolvedRegion)
|
||||
}
|
||||
|
||||
// newClient creates, initializes and returns a new service client instance.
|
||||
func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint, signingRegion, signingName string) *MarketplaceMetering {
|
||||
func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint, signingRegion, signingName, resolvedRegion string) *MarketplaceMetering {
|
||||
svc := &MarketplaceMetering{
|
||||
Client: client.New(
|
||||
cfg,
|
||||
metadata.ClientInfo{
|
||||
ServiceName: ServiceName,
|
||||
ServiceID: ServiceID,
|
||||
SigningName: signingName,
|
||||
SigningRegion: signingRegion,
|
||||
PartitionID: partitionID,
|
||||
Endpoint: endpoint,
|
||||
APIVersion: "2016-01-14",
|
||||
JSONVersion: "1.1",
|
||||
TargetPrefix: "AWSMPMeteringService",
|
||||
ServiceName: ServiceName,
|
||||
ServiceID: ServiceID,
|
||||
SigningName: signingName,
|
||||
SigningRegion: signingRegion,
|
||||
PartitionID: partitionID,
|
||||
Endpoint: endpoint,
|
||||
APIVersion: "2016-01-14",
|
||||
ResolvedRegion: resolvedRegion,
|
||||
JSONVersion: "1.1",
|
||||
TargetPrefix: "AWSMPMeteringService",
|
||||
},
|
||||
handlers,
|
||||
),
|
||||
|
||||
8
vendor/github.com/aws/aws-sdk-go/service/sso/api.go
сгенерированный
поставляемый
8
vendor/github.com/aws/aws-sdk-go/service/sso/api.go
сгенерированный
поставляемый
@@ -550,7 +550,7 @@ func (s *AccountInfo) SetEmailAddress(v string) *AccountInfo {
|
||||
}
|
||||
|
||||
type GetRoleCredentialsInput struct {
|
||||
_ struct{} `type:"structure"`
|
||||
_ struct{} `type:"structure" nopayload:"true"`
|
||||
|
||||
// The token issued by the CreateToken API call. For more information, see CreateToken
|
||||
// (https://docs.aws.amazon.com/singlesignon/latest/OIDCAPIReference/API_CreateToken.html)
|
||||
@@ -726,7 +726,7 @@ func (s *InvalidRequestException) RequestID() string {
|
||||
}
|
||||
|
||||
type ListAccountRolesInput struct {
|
||||
_ struct{} `type:"structure"`
|
||||
_ struct{} `type:"structure" nopayload:"true"`
|
||||
|
||||
// The token issued by the CreateToken API call. For more information, see CreateToken
|
||||
// (https://docs.aws.amazon.com/singlesignon/latest/OIDCAPIReference/API_CreateToken.html)
|
||||
@@ -855,7 +855,7 @@ func (s *ListAccountRolesOutput) SetRoleList(v []*RoleInfo) *ListAccountRolesOut
|
||||
}
|
||||
|
||||
type ListAccountsInput struct {
|
||||
_ struct{} `type:"structure"`
|
||||
_ struct{} `type:"structure" nopayload:"true"`
|
||||
|
||||
// The token issued by the CreateToken API call. For more information, see CreateToken
|
||||
// (https://docs.aws.amazon.com/singlesignon/latest/OIDCAPIReference/API_CreateToken.html)
|
||||
@@ -970,7 +970,7 @@ func (s *ListAccountsOutput) SetNextToken(v string) *ListAccountsOutput {
|
||||
}
|
||||
|
||||
type LogoutInput struct {
|
||||
_ struct{} `type:"structure"`
|
||||
_ struct{} `type:"structure" nopayload:"true"`
|
||||
|
||||
// The token issued by the CreateToken API call. For more information, see CreateToken
|
||||
// (https://docs.aws.amazon.com/singlesignon/latest/OIDCAPIReference/API_CreateToken.html)
|
||||
|
||||
19
vendor/github.com/aws/aws-sdk-go/service/sso/service.go
сгенерированный
поставляемый
19
vendor/github.com/aws/aws-sdk-go/service/sso/service.go
сгенерированный
поставляемый
@@ -52,22 +52,23 @@ func New(p client.ConfigProvider, cfgs ...*aws.Config) *SSO {
|
||||
if c.SigningNameDerived || len(c.SigningName) == 0 {
|
||||
c.SigningName = "awsssoportal"
|
||||
}
|
||||
return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName)
|
||||
return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName, c.ResolvedRegion)
|
||||
}
|
||||
|
||||
// newClient creates, initializes and returns a new service client instance.
|
||||
func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint, signingRegion, signingName string) *SSO {
|
||||
func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint, signingRegion, signingName, resolvedRegion string) *SSO {
|
||||
svc := &SSO{
|
||||
Client: client.New(
|
||||
cfg,
|
||||
metadata.ClientInfo{
|
||||
ServiceName: ServiceName,
|
||||
ServiceID: ServiceID,
|
||||
SigningName: signingName,
|
||||
SigningRegion: signingRegion,
|
||||
PartitionID: partitionID,
|
||||
Endpoint: endpoint,
|
||||
APIVersion: "2019-06-10",
|
||||
ServiceName: ServiceName,
|
||||
ServiceID: ServiceID,
|
||||
SigningName: signingName,
|
||||
SigningRegion: signingRegion,
|
||||
PartitionID: partitionID,
|
||||
Endpoint: endpoint,
|
||||
APIVersion: "2019-06-10",
|
||||
ResolvedRegion: resolvedRegion,
|
||||
},
|
||||
handlers,
|
||||
),
|
||||
|
||||
168
vendor/github.com/aws/aws-sdk-go/service/sts/api.go
сгенерированный
поставляемый
168
vendor/github.com/aws/aws-sdk-go/service/sts/api.go
сгенерированный
поставляемый
@@ -63,14 +63,15 @@ func (c *STS) AssumeRoleRequest(input *AssumeRoleInput) (req *request.Request, o
|
||||
// or for cross-account access. For a comparison of AssumeRole with other API
|
||||
// operations that produce temporary credentials, see Requesting Temporary Security
|
||||
// Credentials (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html)
|
||||
// and Comparing the STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
|
||||
// and Comparing the Amazon Web Services STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
|
||||
// in the IAM User Guide.
|
||||
//
|
||||
// Permissions
|
||||
//
|
||||
// The temporary security credentials created by AssumeRole can be used to make
|
||||
// API calls to any Amazon Web Services service with the following exception:
|
||||
// You cannot call the STS GetFederationToken or GetSessionToken API operations.
|
||||
// You cannot call the Amazon Web Services STS GetFederationToken or GetSessionToken
|
||||
// API operations.
|
||||
//
|
||||
// (Optional) You can pass inline or managed session policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session)
|
||||
// to this operation. You can pass a single JSON policy document to use as an
|
||||
@@ -86,26 +87,33 @@ func (c *STS) AssumeRoleRequest(input *AssumeRoleInput) (req *request.Request, o
|
||||
// For more information, see Session Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session)
|
||||
// in the IAM User Guide.
|
||||
//
|
||||
// To assume a role from a different account, your account must be trusted by
|
||||
// the role. The trust relationship is defined in the role's trust policy when
|
||||
// the role is created. That trust policy states which accounts are allowed
|
||||
// to delegate that access to users in the account.
|
||||
// When you create a role, you create two policies: A role trust policy that
|
||||
// specifies who can assume the role and a permissions policy that specifies
|
||||
// what can be done with the role. You specify the trusted principal who is
|
||||
// allowed to assume the role in the role trust policy.
|
||||
//
|
||||
// To assume a role from a different account, your Amazon Web Services account
|
||||
// must be trusted by the role. The trust relationship is defined in the role's
|
||||
// trust policy when the role is created. That trust policy states which accounts
|
||||
// are allowed to delegate that access to users in the account.
|
||||
//
|
||||
// A user who wants to access a role in a different account must also have permissions
|
||||
// that are delegated from the user account administrator. The administrator
|
||||
// must attach a policy that allows the user to call AssumeRole for the ARN
|
||||
// of the role in the other account. If the user is in the same account as the
|
||||
// role, then you can do either of the following:
|
||||
// of the role in the other account.
|
||||
//
|
||||
// * Attach a policy to the user (identical to the previous user in a different
|
||||
// account).
|
||||
// To allow a user to assume a role in the same account, you can do either of
|
||||
// the following:
|
||||
//
|
||||
// * Attach a policy to the user that allows the user to call AssumeRole
|
||||
// (as long as the role's trust policy trusts the account).
|
||||
//
|
||||
// * Add the user as a principal directly in the role's trust policy.
|
||||
//
|
||||
// In this case, the trust policy acts as an IAM resource-based policy. Users
|
||||
// in the same account as the role do not need explicit permission to assume
|
||||
// the role. For more information about trust policies and resource-based policies,
|
||||
// see IAM Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html)
|
||||
// You can do either because the role’s trust policy acts as an IAM resource-based
|
||||
// policy. When a resource-based policy grants access to a principal in the
|
||||
// same account, no additional identity-based policy is required. For more information
|
||||
// about trust policies and resource-based policies, see IAM Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html)
|
||||
// in the IAM User Guide.
|
||||
//
|
||||
// Tags
|
||||
@@ -170,7 +178,7 @@ func (c *STS) AssumeRoleRequest(input *AssumeRoleInput) (req *request.Request, o
|
||||
//
|
||||
// You could receive this error even though you meet other defined session policy
|
||||
// and session tag limits. For more information, see IAM and STS Entity Character
|
||||
// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html)
|
||||
// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html#reference_iam-limits-entity-length)
|
||||
// in the IAM User Guide.
|
||||
//
|
||||
// * ErrCodeRegionDisabledException "RegionDisabledException"
|
||||
@@ -258,7 +266,7 @@ func (c *STS) AssumeRoleWithSAMLRequest(input *AssumeRoleWithSAMLInput) (req *re
|
||||
// Services access without user-specific credentials or configuration. For a
|
||||
// comparison of AssumeRoleWithSAML with the other API operations that produce
|
||||
// temporary credentials, see Requesting Temporary Security Credentials (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html)
|
||||
// and Comparing the STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
|
||||
// and Comparing the Amazon Web Services STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
|
||||
// in the IAM User Guide.
|
||||
//
|
||||
// The temporary security credentials returned by this operation consist of
|
||||
@@ -403,7 +411,7 @@ func (c *STS) AssumeRoleWithSAMLRequest(input *AssumeRoleWithSAMLInput) (req *re
|
||||
//
|
||||
// You could receive this error even though you meet other defined session policy
|
||||
// and session tag limits. For more information, see IAM and STS Entity Character
|
||||
// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html)
|
||||
// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html#reference_iam-limits-entity-length)
|
||||
// in the IAM User Guide.
|
||||
//
|
||||
// * ErrCodeIDPRejectedClaimException "IDPRejectedClaim"
|
||||
@@ -523,7 +531,7 @@ func (c *STS) AssumeRoleWithWebIdentityRequest(input *AssumeRoleWithWebIdentityI
|
||||
// by using a token from the web identity provider. For a comparison of AssumeRoleWithWebIdentity
|
||||
// with the other API operations that produce temporary credentials, see Requesting
|
||||
// Temporary Security Credentials (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html)
|
||||
// and Comparing the STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
|
||||
// and Comparing the Amazon Web Services STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
|
||||
// in the IAM User Guide.
|
||||
//
|
||||
// The temporary security credentials returned by this API consist of an access
|
||||
@@ -661,7 +669,7 @@ func (c *STS) AssumeRoleWithWebIdentityRequest(input *AssumeRoleWithWebIdentityI
|
||||
//
|
||||
// You could receive this error even though you meet other defined session policy
|
||||
// and session tag limits. For more information, see IAM and STS Entity Character
|
||||
// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html)
|
||||
// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html#reference_iam-limits-entity-length)
|
||||
// in the IAM User Guide.
|
||||
//
|
||||
// * ErrCodeIDPRejectedClaimException "IDPRejectedClaim"
|
||||
@@ -776,10 +784,11 @@ func (c *STS) DecodeAuthorizationMessageRequest(input *DecodeAuthorizationMessag
|
||||
// code.
|
||||
//
|
||||
// The message is encoded because the details of the authorization status can
|
||||
// constitute privileged information that the user who requested the operation
|
||||
// contain privileged information that the user who requested the operation
|
||||
// should not see. To decode an authorization status message, a user must be
|
||||
// granted permissions via an IAM policy to request the DecodeAuthorizationMessage
|
||||
// (sts:DecodeAuthorizationMessage) action.
|
||||
// granted permissions through an IAM policy (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html)
|
||||
// to request the DecodeAuthorizationMessage (sts:DecodeAuthorizationMessage)
|
||||
// action.
|
||||
//
|
||||
// The decoded message includes the following type of information:
|
||||
//
|
||||
@@ -1065,7 +1074,7 @@ func (c *STS) GetFederationTokenRequest(input *GetFederationTokenInput) (req *re
|
||||
// For a comparison of GetFederationToken with the other API operations that
|
||||
// produce temporary credentials, see Requesting Temporary Security Credentials
|
||||
// (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html)
|
||||
// and Comparing the STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
|
||||
// and Comparing the Amazon Web Services STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
|
||||
// in the IAM User Guide.
|
||||
//
|
||||
// You can create a mobile-based or browser-based app that can authenticate
|
||||
@@ -1088,9 +1097,9 @@ func (c *STS) GetFederationTokenRequest(input *GetFederationTokenInput) (req *re
|
||||
//
|
||||
// The temporary credentials are valid for the specified duration, from 900
|
||||
// seconds (15 minutes) up to a maximum of 129,600 seconds (36 hours). The default
|
||||
// session duration is 43,200 seconds (12 hours). Temporary credentials that
|
||||
// are obtained by using Amazon Web Services account root user credentials have
|
||||
// a maximum duration of 3,600 seconds (1 hour).
|
||||
// session duration is 43,200 seconds (12 hours). Temporary credentials obtained
|
||||
// by using the Amazon Web Services account root user credentials have a maximum
|
||||
// duration of 3,600 seconds (1 hour).
|
||||
//
|
||||
// Permissions
|
||||
//
|
||||
@@ -1141,63 +1150,6 @@ func (c *STS) GetFederationTokenRequest(input *GetFederationTokenInput) (req *re
|
||||
// (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#api_assumerolewithwebidentity)
|
||||
// in the IAM User Guide.
|
||||
//
|
||||
// You can also call GetFederationToken using the security credentials of an
|
||||
// Amazon Web Services account root user, but we do not recommend it. Instead,
|
||||
// we recommend that you create an IAM user for the purpose of the proxy application.
|
||||
// Then attach a policy to the IAM user that limits federated users to only
|
||||
// the actions and resources that they need to access. For more information,
|
||||
// see IAM Best Practices (https://docs.aws.amazon.com/IAM/latest/UserGuide/best-practices.html)
|
||||
// in the IAM User Guide.
|
||||
//
|
||||
// Session duration
|
||||
//
|
||||
// The temporary credentials are valid for the specified duration, from 900
|
||||
// seconds (15 minutes) up to a maximum of 129,600 seconds (36 hours). The default
|
||||
// session duration is 43,200 seconds (12 hours). Temporary credentials that
|
||||
// are obtained by using Amazon Web Services account root user credentials have
|
||||
// a maximum duration of 3,600 seconds (1 hour).
|
||||
//
|
||||
// Permissions
|
||||
//
|
||||
// You can use the temporary credentials created by GetFederationToken in any
|
||||
// Amazon Web Services service except the following:
|
||||
//
|
||||
// * You cannot call any IAM operations using the CLI or the Amazon Web Services
|
||||
// API.
|
||||
//
|
||||
// * You cannot call any STS operations except GetCallerIdentity.
|
||||
//
|
||||
// You must pass an inline or managed session policy (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session)
|
||||
// to this operation. You can pass a single JSON policy document to use as an
|
||||
// inline session policy. You can also specify up to 10 managed policies to
|
||||
// use as managed session policies. The plain text that you use for both inline
|
||||
// and managed session policies can't exceed 2,048 characters.
|
||||
//
|
||||
// Though the session policy parameters are optional, if you do not pass a policy,
|
||||
// then the resulting federated user session has no permissions. When you pass
|
||||
// session policies, the session permissions are the intersection of the IAM
|
||||
// user policies and the session policies that you pass. This gives you a way
|
||||
// to further restrict the permissions for a federated user. You cannot use
|
||||
// session policies to grant more permissions than those that are defined in
|
||||
// the permissions policy of the IAM user. For more information, see Session
|
||||
// Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session)
|
||||
// in the IAM User Guide. For information about using GetFederationToken to
|
||||
// create temporary security credentials, see GetFederationToken—Federation
|
||||
// Through a Custom Identity Broker (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#api_getfederationtoken).
|
||||
//
|
||||
// You can use the credentials to access a resource that has a resource-based
|
||||
// policy. If that policy specifically references the federated user session
|
||||
// in the Principal element of the policy, the session has the permissions allowed
|
||||
// by the policy. These permissions are granted in addition to the permissions
|
||||
// granted by the session policies.
|
||||
//
|
||||
// Tags
|
||||
//
|
||||
// (Optional) You can pass tag key-value pairs to your session. These are called
|
||||
// session tags. For more information about session tags, see Passing Session
|
||||
// Tags in STS (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html)
|
||||
// in the IAM User Guide.
|
||||
//
|
||||
// An administrator must grant you the permissions necessary to pass session
|
||||
// tags. The administrator can also create granular permissions to allow you
|
||||
// to pass only specific session tags. For more information, see Tutorial: Using
|
||||
@@ -1234,7 +1186,7 @@ func (c *STS) GetFederationTokenRequest(input *GetFederationTokenInput) (req *re
|
||||
//
|
||||
// You could receive this error even though you meet other defined session policy
|
||||
// and session tag limits. For more information, see IAM and STS Entity Character
|
||||
// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html)
|
||||
// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html#reference_iam-limits-entity-length)
|
||||
// in the IAM User Guide.
|
||||
//
|
||||
// * ErrCodeRegionDisabledException "RegionDisabledException"
|
||||
@@ -1323,7 +1275,7 @@ func (c *STS) GetSessionTokenRequest(input *GetSessionTokenInput) (req *request.
|
||||
// then the API returns an access denied error. For a comparison of GetSessionToken
|
||||
// with the other API operations that produce temporary credentials, see Requesting
|
||||
// Temporary Security Credentials (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html)
|
||||
// and Comparing the STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
|
||||
// and Comparing the Amazon Web Services STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
|
||||
// in the IAM User Guide.
|
||||
//
|
||||
// Session Duration
|
||||
@@ -1404,15 +1356,23 @@ func (c *STS) GetSessionTokenWithContext(ctx aws.Context, input *GetSessionToken
|
||||
type AssumeRoleInput struct {
|
||||
_ struct{} `type:"structure"`
|
||||
|
||||
// The duration, in seconds, of the role session. The value specified can can
|
||||
// range from 900 seconds (15 minutes) up to the maximum session duration that
|
||||
// is set for the role. The maximum session duration setting can have a value
|
||||
// from 1 hour to 12 hours. If you specify a value higher than this setting
|
||||
// or the administrator setting (whichever is lower), the operation fails. For
|
||||
// example, if you specify a session duration of 12 hours, but your administrator
|
||||
// set the maximum session duration to 6 hours, your operation fails. To learn
|
||||
// how to view the maximum value for your role, see View the Maximum Session
|
||||
// Duration Setting for a Role (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html#id_roles_use_view-role-max-session)
|
||||
// The duration, in seconds, of the role session. The value specified can range
|
||||
// from 900 seconds (15 minutes) up to the maximum session duration set for
|
||||
// the role. The maximum session duration setting can have a value from 1 hour
|
||||
// to 12 hours. If you specify a value higher than this setting or the administrator
|
||||
// setting (whichever is lower), the operation fails. For example, if you specify
|
||||
// a session duration of 12 hours, but your administrator set the maximum session
|
||||
// duration to 6 hours, your operation fails.
|
||||
//
|
||||
// Role chaining limits your Amazon Web Services CLI or Amazon Web Services
|
||||
// API role session to a maximum of one hour. When you use the AssumeRole API
|
||||
// operation to assume a role, you can specify the duration of your role session
|
||||
// with the DurationSeconds parameter. You can specify a parameter value of
|
||||
// up to 43200 seconds (12 hours), depending on the maximum session duration
|
||||
// setting for your role. However, if you assume a role using role chaining
|
||||
// and provide a DurationSeconds parameter value greater than one hour, the
|
||||
// operation fails. To learn how to view the maximum value for your role, see
|
||||
// View the Maximum Session Duration Setting for a Role (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html#id_roles_use_view-role-max-session)
|
||||
// in the IAM User Guide.
|
||||
//
|
||||
// By default, the value is set to 3600 seconds.
|
||||
@@ -1422,7 +1382,7 @@ type AssumeRoleInput struct {
|
||||
// to the federation endpoint for a console sign-in token takes a SessionDuration
|
||||
// parameter that specifies the maximum length of the console session. For more
|
||||
// information, see Creating a URL that Enables Federated Users to Access the
|
||||
// Management Console (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_enable-console-custom-url.html)
|
||||
// Amazon Web Services Management Console (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_enable-console-custom-url.html)
|
||||
// in the IAM User Guide.
|
||||
DurationSeconds *int64 `min:"900" type:"integer"`
|
||||
|
||||
@@ -1548,7 +1508,7 @@ type AssumeRoleInput struct {
|
||||
|
||||
// A list of session tags that you want to pass. Each session tag consists of
|
||||
// a key name and an associated value. For more information about session tags,
|
||||
// see Tagging STS Sessions (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html)
|
||||
// see Tagging Amazon Web Services STS Sessions (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html)
|
||||
// in the IAM User Guide.
|
||||
//
|
||||
// This parameter is optional. You can pass up to 50 session tags. The plaintext
|
||||
@@ -1858,7 +1818,7 @@ type AssumeRoleWithSAMLInput struct {
|
||||
// to the federation endpoint for a console sign-in token takes a SessionDuration
|
||||
// parameter that specifies the maximum length of the console session. For more
|
||||
// information, see Creating a URL that Enables Federated Users to Access the
|
||||
// Management Console (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_enable-console-custom-url.html)
|
||||
// Amazon Web Services Management Console (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_enable-console-custom-url.html)
|
||||
// in the IAM User Guide.
|
||||
DurationSeconds *int64 `min:"900" type:"integer"`
|
||||
|
||||
@@ -2205,7 +2165,7 @@ type AssumeRoleWithWebIdentityInput struct {
|
||||
// to the federation endpoint for a console sign-in token takes a SessionDuration
|
||||
// parameter that specifies the maximum length of the console session. For more
|
||||
// information, see Creating a URL that Enables Federated Users to Access the
|
||||
// Management Console (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_enable-console-custom-url.html)
|
||||
// Amazon Web Services Management Console (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_enable-console-custom-url.html)
|
||||
// in the IAM User Guide.
|
||||
DurationSeconds *int64 `min:"900" type:"integer"`
|
||||
|
||||
@@ -2705,7 +2665,7 @@ func (s *DecodeAuthorizationMessageInput) SetEncodedMessage(v string) *DecodeAut
|
||||
type DecodeAuthorizationMessageOutput struct {
|
||||
_ struct{} `type:"structure"`
|
||||
|
||||
// An XML document that contains the decoded message.
|
||||
// The API returns a response with the decoded message.
|
||||
DecodedMessage *string `type:"string"`
|
||||
}
|
||||
|
||||
@@ -3237,8 +3197,8 @@ type GetSessionTokenInput struct {
|
||||
// user has a policy that requires MFA authentication. The value is either the
|
||||
// serial number for a hardware device (such as GAHT12345678) or an Amazon Resource
|
||||
// Name (ARN) for a virtual device (such as arn:aws:iam::123456789012:mfa/user).
|
||||
// You can find the device for an IAM user by going to the Management Console
|
||||
// and viewing the user's security credentials.
|
||||
// You can find the device for an IAM user by going to the Amazon Web Services
|
||||
// Management Console and viewing the user's security credentials.
|
||||
//
|
||||
// The regex used to validate this parameter is a string of characters consisting
|
||||
// of upper- and lower-case alphanumeric characters with no spaces. You can
|
||||
@@ -3400,9 +3360,9 @@ func (s *PolicyDescriptorType) SetArn(v string) *PolicyDescriptorType {
|
||||
|
||||
// You can pass custom key-value pair attributes when you assume a role or federate
|
||||
// a user. These are called session tags. You can then use the session tags
|
||||
// to control access to resources. For more information, see Tagging STS Sessions
|
||||
// (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html) in
|
||||
// the IAM User Guide.
|
||||
// to control access to resources. For more information, see Tagging Amazon
|
||||
// Web Services STS Sessions (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html)
|
||||
// in the IAM User Guide.
|
||||
type Tag struct {
|
||||
_ struct{} `type:"structure"`
|
||||
|
||||
|
||||
2
vendor/github.com/aws/aws-sdk-go/service/sts/errors.go
сгенерированный
поставляемый
2
vendor/github.com/aws/aws-sdk-go/service/sts/errors.go
сгенерированный
поставляемый
@@ -67,7 +67,7 @@ const (
|
||||
//
|
||||
// You could receive this error even though you meet other defined session policy
|
||||
// and session tag limits. For more information, see IAM and STS Entity Character
|
||||
// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html)
|
||||
// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-quotas.html#reference_iam-limits-entity-length)
|
||||
// in the IAM User Guide.
|
||||
ErrCodePackedPolicyTooLargeException = "PackedPolicyTooLarge"
|
||||
|
||||
|
||||
19
vendor/github.com/aws/aws-sdk-go/service/sts/service.go
сгенерированный
поставляемый
19
vendor/github.com/aws/aws-sdk-go/service/sts/service.go
сгенерированный
поставляемый
@@ -48,22 +48,23 @@ const (
|
||||
// svc := sts.New(mySession, aws.NewConfig().WithRegion("us-west-2"))
|
||||
func New(p client.ConfigProvider, cfgs ...*aws.Config) *STS {
|
||||
c := p.ClientConfig(EndpointsID, cfgs...)
|
||||
return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName)
|
||||
return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName, c.ResolvedRegion)
|
||||
}
|
||||
|
||||
// newClient creates, initializes and returns a new service client instance.
|
||||
func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint, signingRegion, signingName string) *STS {
|
||||
func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint, signingRegion, signingName, resolvedRegion string) *STS {
|
||||
svc := &STS{
|
||||
Client: client.New(
|
||||
cfg,
|
||||
metadata.ClientInfo{
|
||||
ServiceName: ServiceName,
|
||||
ServiceID: ServiceID,
|
||||
SigningName: signingName,
|
||||
SigningRegion: signingRegion,
|
||||
PartitionID: partitionID,
|
||||
Endpoint: endpoint,
|
||||
APIVersion: "2011-06-15",
|
||||
ServiceName: ServiceName,
|
||||
ServiceID: ServiceID,
|
||||
SigningName: signingName,
|
||||
SigningRegion: signingRegion,
|
||||
PartitionID: partitionID,
|
||||
Endpoint: endpoint,
|
||||
APIVersion: "2011-06-15",
|
||||
ResolvedRegion: resolvedRegion,
|
||||
},
|
||||
handlers,
|
||||
),
|
||||
|
||||
3
vendor/github.com/dsnet/compress/go.mod
сгенерированный
поставляемый
3
vendor/github.com/dsnet/compress/go.mod
сгенерированный
поставляемый
@@ -4,7 +4,8 @@ go 1.9
|
||||
|
||||
require (
|
||||
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780
|
||||
github.com/google/go-cmp v0.5.5 // indirect
|
||||
github.com/klauspost/compress v1.4.1
|
||||
github.com/klauspost/cpuid v1.2.0 // indirect
|
||||
github.com/ulikunitz/xz v0.5.6
|
||||
github.com/ulikunitz/xz v0.5.8
|
||||
)
|
||||
|
||||
8
vendor/github.com/dsnet/compress/go.sum
сгенерированный
поставляемый
8
vendor/github.com/dsnet/compress/go.sum
сгенерированный
поставляемый
@@ -1,8 +1,12 @@
|
||||
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780 h1:tFh1tRc4CA31yP6qDcu+Trax5wW5GuMxvkIba07qVLY=
|
||||
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
|
||||
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E=
|
||||
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
|
||||
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8=
|
||||
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
|
||||
github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ=
|
||||
github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
||||
23
vendor/github.com/fcjr/aia-transport-go/.travis.yml
сгенерированный
поставляемый
Обычный файл
23
vendor/github.com/fcjr/aia-transport-go/.travis.yml
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,23 @@
|
||||
language: go
|
||||
|
||||
go:
|
||||
- "1.13.x"
|
||||
- "1.x"
|
||||
# - tip
|
||||
|
||||
os:
|
||||
- linux
|
||||
- osx
|
||||
- windows
|
||||
|
||||
before_install:
|
||||
- go get -t -v ./...
|
||||
|
||||
before_script:
|
||||
- go vet ./...
|
||||
|
||||
script:
|
||||
- go test -race -coverprofile=coverage.txt -covermode=atomic
|
||||
|
||||
after_success:
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
21
vendor/github.com/fcjr/aia-transport-go/LICENSE
сгенерированный
поставляемый
Обычный файл
21
vendor/github.com/fcjr/aia-transport-go/LICENSE
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2020 Frank Chiarulli Jr.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
55
vendor/github.com/fcjr/aia-transport-go/README.md
сгенерированный
поставляемый
Обычный файл
55
vendor/github.com/fcjr/aia-transport-go/README.md
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,55 @@
|
||||
# AIA (Authority Information Access) Transport Go
|
||||
[![GoDoc][doc-img]][doc] [![Build Status][ci-img]][ci] [![Go Report Card][report-card-img]][report-card] [![GolangCI][golangci-lint-img]][golangci-lint] [![Coverage Status][cov-img]][cov]
|
||||
|
||||
AIA-Transport-Go provides an http.Transport which uses the AIA (Authority Information Access) X.509 extension to resolve incomplete certificate chains during the tls handshake. See [rfc3280](https://tools.ietf.org/html/rfc3280#section-4.2.2.1) for more details.
|
||||
|
||||
## Installation
|
||||
|
||||
|
||||
`go get github.com/fcjr/aia-transport-go`
|
||||
|
||||
|
||||
## Usage
|
||||
|
||||
```go
|
||||
tr, err := aia.NewTransport()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
client := http.Client{
|
||||
Transport: tr,
|
||||
}
|
||||
res, err := client.Get("https://incomplete-chain.badssl.com/")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Println(res.Status)
|
||||
```
|
||||
|
||||
## Todos
|
||||
|
||||
* [X] Follow single incomplete AIA chain
|
||||
* [X] Tests
|
||||
* [X] CI & Code Coverage
|
||||
* [X] Documentation
|
||||
* [X] Chain Caching
|
||||
* [ ] Certificate Caching
|
||||
* [ ] Follow all possible issuing urls
|
||||
* [ ] Benchmarks
|
||||
|
||||
## Caveats
|
||||
|
||||
This library relies on [x509.SystemCertPool()](https://golang.org/pkg/crypto/x509/#SystemCertPool) to gather the initial system root certificates to validate against. This function is not implemented on windows, however windows will resolve incomplete certificate chains via AIA automatically so this library simply returns a default http.Transport on windows which should be sufficient.[^1]
|
||||
|
||||
[^1]: https://github.com/golang/go/issues/31773#issuecomment-582176197
|
||||
|
||||
[doc-img]: https://img.shields.io/static/v1?label=godoc&message=reference&color=blue
|
||||
[doc]: https://pkg.go.dev/github.com/fcjr/aia-transport-go?tab=doc
|
||||
[ci-img]: https://travis-ci.org/fcjr/aia-transport-go.svg?branch=master
|
||||
[ci]: https://travis-ci.org/fcjr/aia-transport-go
|
||||
[report-card-img]: https://goreportcard.com/badge/github.com/fcjr/aia-transport-go
|
||||
[report-card]: https://goreportcard.com/report/github.com/fcjr/aia-transport-go
|
||||
[golangci-lint-img]: https://github.com/fcjr/aia-transport-go/workflows/golangci-lint/badge.svg
|
||||
[golangci-lint]: https://github.com/fcjr/aia-transport-go/actions?query=workflow%3Agolangci-lint
|
||||
[cov-img]: https://codecov.io/gh/fcjr/aia-transport-go/branch/master/graph/badge.svg
|
||||
[cov]: https://codecov.io/gh/fcjr/aia-transport-go
|
||||
15
vendor/github.com/fcjr/aia-transport-go/codecov.yml
сгенерированный
поставляемый
Обычный файл
15
vendor/github.com/fcjr/aia-transport-go/codecov.yml
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,15 @@
|
||||
codecov:
|
||||
require_ci_to_pass: yes
|
||||
notify:
|
||||
after_n_builds: 6
|
||||
wait_for_ci: yes
|
||||
|
||||
coverage:
|
||||
range: 60..80
|
||||
status:
|
||||
project:
|
||||
default:
|
||||
threshold: 1 # check will fail if coverage drops >1%
|
||||
patch:
|
||||
default:
|
||||
threshold: 1 # check will fail if coverage drops >1%
|
||||
22
vendor/github.com/fcjr/aia-transport-go/doc.go
сгенерированный
поставляемый
Обычный файл
22
vendor/github.com/fcjr/aia-transport-go/doc.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,22 @@
|
||||
// Package aia provides an http.Transport which uses the AIA (Authority Information Access)
|
||||
// X.509 extension to resolve incomplete certificate chains during the tls handshake.
|
||||
// See https://tools.ietf.org/html/rfc3280#section-4.2.2.1 for more details.
|
||||
//
|
||||
// Usage
|
||||
//
|
||||
// To use simply create a new transport via NewTransport()
|
||||
// and use it in your http.Client.
|
||||
//
|
||||
// tr, err := aia.NewTransport()
|
||||
// if err != nil {
|
||||
// log.Fatal(err)
|
||||
// }
|
||||
// client := http.Client{
|
||||
// Transport: tr,
|
||||
// }
|
||||
// res, err := client.Get("https://incomplete-chain.badssl.com/")
|
||||
// if err != nil {
|
||||
// log.Fatal(err)
|
||||
// }
|
||||
// fmt.Println(res.Status)
|
||||
package aia //import "github.com/fcjr/aia-transport-go"
|
||||
3
vendor/github.com/fcjr/aia-transport-go/go.mod
сгенерированный
поставляемый
Обычный файл
3
vendor/github.com/fcjr/aia-transport-go/go.mod
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,3 @@
|
||||
module github.com/fcjr/aia-transport-go
|
||||
|
||||
go 1.13
|
||||
117
vendor/github.com/fcjr/aia-transport-go/transport.go
сгенерированный
поставляемый
Обычный файл
117
vendor/github.com/fcjr/aia-transport-go/transport.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,117 @@
|
||||
package aia
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"time"
|
||||
)
|
||||
|
||||
// NewTransport returns a http.Transport that supports AIA (Authority Information Access) resolution
|
||||
// for incomplete certificate chains.
|
||||
func NewTransport() (*http.Transport, error) {
|
||||
|
||||
// Support windows.
|
||||
if runtime.GOOS == "windows" {
|
||||
return &http.Transport{}, nil
|
||||
}
|
||||
|
||||
rootCAs, err := x509.SystemCertPool()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &http.Transport{
|
||||
TLSClientConfig: &tls.Config{
|
||||
RootCAs: rootCAs,
|
||||
},
|
||||
DialTLS: func(network, addr string) (net.Conn, error) {
|
||||
conn, err := tls.Dial(network, addr, &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
RootCAs: rootCAs,
|
||||
VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
|
||||
serverName, _, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return verifyPeerCerts(rootCAs, serverName, rawCerts)
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return conn, err
|
||||
}
|
||||
return conn, nil
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func verifyPeerCerts(rootCAs *x509.CertPool, serverName string, rawCerts [][]byte) error {
|
||||
certs := make([]*x509.Certificate, len(rawCerts))
|
||||
for i, asn1Data := range rawCerts {
|
||||
cert, err := x509.ParseCertificate(asn1Data)
|
||||
if err != nil {
|
||||
return errors.New("failed to parse certificate from server: " + err.Error())
|
||||
}
|
||||
certs[i] = cert
|
||||
}
|
||||
|
||||
opts := &x509.VerifyOptions{
|
||||
Roots: rootCAs,
|
||||
CurrentTime: time.Now(),
|
||||
DNSName: serverName,
|
||||
Intermediates: x509.NewCertPool(),
|
||||
}
|
||||
for _, cert := range certs[1:] {
|
||||
opts.Intermediates.AddCert(cert)
|
||||
}
|
||||
|
||||
_, err := certs[0].Verify(*opts)
|
||||
if err != nil {
|
||||
if _, ok := err.(x509.UnknownAuthorityError); ok {
|
||||
if len(certs[0].IssuingCertificateURL) >= 1 && certs[0].IssuingCertificateURL[0] != "" {
|
||||
return verifyIncompleteChain(certs[0].IssuingCertificateURL[0], certs[0], opts)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyIncompleteChain(issuingCertificateURL string, baseCert *x509.Certificate, opts *x509.VerifyOptions) error {
|
||||
issuer, err := getCert(issuingCertificateURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
opts.Intermediates.AddCert(issuer)
|
||||
_, err = baseCert.Verify(*opts)
|
||||
if err != nil {
|
||||
if _, ok := err.(x509.UnknownAuthorityError); ok {
|
||||
if len(issuer.IssuingCertificateURL) >= 1 && issuer.IssuingCertificateURL[0] != "" {
|
||||
return verifyIncompleteChain(issuer.IssuingCertificateURL[0], baseCert, opts)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getCert(url string) (*x509.Certificate, error) {
|
||||
resp, err := http.Get(url)
|
||||
if resp != nil {
|
||||
defer resp.Body.Close()
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
data, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return x509.ParseCertificate(data)
|
||||
}
|
||||
21
vendor/github.com/go-resty/resty/v2/.travis.yml
сгенерированный
поставляемый
21
vendor/github.com/go-resty/resty/v2/.travis.yml
сгенерированный
поставляемый
@@ -1,21 +0,0 @@
|
||||
language: go
|
||||
|
||||
os: linux
|
||||
|
||||
go: # use travis ci resource effectively, keep always latest 2 versions and tip :)
|
||||
- 1.15.x
|
||||
- 1.14.x
|
||||
- tip
|
||||
|
||||
install:
|
||||
- go get -v -t ./...
|
||||
|
||||
script:
|
||||
- go test ./... -race -coverprofile=coverage.txt -covermode=atomic
|
||||
|
||||
after_success:
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
jobs:
|
||||
allow_failures:
|
||||
- go: tip
|
||||
60
vendor/github.com/go-resty/resty/v2/BUILD.bazel
сгенерированный
поставляемый
60
vendor/github.com/go-resty/resty/v2/BUILD.bazel
сгенерированный
поставляемый
@@ -1,36 +1,48 @@
|
||||
package(default_visibility = ["//visibility:private"])
|
||||
|
||||
load("@bazel_gazelle//:def.bzl", "gazelle")
|
||||
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
|
||||
load("@bazel_gazelle//:def.bzl", "gazelle")
|
||||
|
||||
gazelle(
|
||||
name = "gazelle",
|
||||
command = "fix",
|
||||
prefix = "github.com/go-resty/resty/v2",
|
||||
)
|
||||
# gazelle:prefix github.com/go-resty/resty/v2
|
||||
# gazelle:go_naming_convention import_alias
|
||||
gazelle(name = "gazelle")
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = glob(
|
||||
["*.go"],
|
||||
exclude = ["*_test.go"],
|
||||
),
|
||||
name = "resty",
|
||||
srcs = [
|
||||
"client.go",
|
||||
"middleware.go",
|
||||
"redirect.go",
|
||||
"request.go",
|
||||
"response.go",
|
||||
"resty.go",
|
||||
"retry.go",
|
||||
"trace.go",
|
||||
"transport.go",
|
||||
"transport112.go",
|
||||
"util.go",
|
||||
],
|
||||
importpath = "github.com/go-resty/resty/v2",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["@org_golang_x_net//publicsuffix:go_default_library"],
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs =
|
||||
glob(
|
||||
["*_test.go"],
|
||||
exclude = ["example_test.go"],
|
||||
),
|
||||
data = glob([".testdata/*"]),
|
||||
embed = [":go_default_library"],
|
||||
importpath = "github.com/go-resty/resty/v2",
|
||||
deps = [
|
||||
"@org_golang_x_net//proxy:go_default_library",
|
||||
name = "resty_test",
|
||||
srcs = [
|
||||
"client_test.go",
|
||||
"context_test.go",
|
||||
"example_test.go",
|
||||
"request_test.go",
|
||||
"resty_test.go",
|
||||
"retry_test.go",
|
||||
"util_test.go",
|
||||
],
|
||||
data = glob([".testdata/*"]),
|
||||
embed = [":resty"],
|
||||
deps = ["@org_golang_x_net//proxy:go_default_library"],
|
||||
)
|
||||
|
||||
alias(
|
||||
name = "go_default_library",
|
||||
actual = ":resty",
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
31
vendor/github.com/go-resty/resty/v2/README.md
сгенерированный
поставляемый
31
vendor/github.com/go-resty/resty/v2/README.md
сгенерированный
поставляемый
@@ -4,7 +4,7 @@
|
||||
<p align="center"><a href="#features">Features</a> section describes in detail about Resty capabilities</p>
|
||||
</p>
|
||||
<p align="center">
|
||||
<p align="center"><a href="https://travis-ci.org/go-resty/resty"><img src="https://travis-ci.org/go-resty/resty.svg?branch=master" alt="Build Status"></a> <a href="https://codecov.io/gh/go-resty/resty/branch/master"><img src="https://codecov.io/gh/go-resty/resty/branch/master/graph/badge.svg" alt="Code Coverage"></a> <a href="https://goreportcard.com/report/go-resty/resty"><img src="https://goreportcard.com/badge/go-resty/resty" alt="Go Report Card"></a> <a href="https://github.com/go-resty/resty/releases/latest"><img src="https://img.shields.io/badge/version-2.6.0-blue.svg" alt="Release Version"></a> <a href="https://pkg.go.dev/github.com/go-resty/resty/v2"><img src="https://pkg.go.dev/badge/github.com/go-resty/resty" alt="GoDoc"></a> <a href="LICENSE"><img src="https://img.shields.io/github/license/go-resty/resty.svg" alt="License"></a> <a href="https://github.com/avelino/awesome-go"><img src="https://awesome.re/mentioned-badge.svg" alt="Mentioned in Awesome Go"></a></p>
|
||||
<p align="center"><a href="https://github.com/go-resty/resty/actions/workflows/ci.yml?query=branch%3Amaster"><img src="https://github.com/go-resty/resty/actions/workflows/ci.yml/badge.svg" alt="Build Status"></a> <a href="https://codecov.io/gh/go-resty/resty/branch/master"><img src="https://codecov.io/gh/go-resty/resty/branch/master/graph/badge.svg" alt="Code Coverage"></a> <a href="https://goreportcard.com/report/go-resty/resty"><img src="https://goreportcard.com/badge/go-resty/resty" alt="Go Report Card"></a> <a href="https://github.com/go-resty/resty/releases/latest"><img src="https://img.shields.io/badge/version-2.7.0-blue.svg" alt="Release Version"></a> <a href="https://pkg.go.dev/github.com/go-resty/resty/v2"><img src="https://pkg.go.dev/badge/github.com/go-resty/resty" alt="GoDoc"></a> <a href="LICENSE"><img src="https://img.shields.io/github/license/go-resty/resty.svg" alt="License"></a> <a href="https://github.com/avelino/awesome-go"><img src="https://awesome.re/mentioned-badge.svg" alt="Mentioned in Awesome Go"></a></p>
|
||||
</p>
|
||||
<p align="center">
|
||||
<h4 align="center">Resty Communication Channels</h4>
|
||||
@@ -13,7 +13,7 @@
|
||||
|
||||
## News
|
||||
|
||||
* v2.6.0 [released](https://github.com/go-resty/resty/releases/tag/v2.6.0) and tagged on Apr 09, 2021.
|
||||
* v2.7.0 [released](https://github.com/go-resty/resty/releases/tag/v2.7.0) and tagged on Nov 03, 2021.
|
||||
* v2.0.0 [released](https://github.com/go-resty/resty/releases/tag/v2.0.0) and tagged on Jul 16, 2019.
|
||||
* v1.12.0 [released](https://github.com/go-resty/resty/releases/tag/v1.12.0) and tagged on Feb 27, 2019.
|
||||
* v1.0 released and tagged on Sep 25, 2017. - Resty's first version was released on Sep 15, 2015 then it grew gradually as a very handy and helpful library. Its been a two years since first release. I'm very thankful to Resty users and its [contributors](https://github.com/go-resty/resty/graphs/contributors).
|
||||
@@ -36,6 +36,7 @@
|
||||
- Success scenario [Request.SetResult()](https://pkg.go.dev/github.com/go-resty/resty/v2#Request.SetResult) and [Response.Result()](https://pkg.go.dev/github.com/go-resty/resty/v2#Response.Result).
|
||||
- Error scenario [Request.SetError()](https://pkg.go.dev/github.com/go-resty/resty/v2#Request.SetError) and [Response.Error()](https://pkg.go.dev/github.com/go-resty/resty/v2#Response.Error).
|
||||
- Supports [RFC7807](https://tools.ietf.org/html/rfc7807) - `application/problem+json` & `application/problem+xml`
|
||||
* Resty provides an option to override [JSON Marshal/Unmarshal and XML Marshal/Unmarshal](#override-json--xml-marshalunmarshal)
|
||||
* Easy to upload one or more file(s) via `multipart/form-data`
|
||||
* Auto detects file content type
|
||||
* Request URL [Path Params (aka URI Params)](https://pkg.go.dev/github.com/go-resty/resty/v2#Request.SetPathParams)
|
||||
@@ -107,7 +108,7 @@ Resty author also published following projects for Go Community.
|
||||
|
||||
```bash
|
||||
# Go Modules
|
||||
require github.com/go-resty/resty/v2 v2.4.0
|
||||
require github.com/go-resty/resty/v2 v2.7.0
|
||||
```
|
||||
|
||||
## Usage
|
||||
@@ -359,6 +360,24 @@ resp, err := client.R().
|
||||
Options("https://myapp.com/servers/nyc-dc-01")
|
||||
```
|
||||
|
||||
#### Override JSON & XML Marshal/Unmarshal
|
||||
|
||||
User could register choice of JSON/XML library into resty or write your own. By default resty registers standard `encoding/json` and `encoding/xml` respectively.
|
||||
```go
|
||||
// Example of registering json-iterator
|
||||
import jsoniter "github.com/json-iterator/go"
|
||||
|
||||
json := jsoniter.ConfigCompatibleWithStandardLibrary
|
||||
|
||||
client := resty.New()
|
||||
client.JSONMarshal = json.Marshal
|
||||
client.JSONUnmarshal = json.Unmarshal
|
||||
|
||||
// similarly user could do for XML too with -
|
||||
client.XMLMarshal
|
||||
client.XMLUnmarshal
|
||||
```
|
||||
|
||||
### Multipart File(s) upload
|
||||
|
||||
#### Using io.Reader
|
||||
@@ -829,13 +848,13 @@ client.SetTransport(&transport).SetScheme("http").SetHostURL(unixSocket)
|
||||
client.R().Get("/index.html")
|
||||
```
|
||||
|
||||
#### Bazel support
|
||||
#### Bazel Support
|
||||
|
||||
Resty can be built, tested and depended upon via [Bazel](https://bazel.build).
|
||||
For example, to run all tests:
|
||||
|
||||
```shell
|
||||
bazel test :go_default_test
|
||||
bazel test :resty_test
|
||||
```
|
||||
|
||||
#### Mocking http requests using [httpmock](https://github.com/jarcoal/httpmock) library
|
||||
@@ -876,7 +895,7 @@ BTW, I'd like to know what you think about `Resty`. Kindly open an issue or send
|
||||
|
||||
## Core Team
|
||||
|
||||
Have a look on [Members](https://github.com/orgs/go-resty/teams/core/members) page.
|
||||
Have a look on [Members](https://github.com/orgs/go-resty/people) page.
|
||||
|
||||
## Contributors
|
||||
|
||||
|
||||
28
vendor/github.com/go-resty/resty/v2/WORKSPACE
сгенерированный
поставляемый
28
vendor/github.com/go-resty/resty/v2/WORKSPACE
сгенерированный
поставляемый
@@ -1,26 +1,30 @@
|
||||
workspace(name = "resty")
|
||||
|
||||
git_repository(
|
||||
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
|
||||
|
||||
http_archive(
|
||||
name = "io_bazel_rules_go",
|
||||
remote = "https://github.com/bazelbuild/rules_go.git",
|
||||
tag = "0.13.0",
|
||||
sha256 = "69de5c704a05ff37862f7e0f5534d4f479418afc21806c887db544a316f3cb6b",
|
||||
urls = [
|
||||
"https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.27.0/rules_go-v0.27.0.tar.gz",
|
||||
"https://github.com/bazelbuild/rules_go/releases/download/v0.27.0/rules_go-v0.27.0.tar.gz",
|
||||
],
|
||||
)
|
||||
|
||||
git_repository(
|
||||
http_archive(
|
||||
name = "bazel_gazelle",
|
||||
remote = "https://github.com/bazelbuild/bazel-gazelle.git",
|
||||
tag = "0.13.0",
|
||||
sha256 = "62ca106be173579c0a167deb23358fdfe71ffa1e4cfdddf5582af26520f1c66f",
|
||||
urls = [
|
||||
"https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.23.0/bazel-gazelle-v0.23.0.tar.gz",
|
||||
"https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.23.0/bazel-gazelle-v0.23.0.tar.gz",
|
||||
],
|
||||
)
|
||||
|
||||
load(
|
||||
"@io_bazel_rules_go//go:def.bzl",
|
||||
"go_rules_dependencies",
|
||||
"go_register_toolchains",
|
||||
)
|
||||
load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
|
||||
|
||||
go_rules_dependencies()
|
||||
|
||||
go_register_toolchains()
|
||||
go_register_toolchains(version = "1.16")
|
||||
|
||||
load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
|
||||
|
||||
|
||||
50
vendor/github.com/go-resty/resty/v2/client.go
сгенерированный
поставляемый
50
vendor/github.com/go-resty/resty/v2/client.go
сгенерированный
поставляемый
@@ -10,6 +10,7 @@ import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -92,9 +93,11 @@ type (
|
||||
// Resty also provides an options to override most of the client settings
|
||||
// at request level.
|
||||
type Client struct {
|
||||
HostURL string
|
||||
BaseURL string
|
||||
HostURL string // Deprecated: use BaseURL instead. To be removed in v3.0.0 release.
|
||||
QueryParam url.Values
|
||||
FormData url.Values
|
||||
PathParams map[string]string
|
||||
Header http.Header
|
||||
UserInfo *User
|
||||
Token string
|
||||
@@ -112,6 +115,8 @@ type Client struct {
|
||||
RetryAfter RetryAfterFunc
|
||||
JSONMarshal func(v interface{}) ([]byte, error)
|
||||
JSONUnmarshal func(data []byte, v interface{}) error
|
||||
XMLMarshal func(v interface{}) ([]byte, error)
|
||||
XMLUnmarshal func(data []byte, v interface{}) error
|
||||
|
||||
// HeaderAuthorizationKey is used to set/access Request Authorization header
|
||||
// value when `SetAuthToken` option is used.
|
||||
@@ -125,7 +130,6 @@ type Client struct {
|
||||
debugBodySizeLimit int64
|
||||
outputDirectory string
|
||||
scheme string
|
||||
pathParams map[string]string
|
||||
log Logger
|
||||
httpClient *http.Client
|
||||
proxyURL *url.URL
|
||||
@@ -154,8 +158,25 @@ type User struct {
|
||||
//
|
||||
// // Setting HTTPS address
|
||||
// client.SetHostURL("https://myjeeva.com")
|
||||
//
|
||||
// Deprecated: use SetBaseURL instead. To be removed in v3.0.0 release.
|
||||
func (c *Client) SetHostURL(url string) *Client {
|
||||
c.HostURL = strings.TrimRight(url, "/")
|
||||
c.SetBaseURL(url)
|
||||
return c
|
||||
}
|
||||
|
||||
// SetBaseURL method is to set Base URL in the client instance. It will be used with request
|
||||
// raised from this client with relative URL
|
||||
// // Setting HTTP address
|
||||
// client.SetBaseURL("http://myjeeva.com")
|
||||
//
|
||||
// // Setting HTTPS address
|
||||
// client.SetBaseURL("https://myjeeva.com")
|
||||
//
|
||||
// Since v2.7.0
|
||||
func (c *Client) SetBaseURL(url string) *Client {
|
||||
c.BaseURL = strings.TrimRight(url, "/")
|
||||
c.HostURL = c.BaseURL
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -367,7 +388,7 @@ func (c *Client) R() *Request {
|
||||
client: c,
|
||||
multipartFiles: []*File{},
|
||||
multipartFields: []*MultipartField{},
|
||||
pathParams: map[string]string{},
|
||||
PathParams: map[string]string{},
|
||||
jsonEscapeHTML: true,
|
||||
}
|
||||
return r
|
||||
@@ -589,6 +610,9 @@ func (c *Client) SetRetryAfter(callback RetryAfterFunc) *Client {
|
||||
// AddRetryCondition method adds a retry condition function to array of functions
|
||||
// that are checked to determine if the request is retried. The request will
|
||||
// retry if any of the functions return true and error is nil.
|
||||
//
|
||||
// Note: These retry conditions are applied on all Request made using this Client.
|
||||
// For Request specific retry conditions check *Request.AddRetryCondition
|
||||
func (c *Client) AddRetryCondition(condition RetryConditionFunc) *Client {
|
||||
c.RetryConditions = append(c.RetryConditions, condition)
|
||||
return c
|
||||
@@ -758,7 +782,7 @@ func (c *Client) SetTransport(transport http.RoundTripper) *Client {
|
||||
// client.SetScheme("http")
|
||||
func (c *Client) SetScheme(scheme string) *Client {
|
||||
if !IsStringEmpty(scheme) {
|
||||
c.scheme = scheme
|
||||
c.scheme = strings.TrimSpace(scheme)
|
||||
}
|
||||
return c
|
||||
}
|
||||
@@ -793,7 +817,7 @@ func (c *Client) SetDoNotParseResponse(parse bool) *Client {
|
||||
// Also it can be overridden at request level Path Params options,
|
||||
// see `Request.SetPathParam` or `Request.SetPathParams`.
|
||||
func (c *Client) SetPathParam(param, value string) *Client {
|
||||
c.pathParams[param] = value
|
||||
c.PathParams[param] = value
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -869,7 +893,6 @@ func (c *Client) GetClient() *http.Client {
|
||||
// Executes method executes the given `Request` object and returns response
|
||||
// error.
|
||||
func (c *Client) execute(req *Request) (*Response, error) {
|
||||
defer releaseBuffer(req.bodyBuf)
|
||||
// Apply Request middleware
|
||||
var err error
|
||||
|
||||
@@ -903,6 +926,8 @@ func (c *Client) execute(req *Request) (*Response, error) {
|
||||
return nil, wrapNoRetryErr(err)
|
||||
}
|
||||
|
||||
req.RawRequest.Body = newRequestBodyReleaser(req.RawRequest.Body, req.bodyBuf)
|
||||
|
||||
req.Time = time.Now()
|
||||
resp, err := c.httpClient.Do(req.RawRequest)
|
||||
|
||||
@@ -1052,13 +1077,16 @@ func createClient(hc *http.Client) *Client {
|
||||
Cookies: make([]*http.Cookie, 0),
|
||||
RetryWaitTime: defaultWaitTime,
|
||||
RetryMaxWaitTime: defaultMaxWaitTime,
|
||||
PathParams: make(map[string]string),
|
||||
JSONMarshal: json.Marshal,
|
||||
JSONUnmarshal: json.Unmarshal,
|
||||
XMLMarshal: xml.Marshal,
|
||||
XMLUnmarshal: xml.Unmarshal,
|
||||
HeaderAuthorizationKey: http.CanonicalHeaderKey("Authorization"),
|
||||
jsonEscapeHTML: true,
|
||||
httpClient: hc,
|
||||
debugBodySizeLimit: math.MaxInt32,
|
||||
pathParams: make(map[string]string),
|
||||
|
||||
jsonEscapeHTML: true,
|
||||
httpClient: hc,
|
||||
debugBodySizeLimit: math.MaxInt32,
|
||||
}
|
||||
|
||||
// Logger
|
||||
|
||||
2
vendor/github.com/go-resty/resty/v2/go.mod
сгенерированный
поставляемый
2
vendor/github.com/go-resty/resty/v2/go.mod
сгенерированный
поставляемый
@@ -1,5 +1,5 @@
|
||||
module github.com/go-resty/resty/v2
|
||||
|
||||
require golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
|
||||
require golang.org/x/net v0.0.0-20211029224645-99673261e6eb
|
||||
|
||||
go 1.11
|
||||
|
||||
8
vendor/github.com/go-resty/resty/v2/go.sum
сгенерированный
поставляемый
8
vendor/github.com/go-resty/resty/v2/go.sum
сгенерированный
поставляемый
@@ -1,7 +1,7 @@
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20211029224645-99673261e6eb h1:pirldcYWx7rx7kE5r+9WsOXPXK0+WH5+uZ7uPmJ44uM=
|
||||
golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
|
||||
24
vendor/github.com/go-resty/resty/v2/middleware.go
сгенерированный
поставляемый
24
vendor/github.com/go-resty/resty/v2/middleware.go
сгенерированный
поставляемый
@@ -6,7 +6,6 @@ package resty
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -29,13 +28,13 @@ const debugRequestLogKey = "__restyDebugRequestLog"
|
||||
|
||||
func parseRequestURL(c *Client, r *Request) error {
|
||||
// GitHub #103 Path Params
|
||||
if len(r.pathParams) > 0 {
|
||||
for p, v := range r.pathParams {
|
||||
if len(r.PathParams) > 0 {
|
||||
for p, v := range r.PathParams {
|
||||
r.URL = strings.Replace(r.URL, "{"+p+"}", url.PathEscape(v), -1)
|
||||
}
|
||||
}
|
||||
if len(c.pathParams) > 0 {
|
||||
for p, v := range c.pathParams {
|
||||
if len(c.PathParams) > 0 {
|
||||
for p, v := range c.PathParams {
|
||||
r.URL = strings.Replace(r.URL, "{"+p+"}", url.PathEscape(v), -1)
|
||||
}
|
||||
}
|
||||
@@ -60,6 +59,11 @@ func parseRequestURL(c *Client, r *Request) error {
|
||||
}
|
||||
}
|
||||
|
||||
// GH #407 && #318
|
||||
if reqURL.Scheme == "" && len(c.scheme) > 0 {
|
||||
reqURL.Scheme = c.scheme
|
||||
}
|
||||
|
||||
// Adding Query Param
|
||||
query := make(url.Values)
|
||||
for k, v := range c.QueryParam {
|
||||
@@ -191,12 +195,6 @@ func createHTTPRequest(c *Client, r *Request) (err error) {
|
||||
r.RawRequest.AddCookie(cookie)
|
||||
}
|
||||
|
||||
// it's for non-http scheme option
|
||||
if r.RawRequest.URL != nil && r.RawRequest.URL.Scheme == "" {
|
||||
r.RawRequest.URL.Scheme = c.scheme
|
||||
r.RawRequest.URL.Host = r.URL
|
||||
}
|
||||
|
||||
// Enable trace
|
||||
if c.trace || r.trace {
|
||||
r.clientTrace = &clientTrace{}
|
||||
@@ -458,12 +456,12 @@ func handleRequestBody(c *Client, r *Request) (err error) {
|
||||
bodyBytes = []byte(s)
|
||||
} else if IsJSONType(contentType) &&
|
||||
(kind == reflect.Struct || kind == reflect.Map || kind == reflect.Slice) {
|
||||
bodyBytes, err = jsonMarshal(c, r, r.Body)
|
||||
r.bodyBuf, err = jsonMarshal(c, r, r.Body)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
} else if IsXMLType(contentType) && (kind == reflect.Struct) {
|
||||
bodyBytes, err = xml.Marshal(r.Body)
|
||||
bodyBytes, err = c.XMLMarshal(r.Body)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
46
vendor/github.com/go-resty/resty/v2/request.go
сгенерированный
поставляемый
46
vendor/github.com/go-resty/resty/v2/request.go
сгенерированный
поставляемый
@@ -33,6 +33,7 @@ type Request struct {
|
||||
AuthScheme string
|
||||
QueryParam url.Values
|
||||
FormData url.Values
|
||||
PathParams map[string]string
|
||||
Header http.Header
|
||||
Time time.Time
|
||||
Body interface{}
|
||||
@@ -60,13 +61,13 @@ type Request struct {
|
||||
fallbackContentType string
|
||||
forceContentType string
|
||||
ctx context.Context
|
||||
pathParams map[string]string
|
||||
values map[string]interface{}
|
||||
client *Client
|
||||
bodyBuf *bytes.Buffer
|
||||
clientTrace *clientTrace
|
||||
multipartFiles []*File
|
||||
multipartFields []*MultipartField
|
||||
retryConditions []RetryConditionFunc
|
||||
}
|
||||
|
||||
// Context method returns the Context if its already set in request
|
||||
@@ -117,6 +118,22 @@ func (r *Request) SetHeaders(headers map[string]string) *Request {
|
||||
return r
|
||||
}
|
||||
|
||||
// SetHeaderMultiValues sets multiple headers fields and its values is list of strings at one go in the current request.
|
||||
//
|
||||
// For Example: To set `Accept` as `text/html, application/xhtml+xml, application/xml;q=0.9, image/webp, */*;q=0.8`
|
||||
//
|
||||
// client.R().
|
||||
// SetHeaderMultiValues(map[string][]string{
|
||||
// "Accept": []string{"text/html", "application/xhtml+xml", "application/xml;q=0.9", "image/webp", "*/*;q=0.8"},
|
||||
// })
|
||||
// Also you can override header value, which was set at client instance level.
|
||||
func (r *Request) SetHeaderMultiValues(headers map[string][]string) *Request {
|
||||
for key, values := range headers {
|
||||
r.SetHeader(key, strings.Join(values, ", "))
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// SetHeaderVerbatim method is to set a single header field and its value verbatim in the current request.
|
||||
//
|
||||
// For Example: To set `all_lowercase` and `UPPERCASE` as `available`.
|
||||
@@ -494,7 +511,7 @@ func (r *Request) SetDoNotParseResponse(parse bool) *Request {
|
||||
// It replaces the value of the key while composing the request URL. Also you can
|
||||
// override Path Params value, which was set at client instance level.
|
||||
func (r *Request) SetPathParam(param, value string) *Request {
|
||||
r.pathParams[param] = value
|
||||
r.PathParams[param] = value
|
||||
return r
|
||||
}
|
||||
|
||||
@@ -577,6 +594,18 @@ func (r *Request) SetCookies(rs []*http.Cookie) *Request {
|
||||
return r
|
||||
}
|
||||
|
||||
// AddRetryCondition method adds a retry condition function to the request's
|
||||
// array of functions that are checked to determine if the request is retried.
|
||||
// The request will retry if any of the functions return true and error is nil.
|
||||
//
|
||||
// Note: These retry conditions are checked before all retry conditions of the client.
|
||||
//
|
||||
// Since v2.7.0
|
||||
func (r *Request) AddRetryCondition(condition RetryConditionFunc) *Request {
|
||||
r.retryConditions = append(r.retryConditions, condition)
|
||||
return r
|
||||
}
|
||||
|
||||
//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
|
||||
// HTTP request tracing
|
||||
//_______________________________________________________________________
|
||||
@@ -747,7 +776,7 @@ func (r *Request) Execute(method, url string) (*Response, error) {
|
||||
Retries(r.client.RetryCount),
|
||||
WaitTime(r.client.RetryWaitTime),
|
||||
MaxWaitTime(r.client.RetryMaxWaitTime),
|
||||
RetryConditions(r.client.RetryConditions),
|
||||
RetryConditions(append(r.retryConditions, r.client.RetryConditions...)),
|
||||
RetryHooks(r.client.RetryHooks),
|
||||
)
|
||||
|
||||
@@ -854,11 +883,14 @@ func (r *Request) initValuesMap() {
|
||||
}
|
||||
}
|
||||
|
||||
var noescapeJSONMarshal = func(v interface{}) ([]byte, error) {
|
||||
var noescapeJSONMarshal = func(v interface{}) (*bytes.Buffer, error) {
|
||||
buf := acquireBuffer()
|
||||
defer releaseBuffer(buf)
|
||||
encoder := json.NewEncoder(buf)
|
||||
encoder.SetEscapeHTML(false)
|
||||
err := encoder.Encode(v)
|
||||
return buf.Bytes(), err
|
||||
if err := encoder.Encode(v); err != nil {
|
||||
releaseBuffer(buf)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
2
vendor/github.com/go-resty/resty/v2/resty.go
сгенерированный
поставляемый
2
vendor/github.com/go-resty/resty/v2/resty.go
сгенерированный
поставляемый
@@ -14,7 +14,7 @@ import (
|
||||
)
|
||||
|
||||
// Version # of resty
|
||||
const Version = "2.6.0"
|
||||
const Version = "2.7.0"
|
||||
|
||||
// New method creates a new Resty client.
|
||||
func New() *Client {
|
||||
|
||||
6
vendor/github.com/go-resty/resty/v2/retry.go
сгенерированный
поставляемый
6
vendor/github.com/go-resty/resty/v2/retry.go
сгенерированный
поставляемый
@@ -129,6 +129,12 @@ func Backoff(operation func() (*Response, error), options ...Option) error {
|
||||
hook(resp, err)
|
||||
}
|
||||
|
||||
// Don't need to wait when no retries left.
|
||||
// Still run retry hooks even on last retry to keep compatibility.
|
||||
if attempt == opts.maxRetries {
|
||||
return err
|
||||
}
|
||||
|
||||
waitTime, err2 := sleepDuration(resp, opts.waitTime, opts.maxWaitTime, attempt)
|
||||
if err2 != nil {
|
||||
if err == nil {
|
||||
|
||||
50
vendor/github.com/go-resty/resty/v2/util.go
сгенерированный
поставляемый
50
vendor/github.com/go-resty/resty/v2/util.go
сгенерированный
поставляемый
@@ -6,7 +6,6 @@ package resty
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
@@ -19,6 +18,7 @@ import (
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
//‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
|
||||
@@ -108,7 +108,7 @@ func Unmarshalc(c *Client, ct string, b []byte, d interface{}) (err error) {
|
||||
if IsJSONType(ct) {
|
||||
err = c.JSONUnmarshal(b, d)
|
||||
} else if IsXMLType(ct) {
|
||||
err = xml.Unmarshal(b, d)
|
||||
err = c.XMLUnmarshal(b, d)
|
||||
}
|
||||
|
||||
return
|
||||
@@ -139,13 +139,19 @@ type ResponseLog struct {
|
||||
//_______________________________________________________________________
|
||||
|
||||
// way to disable the HTML escape as opt-in
|
||||
func jsonMarshal(c *Client, r *Request, d interface{}) ([]byte, error) {
|
||||
if !r.jsonEscapeHTML {
|
||||
return noescapeJSONMarshal(d)
|
||||
} else if !c.jsonEscapeHTML {
|
||||
func jsonMarshal(c *Client, r *Request, d interface{}) (*bytes.Buffer, error) {
|
||||
if !r.jsonEscapeHTML || !c.jsonEscapeHTML {
|
||||
return noescapeJSONMarshal(d)
|
||||
}
|
||||
return c.JSONMarshal(d)
|
||||
|
||||
data, err := c.JSONMarshal(d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
buf := acquireBuffer()
|
||||
_, _ = buf.Write(data)
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
func firstNonEmpty(v ...string) string {
|
||||
@@ -195,7 +201,7 @@ func writeMultipartFormFile(w *multipart.Writer, fieldName, fileName string, r i
|
||||
// Auto detect actual multipart content type
|
||||
cbuf := make([]byte, 512)
|
||||
size, err := r.Read(cbuf)
|
||||
if err != nil {
|
||||
if err != nil && err != io.EOF {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -283,6 +289,34 @@ func releaseBuffer(buf *bytes.Buffer) {
|
||||
}
|
||||
}
|
||||
|
||||
// requestBodyReleaser wraps requests's body and implements custom Close for it.
|
||||
// The Close method closes original body and releases request body back to sync.Pool.
|
||||
type requestBodyReleaser struct {
|
||||
releaseOnce sync.Once
|
||||
reqBuf *bytes.Buffer
|
||||
io.ReadCloser
|
||||
}
|
||||
|
||||
func newRequestBodyReleaser(respBody io.ReadCloser, reqBuf *bytes.Buffer) io.ReadCloser {
|
||||
if reqBuf == nil {
|
||||
return respBody
|
||||
}
|
||||
|
||||
return &requestBodyReleaser{
|
||||
reqBuf: reqBuf,
|
||||
ReadCloser: respBody,
|
||||
}
|
||||
}
|
||||
|
||||
func (rr *requestBodyReleaser) Close() error {
|
||||
err := rr.ReadCloser.Close()
|
||||
rr.releaseOnce.Do(func() {
|
||||
releaseBuffer(rr.reqBuf)
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func closeq(v interface{}) {
|
||||
if c, ok := v.(io.Closer); ok {
|
||||
silently(c.Close())
|
||||
|
||||
8
vendor/github.com/golang-migrate/migrate/v4/Dockerfile.github-actions
сгенерированный
поставляемый
8
vendor/github.com/golang-migrate/migrate/v4/Dockerfile.github-actions
сгенерированный
поставляемый
@@ -2,8 +2,10 @@ FROM alpine:3.13
|
||||
|
||||
RUN apk add --no-cache ca-certificates
|
||||
|
||||
COPY migrate /usr/bin/migrate
|
||||
RUN ln -s /usr/bin/migrate /migrate
|
||||
COPY migrate /usr/local/bin/migrate
|
||||
|
||||
RUN ln -s /usr/local/bin/migrate /usr/bin/migrate
|
||||
RUN ln -s /usr/local/bin/migrate /migrate
|
||||
|
||||
ENTRYPOINT ["migrate"]
|
||||
CMD ["--help"]
|
||||
CMD ["--help"]
|
||||
37
vendor/github.com/golang-migrate/migrate/v4/go.mod
сгенерированный
поставляемый
37
vendor/github.com/golang-migrate/migrate/v4/go.mod
сгенерированный
поставляемый
@@ -5,27 +5,24 @@ require (
|
||||
cloud.google.com/go/storage v1.10.0
|
||||
github.com/Azure/go-autorest/autorest/adal v0.9.14
|
||||
github.com/ClickHouse/clickhouse-go v1.4.3
|
||||
github.com/apache/arrow/go/arrow v0.0.0-20210521153258-78c88a9f517b // indirect
|
||||
github.com/Microsoft/go-winio v0.5.0 // indirect
|
||||
github.com/apache/arrow/go/arrow v0.0.0-20211013220434-5962184e7a30 // indirect
|
||||
github.com/aws/aws-sdk-go v1.17.7
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.2.1 // indirect
|
||||
github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 // indirect
|
||||
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.4 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.0.2
|
||||
github.com/cockroachdb/cockroach-go/v2 v2.1.1
|
||||
github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369 // indirect
|
||||
github.com/denisenkom/go-mssqldb v0.10.0
|
||||
github.com/dhui/dktest v0.3.4
|
||||
github.com/docker/docker v17.12.0-ce-rc1.0.20210128214336-420b1d36250f+incompatible
|
||||
github.com/dhui/dktest v0.3.7
|
||||
github.com/docker/docker v20.10.9+incompatible
|
||||
github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 // indirect
|
||||
github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
|
||||
github.com/fsouza/fake-gcs-server v1.17.0
|
||||
github.com/gabriel-vasile/mimetype v1.4.0 // indirect
|
||||
github.com/go-sql-driver/mysql v1.5.0
|
||||
github.com/gobuffalo/here v0.6.0
|
||||
github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4
|
||||
github.com/gocql/gocql v0.0.0-20210515062232-b7ef815b4556
|
||||
github.com/gofrs/uuid v4.0.0+incompatible // indirect
|
||||
github.com/google/flatbuffers v2.0.0+incompatible // indirect
|
||||
github.com/google/go-github/v35 v35.2.0
|
||||
github.com/google/uuid v1.2.0 // indirect
|
||||
github.com/gorilla/mux v1.7.4 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.0
|
||||
github.com/jackc/pgconn v1.8.0
|
||||
@@ -33,29 +30,29 @@ require (
|
||||
github.com/jackc/pgproto3/v2 v2.0.7 // indirect
|
||||
github.com/jackc/pgx/v4 v4.10.1
|
||||
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
|
||||
github.com/klauspost/compress v1.12.2 // indirect
|
||||
github.com/klauspost/compress v1.13.6 // indirect
|
||||
github.com/ktrysmt/go-bitbucket v0.6.4
|
||||
github.com/lib/pq v1.10.0
|
||||
github.com/markbates/pkger v0.15.1
|
||||
github.com/mattn/go-sqlite3 v1.14.6
|
||||
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
|
||||
github.com/mutecomm/go-sqlcipher/v4 v4.4.0
|
||||
github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8
|
||||
github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba
|
||||
github.com/pierrec/lz4/v4 v4.1.7 // indirect
|
||||
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect
|
||||
github.com/sirupsen/logrus v1.8.1 // indirect
|
||||
github.com/snowflakedb/gosnowflake v1.4.3
|
||||
github.com/stretchr/testify v1.6.1
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
|
||||
github.com/snowflakedb/gosnowflake v1.6.3
|
||||
github.com/stretchr/testify v1.7.0
|
||||
github.com/xanzy/go-gitlab v0.15.0
|
||||
gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b // indirect
|
||||
go.mongodb.org/mongo-driver v1.7.0
|
||||
go.uber.org/atomic v1.6.0
|
||||
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a // indirect
|
||||
golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 // indirect
|
||||
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
|
||||
golang.org/x/net v0.0.0-20211013171255-e13a2654a71e // indirect
|
||||
golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect
|
||||
golang.org/x/tools v0.1.5
|
||||
google.golang.org/api v0.51.0
|
||||
google.golang.org/genproto v0.0.0-20210726143408-b02e89920bf0
|
||||
google.golang.org/genproto v0.0.0-20211013025323-ce878158c4d4
|
||||
google.golang.org/grpc v1.41.0 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
modernc.org/db v1.0.0 // indirect
|
||||
modernc.org/file v1.0.0 // indirect
|
||||
|
||||
790
vendor/github.com/golang-migrate/migrate/v4/go.sum
сгенерированный
поставляемый
790
vendor/github.com/golang-migrate/migrate/v4/go.sum
сгенерированный
поставляемый
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
11
vendor/github.com/hashicorp/memberlist/config.go
сгенерированный
поставляемый
11
vendor/github.com/hashicorp/memberlist/config.go
сгенерированный
поставляемый
@@ -21,6 +21,17 @@ type Config struct {
|
||||
// make a NetTransport using BindAddr and BindPort from this structure.
|
||||
Transport Transport
|
||||
|
||||
// Label is an optional set of bytes to include on the outside of each
|
||||
// packet and stream.
|
||||
//
|
||||
// If gossip encryption is enabled and this is set it is treated as GCM
|
||||
// authenticated data.
|
||||
Label string
|
||||
|
||||
// SkipInboundLabelCheck skips the check that inbound packets and gossip
|
||||
// streams need to be label prefixed.
|
||||
SkipInboundLabelCheck bool
|
||||
|
||||
// Configuration related to what address to bind to and ports to
|
||||
// listen on. The port is used for both UDP and TCP gossip. It is
|
||||
// assumed other nodes are running on this port, but they do not need
|
||||
|
||||
178
vendor/github.com/hashicorp/memberlist/label.go
сгенерированный
поставляемый
Обычный файл
178
vendor/github.com/hashicorp/memberlist/label.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,178 @@
|
||||
package memberlist
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
)
|
||||
|
||||
// General approach is to prefix all packets and streams with the same structure:
|
||||
//
|
||||
// magic type byte (244): uint8
|
||||
// length of label name: uint8 (because labels can't be longer than 255 bytes)
|
||||
// label name: []uint8
|
||||
|
||||
// LabelMaxSize is the maximum length of a packet or stream label.
|
||||
const LabelMaxSize = 255
|
||||
|
||||
// AddLabelHeaderToPacket prefixes outgoing packets with the correct header if
|
||||
// the label is not empty.
|
||||
func AddLabelHeaderToPacket(buf []byte, label string) ([]byte, error) {
|
||||
if label == "" {
|
||||
return buf, nil
|
||||
}
|
||||
if len(label) > LabelMaxSize {
|
||||
return nil, fmt.Errorf("label %q is too long", label)
|
||||
}
|
||||
|
||||
return makeLabelHeader(label, buf), nil
|
||||
}
|
||||
|
||||
// RemoveLabelHeaderFromPacket removes any label header from the provided
|
||||
// packet and returns it along with the remaining packet contents.
|
||||
func RemoveLabelHeaderFromPacket(buf []byte) (newBuf []byte, label string, err error) {
|
||||
if len(buf) == 0 {
|
||||
return buf, "", nil // can't possibly be labeled
|
||||
}
|
||||
|
||||
// [type:byte] [size:byte] [size bytes]
|
||||
|
||||
msgType := messageType(buf[0])
|
||||
if msgType != hasLabelMsg {
|
||||
return buf, "", nil
|
||||
}
|
||||
|
||||
if len(buf) < 2 {
|
||||
return nil, "", fmt.Errorf("cannot decode label; packet has been truncated")
|
||||
}
|
||||
|
||||
size := int(buf[1])
|
||||
if size < 1 {
|
||||
return nil, "", fmt.Errorf("label header cannot be empty when present")
|
||||
}
|
||||
|
||||
if len(buf) < 2+size {
|
||||
return nil, "", fmt.Errorf("cannot decode label; packet has been truncated")
|
||||
}
|
||||
|
||||
label = string(buf[2 : 2+size])
|
||||
newBuf = buf[2+size:]
|
||||
|
||||
return newBuf, label, nil
|
||||
}
|
||||
|
||||
// AddLabelHeaderToStream prefixes outgoing streams with the correct header if
|
||||
// the label is not empty.
|
||||
func AddLabelHeaderToStream(conn net.Conn, label string) error {
|
||||
if label == "" {
|
||||
return nil
|
||||
}
|
||||
if len(label) > LabelMaxSize {
|
||||
return fmt.Errorf("label %q is too long", label)
|
||||
}
|
||||
|
||||
header := makeLabelHeader(label, nil)
|
||||
|
||||
_, err := conn.Write(header)
|
||||
return err
|
||||
}
|
||||
|
||||
// RemoveLabelHeaderFromStream removes any label header from the beginning of
|
||||
// the stream if present and returns it along with an updated conn with that
|
||||
// header removed.
|
||||
//
|
||||
// Note that on error it is the caller's responsibility to close the
|
||||
// connection.
|
||||
func RemoveLabelHeaderFromStream(conn net.Conn) (net.Conn, string, error) {
|
||||
br := bufio.NewReader(conn)
|
||||
|
||||
// First check for the type byte.
|
||||
peeked, err := br.Peek(1)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
// It is safe to return the original net.Conn at this point because
|
||||
// it never contained any data in the first place so we don't have
|
||||
// to splice the buffer into the conn because both are empty.
|
||||
return conn, "", nil
|
||||
}
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
msgType := messageType(peeked[0])
|
||||
if msgType != hasLabelMsg {
|
||||
conn, err = newPeekedConnFromBufferedReader(conn, br, 0)
|
||||
return conn, "", err
|
||||
}
|
||||
|
||||
// We are guaranteed to get a size byte as well.
|
||||
peeked, err = br.Peek(2)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return nil, "", fmt.Errorf("cannot decode label; stream has been truncated")
|
||||
}
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
size := int(peeked[1])
|
||||
if size < 1 {
|
||||
return nil, "", fmt.Errorf("label header cannot be empty when present")
|
||||
}
|
||||
// NOTE: we don't have to check this against LabelMaxSize because a byte
|
||||
// already has a max value of 255.
|
||||
|
||||
// Once we know the size we can peek the label as well. Note that since we
|
||||
// are using the default bufio.Reader size of 4096, the entire label header
|
||||
// fits in the initial buffer fill so this should be free.
|
||||
peeked, err = br.Peek(2 + size)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return nil, "", fmt.Errorf("cannot decode label; stream has been truncated")
|
||||
}
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
label := string(peeked[2 : 2+size])
|
||||
|
||||
conn, err = newPeekedConnFromBufferedReader(conn, br, 2+size)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
return conn, label, nil
|
||||
}
|
||||
|
||||
// newPeekedConnFromBufferedReader will splice the buffer contents after the
|
||||
// offset into the provided net.Conn and return the result so that the rest of
|
||||
// the buffer contents are returned first when reading from the returned
|
||||
// peekedConn before moving on to the unbuffered conn contents.
|
||||
func newPeekedConnFromBufferedReader(conn net.Conn, br *bufio.Reader, offset int) (*peekedConn, error) {
|
||||
// Extract any of the readahead buffer.
|
||||
peeked, err := br.Peek(br.Buffered())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &peekedConn{
|
||||
Peeked: peeked[offset:],
|
||||
Conn: conn,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func makeLabelHeader(label string, rest []byte) []byte {
|
||||
newBuf := make([]byte, 2, 2+len(label)+len(rest))
|
||||
newBuf[0] = byte(hasLabelMsg)
|
||||
newBuf[1] = byte(len(label))
|
||||
newBuf = append(newBuf, []byte(label)...)
|
||||
if len(rest) > 0 {
|
||||
newBuf = append(newBuf, []byte(rest)...)
|
||||
}
|
||||
return newBuf
|
||||
}
|
||||
|
||||
func labelOverhead(label string) int {
|
||||
if label == "" {
|
||||
return 0
|
||||
}
|
||||
return 2 + len(label)
|
||||
}
|
||||
13
vendor/github.com/hashicorp/memberlist/memberlist.go
сгенерированный
поставляемый
13
vendor/github.com/hashicorp/memberlist/memberlist.go
сгенерированный
поставляемый
@@ -187,6 +187,17 @@ func newMemberlist(conf *Config) (*Memberlist, error) {
|
||||
nodeAwareTransport = &shimNodeAwareTransport{transport}
|
||||
}
|
||||
|
||||
if len(conf.Label) > LabelMaxSize {
|
||||
return nil, fmt.Errorf("could not use %q as a label: too long", conf.Label)
|
||||
}
|
||||
|
||||
if conf.Label != "" {
|
||||
nodeAwareTransport = &labelWrappedTransport{
|
||||
label: conf.Label,
|
||||
NodeAwareTransport: nodeAwareTransport,
|
||||
}
|
||||
}
|
||||
|
||||
m := &Memberlist{
|
||||
config: conf,
|
||||
shutdownCh: make(chan struct{}),
|
||||
@@ -262,7 +273,7 @@ func (m *Memberlist) Join(existing []string) (int, error) {
|
||||
hp := joinHostPort(addr.ip.String(), addr.port)
|
||||
a := Address{Addr: hp, Name: addr.nodeName}
|
||||
if err := m.pushPullNode(a, true); err != nil {
|
||||
err = fmt.Errorf("Failed to join %s: %v", addr.ip, err)
|
||||
err = fmt.Errorf("Failed to join %s: %v", a.Addr, err)
|
||||
errs = multierror.Append(errs, err)
|
||||
m.logger.Printf("[DEBUG] memberlist: %v", err)
|
||||
continue
|
||||
|
||||
132
vendor/github.com/hashicorp/memberlist/net.go
сгенерированный
поставляемый
132
vendor/github.com/hashicorp/memberlist/net.go
сгенерированный
поставляемый
@@ -42,6 +42,9 @@ const (
|
||||
type messageType uint8
|
||||
|
||||
// The list of available message types.
|
||||
//
|
||||
// WARNING: ONLY APPEND TO THIS LIST! The numeric values are part of the
|
||||
// protocol itself.
|
||||
const (
|
||||
pingMsg messageType = iota
|
||||
indirectPingMsg
|
||||
@@ -59,6 +62,13 @@ const (
|
||||
errMsg
|
||||
)
|
||||
|
||||
const (
|
||||
// hasLabelMsg has a deliberately high value so that you can disambiguate
|
||||
// it from the encryptionVersion header which is either 0/1 right now and
|
||||
// also any of the existing messageTypes
|
||||
hasLabelMsg messageType = 244
|
||||
)
|
||||
|
||||
// compressionType is used to specify the compression algorithm
|
||||
type compressionType uint8
|
||||
|
||||
@@ -226,7 +236,32 @@ func (m *Memberlist) handleConn(conn net.Conn) {
|
||||
metrics.IncrCounter([]string{"memberlist", "tcp", "accept"}, 1)
|
||||
|
||||
conn.SetDeadline(time.Now().Add(m.config.TCPTimeout))
|
||||
msgType, bufConn, dec, err := m.readStream(conn)
|
||||
|
||||
var (
|
||||
streamLabel string
|
||||
err error
|
||||
)
|
||||
conn, streamLabel, err = RemoveLabelHeaderFromStream(conn)
|
||||
if err != nil {
|
||||
m.logger.Printf("[ERR] memberlist: failed to receive and remove the stream label header: %s %s", err, LogConn(conn))
|
||||
return
|
||||
}
|
||||
|
||||
if m.config.SkipInboundLabelCheck {
|
||||
if streamLabel != "" {
|
||||
m.logger.Printf("[ERR] memberlist: unexpected double stream label header: %s", LogConn(conn))
|
||||
return
|
||||
}
|
||||
// Set this from config so that the auth data assertions work below.
|
||||
streamLabel = m.config.Label
|
||||
}
|
||||
|
||||
if m.config.Label != streamLabel {
|
||||
m.logger.Printf("[ERR] memberlist: discarding stream with unacceptable label %q: %s", streamLabel, LogConn(conn))
|
||||
return
|
||||
}
|
||||
|
||||
msgType, bufConn, dec, err := m.readStream(conn, streamLabel)
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
m.logger.Printf("[ERR] memberlist: failed to receive: %s %s", err, LogConn(conn))
|
||||
@@ -238,7 +273,7 @@ func (m *Memberlist) handleConn(conn net.Conn) {
|
||||
return
|
||||
}
|
||||
|
||||
err = m.rawSendMsgStream(conn, out.Bytes())
|
||||
err = m.rawSendMsgStream(conn, out.Bytes(), streamLabel)
|
||||
if err != nil {
|
||||
m.logger.Printf("[ERR] memberlist: Failed to send error: %s %s", err, LogConn(conn))
|
||||
return
|
||||
@@ -269,7 +304,7 @@ func (m *Memberlist) handleConn(conn net.Conn) {
|
||||
return
|
||||
}
|
||||
|
||||
if err := m.sendLocalState(conn, join); err != nil {
|
||||
if err := m.sendLocalState(conn, join, streamLabel); err != nil {
|
||||
m.logger.Printf("[ERR] memberlist: Failed to push local state: %s %s", err, LogConn(conn))
|
||||
return
|
||||
}
|
||||
@@ -297,7 +332,7 @@ func (m *Memberlist) handleConn(conn net.Conn) {
|
||||
return
|
||||
}
|
||||
|
||||
err = m.rawSendMsgStream(conn, out.Bytes())
|
||||
err = m.rawSendMsgStream(conn, out.Bytes(), streamLabel)
|
||||
if err != nil {
|
||||
m.logger.Printf("[ERR] memberlist: Failed to send ack: %s %s", err, LogConn(conn))
|
||||
return
|
||||
@@ -322,10 +357,35 @@ func (m *Memberlist) packetListen() {
|
||||
}
|
||||
|
||||
func (m *Memberlist) ingestPacket(buf []byte, from net.Addr, timestamp time.Time) {
|
||||
var (
|
||||
packetLabel string
|
||||
err error
|
||||
)
|
||||
buf, packetLabel, err = RemoveLabelHeaderFromPacket(buf)
|
||||
if err != nil {
|
||||
m.logger.Printf("[ERR] memberlist: %v %s", err, LogAddress(from))
|
||||
return
|
||||
}
|
||||
|
||||
if m.config.SkipInboundLabelCheck {
|
||||
if packetLabel != "" {
|
||||
m.logger.Printf("[ERR] memberlist: unexpected double packet label header: %s", LogAddress(from))
|
||||
return
|
||||
}
|
||||
// Set this from config so that the auth data assertions work below.
|
||||
packetLabel = m.config.Label
|
||||
}
|
||||
|
||||
if m.config.Label != packetLabel {
|
||||
m.logger.Printf("[ERR] memberlist: discarding packet with unacceptable label %q: %s", packetLabel, LogAddress(from))
|
||||
return
|
||||
}
|
||||
|
||||
// Check if encryption is enabled
|
||||
if m.config.EncryptionEnabled() {
|
||||
// Decrypt the payload
|
||||
plain, err := decryptPayload(m.config.Keyring.GetKeys(), buf, nil)
|
||||
authData := []byte(packetLabel)
|
||||
plain, err := decryptPayload(m.config.Keyring.GetKeys(), buf, authData)
|
||||
if err != nil {
|
||||
if !m.config.GossipVerifyIncoming {
|
||||
// Treat the message as plaintext
|
||||
@@ -723,7 +783,7 @@ func (m *Memberlist) encodeAndSendMsg(a Address, msgType messageType, msg interf
|
||||
// opportunistically create a compoundMsg and piggy back other broadcasts.
|
||||
func (m *Memberlist) sendMsg(a Address, msg []byte) error {
|
||||
// Check if we can piggy back any messages
|
||||
bytesAvail := m.config.UDPBufferSize - len(msg) - compoundHeaderOverhead
|
||||
bytesAvail := m.config.UDPBufferSize - len(msg) - compoundHeaderOverhead - labelOverhead(m.config.Label)
|
||||
if m.config.EncryptionEnabled() && m.config.GossipVerifyOutgoing {
|
||||
bytesAvail -= encryptOverhead(m.encryptionVersion())
|
||||
}
|
||||
@@ -795,9 +855,12 @@ func (m *Memberlist) rawSendMsgPacket(a Address, node *Node, msg []byte) error {
|
||||
// Check if we have encryption enabled
|
||||
if m.config.EncryptionEnabled() && m.config.GossipVerifyOutgoing {
|
||||
// Encrypt the payload
|
||||
var buf bytes.Buffer
|
||||
primaryKey := m.config.Keyring.GetPrimaryKey()
|
||||
err := encryptPayload(m.encryptionVersion(), primaryKey, msg, nil, &buf)
|
||||
var (
|
||||
primaryKey = m.config.Keyring.GetPrimaryKey()
|
||||
packetLabel = []byte(m.config.Label)
|
||||
buf bytes.Buffer
|
||||
)
|
||||
err := encryptPayload(m.encryptionVersion(), primaryKey, msg, packetLabel, &buf)
|
||||
if err != nil {
|
||||
m.logger.Printf("[ERR] memberlist: Encryption of message failed: %v", err)
|
||||
return err
|
||||
@@ -812,7 +875,7 @@ func (m *Memberlist) rawSendMsgPacket(a Address, node *Node, msg []byte) error {
|
||||
|
||||
// rawSendMsgStream is used to stream a message to another host without
|
||||
// modification, other than applying compression and encryption if enabled.
|
||||
func (m *Memberlist) rawSendMsgStream(conn net.Conn, sendBuf []byte) error {
|
||||
func (m *Memberlist) rawSendMsgStream(conn net.Conn, sendBuf []byte, streamLabel string) error {
|
||||
// Check if compression is enabled
|
||||
if m.config.EnableCompression {
|
||||
compBuf, err := compressPayload(sendBuf)
|
||||
@@ -825,7 +888,7 @@ func (m *Memberlist) rawSendMsgStream(conn net.Conn, sendBuf []byte) error {
|
||||
|
||||
// Check if encryption is enabled
|
||||
if m.config.EncryptionEnabled() && m.config.GossipVerifyOutgoing {
|
||||
crypt, err := m.encryptLocalState(sendBuf)
|
||||
crypt, err := m.encryptLocalState(sendBuf, streamLabel)
|
||||
if err != nil {
|
||||
m.logger.Printf("[ERROR] memberlist: Failed to encrypt local state: %v", err)
|
||||
return err
|
||||
@@ -871,7 +934,8 @@ func (m *Memberlist) sendUserMsg(a Address, sendBuf []byte) error {
|
||||
if _, err := bufConn.Write(sendBuf); err != nil {
|
||||
return err
|
||||
}
|
||||
return m.rawSendMsgStream(conn, bufConn.Bytes())
|
||||
|
||||
return m.rawSendMsgStream(conn, bufConn.Bytes(), m.config.Label)
|
||||
}
|
||||
|
||||
// sendAndReceiveState is used to initiate a push/pull over a stream with a
|
||||
@@ -891,12 +955,12 @@ func (m *Memberlist) sendAndReceiveState(a Address, join bool) ([]pushNodeState,
|
||||
metrics.IncrCounter([]string{"memberlist", "tcp", "connect"}, 1)
|
||||
|
||||
// Send our state
|
||||
if err := m.sendLocalState(conn, join); err != nil {
|
||||
if err := m.sendLocalState(conn, join, m.config.Label); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
conn.SetDeadline(time.Now().Add(m.config.TCPTimeout))
|
||||
msgType, bufConn, dec, err := m.readStream(conn)
|
||||
msgType, bufConn, dec, err := m.readStream(conn, m.config.Label)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@@ -921,7 +985,7 @@ func (m *Memberlist) sendAndReceiveState(a Address, join bool) ([]pushNodeState,
|
||||
}
|
||||
|
||||
// sendLocalState is invoked to send our local state over a stream connection.
|
||||
func (m *Memberlist) sendLocalState(conn net.Conn, join bool) error {
|
||||
func (m *Memberlist) sendLocalState(conn net.Conn, join bool, streamLabel string) error {
|
||||
// Setup a deadline
|
||||
conn.SetDeadline(time.Now().Add(m.config.TCPTimeout))
|
||||
|
||||
@@ -978,11 +1042,11 @@ func (m *Memberlist) sendLocalState(conn net.Conn, join bool) error {
|
||||
}
|
||||
|
||||
// Get the send buffer
|
||||
return m.rawSendMsgStream(conn, bufConn.Bytes())
|
||||
return m.rawSendMsgStream(conn, bufConn.Bytes(), streamLabel)
|
||||
}
|
||||
|
||||
// encryptLocalState is used to help encrypt local state before sending
|
||||
func (m *Memberlist) encryptLocalState(sendBuf []byte) ([]byte, error) {
|
||||
func (m *Memberlist) encryptLocalState(sendBuf []byte, streamLabel string) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
|
||||
// Write the encryptMsg byte
|
||||
@@ -995,9 +1059,15 @@ func (m *Memberlist) encryptLocalState(sendBuf []byte) ([]byte, error) {
|
||||
binary.BigEndian.PutUint32(sizeBuf, uint32(encLen))
|
||||
buf.Write(sizeBuf)
|
||||
|
||||
// Authenticated Data is:
|
||||
//
|
||||
// [messageType; byte] [messageLength; uint32] [stream_label; optional]
|
||||
//
|
||||
dataBytes := appendBytes(buf.Bytes()[:5], []byte(streamLabel))
|
||||
|
||||
// Write the encrypted cipher text to the buffer
|
||||
key := m.config.Keyring.GetPrimaryKey()
|
||||
err := encryptPayload(encVsn, key, sendBuf, buf.Bytes()[:5], &buf)
|
||||
err := encryptPayload(encVsn, key, sendBuf, dataBytes, &buf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1005,7 +1075,7 @@ func (m *Memberlist) encryptLocalState(sendBuf []byte) ([]byte, error) {
|
||||
}
|
||||
|
||||
// decryptRemoteState is used to help decrypt the remote state
|
||||
func (m *Memberlist) decryptRemoteState(bufConn io.Reader) ([]byte, error) {
|
||||
func (m *Memberlist) decryptRemoteState(bufConn io.Reader, streamLabel string) ([]byte, error) {
|
||||
// Read in enough to determine message length
|
||||
cipherText := bytes.NewBuffer(nil)
|
||||
cipherText.WriteByte(byte(encryptMsg))
|
||||
@@ -1027,8 +1097,13 @@ func (m *Memberlist) decryptRemoteState(bufConn io.Reader) ([]byte, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Decrypt the cipherText
|
||||
dataBytes := cipherText.Bytes()[:5]
|
||||
// Decrypt the cipherText with some authenticated data
|
||||
//
|
||||
// Authenticated Data is:
|
||||
//
|
||||
// [messageType; byte] [messageLength; uint32] [label_data; optional]
|
||||
//
|
||||
dataBytes := appendBytes(cipherText.Bytes()[:5], []byte(streamLabel))
|
||||
cipherBytes := cipherText.Bytes()[5:]
|
||||
|
||||
// Decrypt the payload
|
||||
@@ -1036,15 +1111,18 @@ func (m *Memberlist) decryptRemoteState(bufConn io.Reader) ([]byte, error) {
|
||||
return decryptPayload(keys, cipherBytes, dataBytes)
|
||||
}
|
||||
|
||||
// readStream is used to read from a stream connection, decrypting and
|
||||
// readStream is used to read messages from a stream connection, decrypting and
|
||||
// decompressing the stream if necessary.
|
||||
func (m *Memberlist) readStream(conn net.Conn) (messageType, io.Reader, *codec.Decoder, error) {
|
||||
//
|
||||
// The provided streamLabel if present will be authenticated during decryption
|
||||
// of each message.
|
||||
func (m *Memberlist) readStream(conn net.Conn, streamLabel string) (messageType, io.Reader, *codec.Decoder, error) {
|
||||
// Created a buffered reader
|
||||
var bufConn io.Reader = bufio.NewReader(conn)
|
||||
|
||||
// Read the message type
|
||||
buf := [1]byte{0}
|
||||
if _, err := bufConn.Read(buf[:]); err != nil {
|
||||
if _, err := io.ReadFull(bufConn, buf[:]); err != nil {
|
||||
return 0, nil, nil, err
|
||||
}
|
||||
msgType := messageType(buf[0])
|
||||
@@ -1056,7 +1134,7 @@ func (m *Memberlist) readStream(conn net.Conn) (messageType, io.Reader, *codec.D
|
||||
fmt.Errorf("Remote state is encrypted and encryption is not configured")
|
||||
}
|
||||
|
||||
plain, err := m.decryptRemoteState(bufConn)
|
||||
plain, err := m.decryptRemoteState(bufConn, streamLabel)
|
||||
if err != nil {
|
||||
return 0, nil, nil, err
|
||||
}
|
||||
@@ -1236,11 +1314,11 @@ func (m *Memberlist) sendPingAndWaitForAck(a Address, ping ping, deadline time.T
|
||||
return false, err
|
||||
}
|
||||
|
||||
if err = m.rawSendMsgStream(conn, out.Bytes()); err != nil {
|
||||
if err = m.rawSendMsgStream(conn, out.Bytes(), m.config.Label); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
msgType, _, dec, err := m.readStream(conn)
|
||||
msgType, _, dec, err := m.readStream(conn, m.config.Label)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
48
vendor/github.com/hashicorp/memberlist/peeked_conn.go
сгенерированный
поставляемый
Обычный файл
48
vendor/github.com/hashicorp/memberlist/peeked_conn.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,48 @@
|
||||
// Copyright 2017 Google Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Originally from: https://github.com/google/tcpproxy/blob/master/tcpproxy.go
|
||||
// at f5c09fbedceb69e4b238dec52cdf9f2fe9a815e2
|
||||
|
||||
package memberlist
|
||||
|
||||
import "net"
|
||||
|
||||
// peekedConn is an incoming connection that has had some bytes read from it
|
||||
// to determine how to route the connection. The Read method stitches
|
||||
// the peeked bytes and unread bytes back together.
|
||||
type peekedConn struct {
|
||||
// Peeked are the bytes that have been read from Conn for the
|
||||
// purposes of route matching, but have not yet been consumed
|
||||
// by Read calls. It set to nil by Read when fully consumed.
|
||||
Peeked []byte
|
||||
|
||||
// Conn is the underlying connection.
|
||||
// It can be type asserted against *net.TCPConn or other types
|
||||
// as needed. It should not be read from directly unless
|
||||
// Peeked is nil.
|
||||
net.Conn
|
||||
}
|
||||
|
||||
func (c *peekedConn) Read(p []byte) (n int, err error) {
|
||||
if len(c.Peeked) > 0 {
|
||||
n = copy(p, c.Peeked)
|
||||
c.Peeked = c.Peeked[n:]
|
||||
if len(c.Peeked) == 0 {
|
||||
c.Peeked = nil
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
return c.Conn.Read(p)
|
||||
}
|
||||
19
vendor/github.com/hashicorp/memberlist/security.go
сгенерированный
поставляемый
19
vendor/github.com/hashicorp/memberlist/security.go
сгенерированный
поставляемый
@@ -199,3 +199,22 @@ func decryptPayload(keys [][]byte, msg []byte, data []byte) ([]byte, error) {
|
||||
|
||||
return nil, fmt.Errorf("No installed keys could decrypt the message")
|
||||
}
|
||||
|
||||
func appendBytes(first []byte, second []byte) []byte {
|
||||
hasFirst := len(first) > 0
|
||||
hasSecond := len(second) > 0
|
||||
|
||||
switch {
|
||||
case hasFirst && hasSecond:
|
||||
out := make([]byte, 0, len(first)+len(second))
|
||||
out = append(out, first...)
|
||||
out = append(out, second...)
|
||||
return out
|
||||
case hasFirst:
|
||||
return first
|
||||
case hasSecond:
|
||||
return second
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
7
vendor/github.com/hashicorp/memberlist/state.go
сгенерированный
поставляемый
7
vendor/github.com/hashicorp/memberlist/state.go
сгенерированный
поставляемый
@@ -274,6 +274,11 @@ func failedRemote(err error) bool {
|
||||
case "dial", "read", "write":
|
||||
return true
|
||||
}
|
||||
} else if strings.HasPrefix(t.Net, "udp") {
|
||||
switch t.Op {
|
||||
case "write":
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
@@ -587,7 +592,7 @@ func (m *Memberlist) gossip() {
|
||||
m.nodeLock.RUnlock()
|
||||
|
||||
// Compute the bytes available
|
||||
bytesAvail := m.config.UDPBufferSize - compoundHeaderOverhead
|
||||
bytesAvail := m.config.UDPBufferSize - compoundHeaderOverhead - labelOverhead(m.config.Label)
|
||||
if m.config.EncryptionEnabled() {
|
||||
bytesAvail -= encryptOverhead(m.encryptionVersion())
|
||||
}
|
||||
|
||||
47
vendor/github.com/hashicorp/memberlist/transport.go
сгенерированный
поставляемый
47
vendor/github.com/hashicorp/memberlist/transport.go
сгенерированный
поставляемый
@@ -111,3 +111,50 @@ func (t *shimNodeAwareTransport) WriteToAddress(b []byte, addr Address) (time.Ti
|
||||
func (t *shimNodeAwareTransport) DialAddressTimeout(addr Address, timeout time.Duration) (net.Conn, error) {
|
||||
return t.DialTimeout(addr.Addr, timeout)
|
||||
}
|
||||
|
||||
type labelWrappedTransport struct {
|
||||
label string
|
||||
NodeAwareTransport
|
||||
}
|
||||
|
||||
var _ NodeAwareTransport = (*labelWrappedTransport)(nil)
|
||||
|
||||
func (t *labelWrappedTransport) WriteToAddress(buf []byte, addr Address) (time.Time, error) {
|
||||
var err error
|
||||
buf, err = AddLabelHeaderToPacket(buf, t.label)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("failed to add label header to packet: %w", err)
|
||||
}
|
||||
return t.NodeAwareTransport.WriteToAddress(buf, addr)
|
||||
}
|
||||
|
||||
func (t *labelWrappedTransport) WriteTo(buf []byte, addr string) (time.Time, error) {
|
||||
var err error
|
||||
buf, err = AddLabelHeaderToPacket(buf, t.label)
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
return t.NodeAwareTransport.WriteTo(buf, addr)
|
||||
}
|
||||
|
||||
func (t *labelWrappedTransport) DialAddressTimeout(addr Address, timeout time.Duration) (net.Conn, error) {
|
||||
conn, err := t.NodeAwareTransport.DialAddressTimeout(addr, timeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := AddLabelHeaderToStream(conn, t.label); err != nil {
|
||||
return nil, fmt.Errorf("failed to add label header to stream: %w", err)
|
||||
}
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
func (t *labelWrappedTransport) DialTimeout(addr string, timeout time.Duration) (net.Conn, error) {
|
||||
conn, err := t.NodeAwareTransport.DialTimeout(addr, timeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := AddLabelHeaderToStream(conn, t.label); err != nil {
|
||||
return nil, fmt.Errorf("failed to add label header to stream: %w", err)
|
||||
}
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
8
vendor/github.com/hashicorp/yamux/session.go
сгенерированный
поставляемый
8
vendor/github.com/hashicorp/yamux/session.go
сгенерированный
поставляемый
@@ -80,7 +80,7 @@ type Session struct {
|
||||
// or to directly send a header
|
||||
type sendReady struct {
|
||||
Hdr []byte
|
||||
Body io.Reader
|
||||
Body []byte
|
||||
Err chan error
|
||||
}
|
||||
|
||||
@@ -352,7 +352,7 @@ func (s *Session) keepalive() {
|
||||
}
|
||||
|
||||
// waitForSendErr waits to send a header, checking for a potential shutdown
|
||||
func (s *Session) waitForSend(hdr header, body io.Reader) error {
|
||||
func (s *Session) waitForSend(hdr header, body []byte) error {
|
||||
errCh := make(chan error, 1)
|
||||
return s.waitForSendErr(hdr, body, errCh)
|
||||
}
|
||||
@@ -360,7 +360,7 @@ func (s *Session) waitForSend(hdr header, body io.Reader) error {
|
||||
// waitForSendErr waits to send a header with optional data, checking for a
|
||||
// potential shutdown. Since there's the expectation that sends can happen
|
||||
// in a timely manner, we enforce the connection write timeout here.
|
||||
func (s *Session) waitForSendErr(hdr header, body io.Reader, errCh chan error) error {
|
||||
func (s *Session) waitForSendErr(hdr header, body []byte, errCh chan error) error {
|
||||
t := timerPool.Get()
|
||||
timer := t.(*time.Timer)
|
||||
timer.Reset(s.config.ConnectionWriteTimeout)
|
||||
@@ -440,7 +440,7 @@ func (s *Session) send() {
|
||||
|
||||
// Send data from a body if given
|
||||
if ready.Body != nil {
|
||||
_, err := io.Copy(s.conn, ready.Body)
|
||||
_, err := s.conn.Write(ready.Body)
|
||||
if err != nil {
|
||||
s.logger.Printf("[ERR] yamux: Failed to write body: %v", err)
|
||||
asyncSendErr(ready.Err, err)
|
||||
|
||||
4
vendor/github.com/hashicorp/yamux/stream.go
сгенерированный
поставляемый
4
vendor/github.com/hashicorp/yamux/stream.go
сгенерированный
поставляемый
@@ -169,7 +169,7 @@ func (s *Stream) Write(b []byte) (n int, err error) {
|
||||
func (s *Stream) write(b []byte) (n int, err error) {
|
||||
var flags uint16
|
||||
var max uint32
|
||||
var body io.Reader
|
||||
var body []byte
|
||||
START:
|
||||
s.stateLock.Lock()
|
||||
switch s.state {
|
||||
@@ -195,7 +195,7 @@ START:
|
||||
|
||||
// Send up to our send window
|
||||
max = min(window, uint32(len(b)))
|
||||
body = bytes.NewReader(b[:max])
|
||||
body = b[:max]
|
||||
|
||||
// Send the header
|
||||
s.sendHdr.encode(typeData, flags, s.id, max)
|
||||
|
||||
11
vendor/github.com/jaytaylor/html2text/README.md
сгенерированный
поставляемый
11
vendor/github.com/jaytaylor/html2text/README.md
сгенерированный
поставляемый
@@ -6,7 +6,6 @@
|
||||
|
||||
### Converts HTML into text of the markdown-flavored variety
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
Ensure your emails are readable by all!
|
||||
@@ -19,7 +18,6 @@ There are still lots of improvements to be had, but FWIW this has worked fine fo
|
||||
|
||||
It requires go 1.x or newer ;)
|
||||
|
||||
|
||||
## Download the package
|
||||
|
||||
```bash
|
||||
@@ -28,6 +26,8 @@ go get jaytaylor.com/html2text
|
||||
|
||||
## Example usage
|
||||
|
||||
### Library
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
@@ -110,6 +110,11 @@ Here is some more information:
|
||||
+-------------+-------------+
|
||||
```
|
||||
|
||||
### Command line
|
||||
|
||||
```
|
||||
echo '<div>hi</div>' | html2text
|
||||
```
|
||||
|
||||
## Unit-tests
|
||||
|
||||
@@ -119,12 +124,10 @@ Running the unit-tests is straightforward and standard:
|
||||
go test
|
||||
```
|
||||
|
||||
|
||||
# License
|
||||
|
||||
Permissive MIT license.
|
||||
|
||||
|
||||
## Contact
|
||||
|
||||
You are more than welcome to open issues and send pull requests if you find a bug or want a new feature.
|
||||
|
||||
23
vendor/github.com/jaytaylor/html2text/html2text.go
сгенерированный
поставляемый
23
vendor/github.com/jaytaylor/html2text/html2text.go
сгенерированный
поставляемый
@@ -18,6 +18,7 @@ type Options struct {
|
||||
PrettyTables bool // Turns on pretty ASCII rendering for table elements.
|
||||
PrettyTablesOptions *PrettyTablesOptions // Configures pretty ASCII rendering for table elements.
|
||||
OmitLinks bool // Turns on omitting links
|
||||
TextOnly bool // Returns only plain text
|
||||
}
|
||||
|
||||
// PrettyTablesOptions overrides tablewriter behaviors
|
||||
@@ -157,6 +158,9 @@ func (ctx *textifyTraverseContext) handleElement(node *html.Node) error {
|
||||
}
|
||||
|
||||
str := subCtx.buf.String()
|
||||
if ctx.options.TextOnly {
|
||||
return ctx.emit(str + ".\n\n")
|
||||
}
|
||||
dividerLen := 0
|
||||
for _, line := range strings.Split(str, "\n") {
|
||||
if lineLen := len([]rune(line)); lineLen-1 > dividerLen {
|
||||
@@ -177,7 +181,9 @@ func (ctx *textifyTraverseContext) handleElement(node *html.Node) error {
|
||||
|
||||
case atom.Blockquote:
|
||||
ctx.blockquoteLevel++
|
||||
ctx.prefix = strings.Repeat(">", ctx.blockquoteLevel) + " "
|
||||
if !ctx.options.TextOnly {
|
||||
ctx.prefix = strings.Repeat(">", ctx.blockquoteLevel) + " "
|
||||
}
|
||||
if err := ctx.emit("\n"); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -190,7 +196,9 @@ func (ctx *textifyTraverseContext) handleElement(node *html.Node) error {
|
||||
return err
|
||||
}
|
||||
ctx.blockquoteLevel--
|
||||
ctx.prefix = strings.Repeat(">", ctx.blockquoteLevel)
|
||||
if !ctx.options.TextOnly {
|
||||
ctx.prefix = strings.Repeat(">", ctx.blockquoteLevel)
|
||||
}
|
||||
if ctx.blockquoteLevel > 0 {
|
||||
ctx.prefix += " "
|
||||
}
|
||||
@@ -213,8 +221,10 @@ func (ctx *textifyTraverseContext) handleElement(node *html.Node) error {
|
||||
return err
|
||||
|
||||
case atom.Li:
|
||||
if err := ctx.emit("* "); err != nil {
|
||||
return err
|
||||
if !ctx.options.TextOnly {
|
||||
if err := ctx.emit("* "); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := ctx.traverseChildren(node); err != nil {
|
||||
@@ -230,6 +240,9 @@ func (ctx *textifyTraverseContext) handleElement(node *html.Node) error {
|
||||
return err
|
||||
}
|
||||
str := subCtx.buf.String()
|
||||
if ctx.options.TextOnly {
|
||||
return ctx.emit(str + ".")
|
||||
}
|
||||
return ctx.emit("*" + str + "*")
|
||||
|
||||
case atom.A:
|
||||
@@ -254,7 +267,7 @@ func (ctx *textifyTraverseContext) handleElement(node *html.Node) error {
|
||||
if attrVal := getAttrVal(node, "href"); attrVal != "" {
|
||||
attrVal = ctx.normalizeHrefLink(attrVal)
|
||||
// Don't print link href if it matches link element content or if the link is empty.
|
||||
if !ctx.options.OmitLinks && attrVal != "" && linkText != attrVal {
|
||||
if (!ctx.options.OmitLinks && attrVal != "" && linkText != attrVal) || !ctx.options.TextOnly {
|
||||
hrefLink = "( " + attrVal + " )"
|
||||
}
|
||||
}
|
||||
|
||||
15
vendor/github.com/klauspost/compress/s2/.gitignore
сгенерированный
поставляемый
Обычный файл
15
vendor/github.com/klauspost/compress/s2/.gitignore
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,15 @@
|
||||
testdata/bench
|
||||
|
||||
# These explicitly listed benchmark data files are for an obsolete version of
|
||||
# snappy_test.go.
|
||||
testdata/alice29.txt
|
||||
testdata/asyoulik.txt
|
||||
testdata/fireworks.jpeg
|
||||
testdata/geo.protodata
|
||||
testdata/html
|
||||
testdata/html_x_4
|
||||
testdata/kppkn.gtb
|
||||
testdata/lcet10.txt
|
||||
testdata/paper-100k.pdf
|
||||
testdata/plrabn12.txt
|
||||
testdata/urls.10K
|
||||
28
vendor/github.com/klauspost/compress/s2/LICENSE
сгенерированный
поставляемый
Обычный файл
28
vendor/github.com/klauspost/compress/s2/LICENSE
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,28 @@
|
||||
Copyright (c) 2011 The Snappy-Go Authors. All rights reserved.
|
||||
Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
717
vendor/github.com/klauspost/compress/s2/README.md
сгенерированный
поставляемый
Обычный файл
717
vendor/github.com/klauspost/compress/s2/README.md
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,717 @@
|
||||
# S2 Compression
|
||||
|
||||
S2 is an extension of [Snappy](https://github.com/google/snappy).
|
||||
|
||||
S2 is aimed for high throughput, which is why it features concurrent compression for bigger payloads.
|
||||
|
||||
Decoding is compatible with Snappy compressed content, but content compressed with S2 cannot be decompressed by Snappy.
|
||||
This means that S2 can seamlessly replace Snappy without converting compressed content.
|
||||
|
||||
S2 can produce Snappy compatible output, faster and better than Snappy.
|
||||
If you want full benefit of the changes you should use s2 without Snappy compatibility.
|
||||
|
||||
S2 is designed to have high throughput on content that cannot be compressed.
|
||||
This is important, so you don't have to worry about spending CPU cycles on already compressed data.
|
||||
|
||||
## Benefits over Snappy
|
||||
|
||||
* Better compression
|
||||
* Adjustable compression (3 levels)
|
||||
* Concurrent stream compression
|
||||
* Faster decompression, even for Snappy compatible content
|
||||
* Ability to quickly skip forward in compressed stream
|
||||
* Compatible with reading Snappy compressed content
|
||||
* Smaller block size overhead on incompressible blocks
|
||||
* Block concatenation
|
||||
* Uncompressed stream mode
|
||||
* Automatic stream size padding
|
||||
* Snappy compatible block compression
|
||||
|
||||
## Drawbacks over Snappy
|
||||
|
||||
* Not optimized for 32 bit systems.
|
||||
* Streams use slightly more memory due to larger blocks and concurrency (configurable).
|
||||
|
||||
# Usage
|
||||
|
||||
Installation: `go get -u github.com/klauspost/compress/s2`
|
||||
|
||||
Full package documentation:
|
||||
|
||||
[![godoc][1]][2]
|
||||
|
||||
[1]: https://godoc.org/github.com/klauspost/compress?status.svg
|
||||
[2]: https://godoc.org/github.com/klauspost/compress/s2
|
||||
|
||||
## Compression
|
||||
|
||||
```Go
|
||||
func EncodeStream(src io.Reader, dst io.Writer) error {
|
||||
enc := s2.NewWriter(dst)
|
||||
_, err := io.Copy(enc, src)
|
||||
if err != nil {
|
||||
enc.Close()
|
||||
return err
|
||||
}
|
||||
// Blocks until compression is done.
|
||||
return enc.Close()
|
||||
}
|
||||
```
|
||||
|
||||
You should always call `enc.Close()`, otherwise you will leak resources and your encode will be incomplete.
|
||||
|
||||
For the best throughput, you should attempt to reuse the `Writer` using the `Reset()` method.
|
||||
|
||||
The Writer in S2 is always buffered, therefore `NewBufferedWriter` in Snappy can be replaced with `NewWriter` in S2.
|
||||
It is possible to flush any buffered data using the `Flush()` method.
|
||||
This will block until all data sent to the encoder has been written to the output.
|
||||
|
||||
S2 also supports the `io.ReaderFrom` interface, which will consume all input from a reader.
|
||||
|
||||
As a final method to compress data, if you have a single block of data you would like to have encoded as a stream,
|
||||
a slightly more efficient method is to use the `EncodeBuffer` method.
|
||||
This will take ownership of the buffer until the stream is closed.
|
||||
|
||||
```Go
|
||||
func EncodeStream(src []byte, dst io.Writer) error {
|
||||
enc := s2.NewWriter(dst)
|
||||
// The encoder owns the buffer until Flush or Close is called.
|
||||
err := enc.EncodeBuffer(buf)
|
||||
if err != nil {
|
||||
enc.Close()
|
||||
return err
|
||||
}
|
||||
// Blocks until compression is done.
|
||||
return enc.Close()
|
||||
}
|
||||
```
|
||||
|
||||
Each call to `EncodeBuffer` will result in discrete blocks being created without buffering,
|
||||
so it should only be used a single time per stream.
|
||||
If you need to write several blocks, you should use the regular io.Writer interface.
|
||||
|
||||
|
||||
## Decompression
|
||||
|
||||
```Go
|
||||
func DecodeStream(src io.Reader, dst io.Writer) error {
|
||||
dec := s2.NewReader(src)
|
||||
_, err := io.Copy(dst, dec)
|
||||
return err
|
||||
}
|
||||
```
|
||||
|
||||
Similar to the Writer, a Reader can be reused using the `Reset` method.
|
||||
|
||||
For the best possible throughput, there is a `EncodeBuffer(buf []byte)` function available.
|
||||
However, it requires that the provided buffer isn't used after it is handed over to S2 and until the stream is flushed or closed.
|
||||
|
||||
For smaller data blocks, there is also a non-streaming interface: `Encode()`, `EncodeBetter()` and `Decode()`.
|
||||
Do however note that these functions (similar to Snappy) does not provide validation of data,
|
||||
so data corruption may be undetected. Stream encoding provides CRC checks of data.
|
||||
|
||||
It is possible to efficiently skip forward in a compressed stream using the `Skip()` method.
|
||||
For big skips the decompressor is able to skip blocks without decompressing them.
|
||||
|
||||
## Single Blocks
|
||||
|
||||
Similar to Snappy S2 offers single block compression.
|
||||
Blocks do not offer the same flexibility and safety as streams,
|
||||
but may be preferable for very small payloads, less than 100K.
|
||||
|
||||
Using a simple `dst := s2.Encode(nil, src)` will compress `src` and return the compressed result.
|
||||
It is possible to provide a destination buffer.
|
||||
If the buffer has a capacity of `s2.MaxEncodedLen(len(src))` it will be used.
|
||||
If not a new will be allocated.
|
||||
|
||||
Alternatively `EncodeBetter`/`EncodeBest` can also be used for better, but slightly slower compression.
|
||||
|
||||
Similarly to decompress a block you can use `dst, err := s2.Decode(nil, src)`.
|
||||
Again an optional destination buffer can be supplied.
|
||||
The `s2.DecodedLen(src)` can be used to get the minimum capacity needed.
|
||||
If that is not satisfied a new buffer will be allocated.
|
||||
|
||||
Block function always operate on a single goroutine since it should only be used for small payloads.
|
||||
|
||||
# Commandline tools
|
||||
|
||||
Some very simply commandline tools are provided; `s2c` for compression and `s2d` for decompression.
|
||||
|
||||
Binaries can be downloaded on the [Releases Page](https://github.com/klauspost/compress/releases).
|
||||
|
||||
Installing then requires Go to be installed. To install them, use:
|
||||
|
||||
`go install github.com/klauspost/compress/s2/cmd/s2c && go install github.com/klauspost/compress/s2/cmd/s2d`
|
||||
|
||||
To build binaries to the current folder use:
|
||||
|
||||
`go build github.com/klauspost/compress/s2/cmd/s2c && go build github.com/klauspost/compress/s2/cmd/s2d`
|
||||
|
||||
|
||||
## s2c
|
||||
|
||||
```
|
||||
Usage: s2c [options] file1 file2
|
||||
|
||||
Compresses all files supplied as input separately.
|
||||
Output files are written as 'filename.ext.s2' or 'filename.ext.snappy'.
|
||||
By default output files will be overwritten.
|
||||
Use - as the only file name to read from stdin and write to stdout.
|
||||
|
||||
Wildcards are accepted: testdir/*.txt will compress all files in testdir ending with .txt
|
||||
Directories can be wildcards as well. testdir/*/*.txt will match testdir/subdir/b.txt
|
||||
|
||||
File names beginning with 'http://' and 'https://' will be downloaded and compressed.
|
||||
Only http response code 200 is accepted.
|
||||
|
||||
Options:
|
||||
-bench int
|
||||
Run benchmark n times. No output will be written
|
||||
-blocksize string
|
||||
Max block size. Examples: 64K, 256K, 1M, 4M. Must be power of two and <= 4MB (default "4M")
|
||||
-c Write all output to stdout. Multiple input files will be concatenated
|
||||
-cpu int
|
||||
Compress using this amount of threads (default 32)
|
||||
-faster
|
||||
Compress faster, but with a minor compression loss
|
||||
-help
|
||||
Display help
|
||||
-o string
|
||||
Write output to another file. Single input file only
|
||||
-pad string
|
||||
Pad size to a multiple of this value, Examples: 500, 64K, 256K, 1M, 4M, etc (default "1")
|
||||
-q Don't write any output to terminal, except errors
|
||||
-rm
|
||||
Delete source file(s) after successful compression
|
||||
-safe
|
||||
Do not overwrite output files
|
||||
-slower
|
||||
Compress more, but a lot slower
|
||||
-snappy
|
||||
Generate Snappy compatible output stream
|
||||
-verify
|
||||
Verify written files
|
||||
|
||||
```
|
||||
|
||||
## s2d
|
||||
|
||||
```
|
||||
Usage: s2d [options] file1 file2
|
||||
|
||||
Decompresses all files supplied as input. Input files must end with '.s2' or '.snappy'.
|
||||
Output file names have the extension removed. By default output files will be overwritten.
|
||||
Use - as the only file name to read from stdin and write to stdout.
|
||||
|
||||
Wildcards are accepted: testdir/*.txt will compress all files in testdir ending with .txt
|
||||
Directories can be wildcards as well. testdir/*/*.txt will match testdir/subdir/b.txt
|
||||
|
||||
File names beginning with 'http://' and 'https://' will be downloaded and decompressed.
|
||||
Extensions on downloaded files are ignored. Only http response code 200 is accepted.
|
||||
|
||||
Options:
|
||||
-bench int
|
||||
Run benchmark n times. No output will be written
|
||||
-c Write all output to stdout. Multiple input files will be concatenated
|
||||
-help
|
||||
Display help
|
||||
-o string
|
||||
Write output to another file. Single input file only
|
||||
-q Don't write any output to terminal, except errors
|
||||
-rm
|
||||
Delete source file(s) after successful decompression
|
||||
-safe
|
||||
Do not overwrite output files
|
||||
-verify
|
||||
Verify files, but do not write output
|
||||
```
|
||||
|
||||
## s2sx: self-extracting archives
|
||||
|
||||
s2sx allows creating self-extracting archives with no dependencies.
|
||||
|
||||
By default, executables are created for the same platforms as the host os,
|
||||
but this can be overridden with `-os` and `-arch` parameters.
|
||||
|
||||
Extracted files have 0666 permissions, except when untar option used.
|
||||
|
||||
```
|
||||
Usage: s2sx [options] file1 file2
|
||||
|
||||
Compresses all files supplied as input separately.
|
||||
If files have '.s2' extension they are assumed to be compressed already.
|
||||
Output files are written as 'filename.s2sx' and with '.exe' for windows targets.
|
||||
If output is big, an additional file with ".more" is written. This must be included as well.
|
||||
By default output files will be overwritten.
|
||||
|
||||
Wildcards are accepted: testdir/*.txt will compress all files in testdir ending with .txt
|
||||
Directories can be wildcards as well. testdir/*/*.txt will match testdir/subdir/b.txt
|
||||
|
||||
Options:
|
||||
-arch string
|
||||
Destination architecture (default "amd64")
|
||||
-c Write all output to stdout. Multiple input files will be concatenated
|
||||
-cpu int
|
||||
Compress using this amount of threads (default 32)
|
||||
-help
|
||||
Display help
|
||||
-max string
|
||||
Maximum executable size. Rest will be written to another file. (default "1G")
|
||||
-os string
|
||||
Destination operating system (default "windows")
|
||||
-q Don't write any output to terminal, except errors
|
||||
-rm
|
||||
Delete source file(s) after successful compression
|
||||
-safe
|
||||
Do not overwrite output files
|
||||
-untar
|
||||
Untar on destination
|
||||
```
|
||||
|
||||
Available platforms are:
|
||||
|
||||
* darwin-amd64
|
||||
* darwin-arm64
|
||||
* linux-amd64
|
||||
* linux-arm
|
||||
* linux-arm64
|
||||
* linux-mips64
|
||||
* linux-ppc64le
|
||||
* windows-386
|
||||
* windows-amd64
|
||||
|
||||
By default, there is a size limit of 1GB for the output executable.
|
||||
|
||||
When this is exceeded the remaining file content is written to a file called
|
||||
output+`.more`. This file must be included for a successful extraction and
|
||||
placed alongside the executable for a successful extraction.
|
||||
|
||||
This file *must* have the same name as the executable, so if the executable is renamed,
|
||||
so must the `.more` file.
|
||||
|
||||
This functionality is disabled with stdin/stdout.
|
||||
|
||||
### Self-extracting TAR files
|
||||
|
||||
If you wrap a TAR file you can specify `-untar` to make it untar on the destination host.
|
||||
|
||||
Files are extracted to the current folder with the path specified in the tar file.
|
||||
|
||||
Note that tar files are not validated before they are wrapped.
|
||||
|
||||
For security reasons files that move below the root folder are not allowed.
|
||||
|
||||
# Performance
|
||||
|
||||
This section will focus on comparisons to Snappy.
|
||||
This package is solely aimed at replacing Snappy as a high speed compression package.
|
||||
If you are mainly looking for better compression [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd)
|
||||
gives better compression, but typically at speeds slightly below "better" mode in this package.
|
||||
|
||||
Compression is increased compared to Snappy, mostly around 5-20% and the throughput is typically 25-40% increased (single threaded) compared to the Snappy Go implementation.
|
||||
|
||||
Streams are concurrently compressed. The stream will be distributed among all available CPU cores for the best possible throughput.
|
||||
|
||||
A "better" compression mode is also available. This allows to trade a bit of speed for a minor compression gain.
|
||||
The content compressed in this mode is fully compatible with the standard decoder.
|
||||
|
||||
Snappy vs S2 **compression** speed on 16 core (32 thread) computer, using all threads and a single thread (1 CPU):
|
||||
|
||||
| File | S2 speed | S2 Throughput | S2 % smaller | S2 "better" | "better" throughput | "better" % smaller |
|
||||
|-----------------------------------------------------------------------------------------------------|----------|---------------|--------------|-------------|---------------------|--------------------|
|
||||
| [rawstudio-mint14.tar](https://files.klauspost.com/compress/rawstudio-mint14.7z) | 12.70x | 10556 MB/s | 7.35% | 4.15x | 3455 MB/s | 12.79% |
|
||||
| (1 CPU) | 1.14x | 948 MB/s | - | 0.42x | 349 MB/s | - |
|
||||
| [github-june-2days-2019.json](https://files.klauspost.com/compress/github-june-2days-2019.json.zst) | 17.13x | 14484 MB/s | 31.60% | 10.09x | 8533 MB/s | 37.71% |
|
||||
| (1 CPU) | 1.33x | 1127 MB/s | - | 0.70x | 589 MB/s | - |
|
||||
| [github-ranks-backup.bin](https://files.klauspost.com/compress/github-ranks-backup.bin.zst) | 15.14x | 12000 MB/s | -5.79% | 6.59x | 5223 MB/s | 5.80% |
|
||||
| (1 CPU) | 1.11x | 877 MB/s | - | 0.47x | 370 MB/s | - |
|
||||
| [consensus.db.10gb](https://files.klauspost.com/compress/consensus.db.10gb.zst) | 14.62x | 12116 MB/s | 15.90% | 5.35x | 4430 MB/s | 16.08% |
|
||||
| (1 CPU) | 1.38x | 1146 MB/s | - | 0.38x | 312 MB/s | - |
|
||||
| [adresser.json](https://files.klauspost.com/compress/adresser.json.zst) | 8.83x | 17579 MB/s | 43.86% | 6.54x | 13011 MB/s | 47.23% |
|
||||
| (1 CPU) | 1.14x | 2259 MB/s | - | 0.74x | 1475 MB/s | - |
|
||||
| [gob-stream](https://files.klauspost.com/compress/gob-stream.7z) | 16.72x | 14019 MB/s | 24.02% | 10.11x | 8477 MB/s | 30.48% |
|
||||
| (1 CPU) | 1.24x | 1043 MB/s | - | 0.70x | 586 MB/s | - |
|
||||
| [10gb.tar](http://mattmahoney.net/dc/10gb.html) | 13.33x | 9254 MB/s | 1.84% | 6.75x | 4686 MB/s | 6.72% |
|
||||
| (1 CPU) | 0.97x | 672 MB/s | - | 0.53x | 366 MB/s | - |
|
||||
| sharnd.out.2gb | 2.11x | 12639 MB/s | 0.01% | 1.98x | 11833 MB/s | 0.01% |
|
||||
| (1 CPU) | 0.93x | 5594 MB/s | - | 1.34x | 8030 MB/s | - |
|
||||
| [enwik9](http://mattmahoney.net/dc/textdata.html) | 19.34x | 8220 MB/s | 3.98% | 7.87x | 3345 MB/s | 15.82% |
|
||||
| (1 CPU) | 1.06x | 452 MB/s | - | 0.50x | 213 MB/s | - |
|
||||
| [silesia.tar](http://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip) | 10.48x | 6124 MB/s | 5.67% | 3.76x | 2197 MB/s | 12.60% |
|
||||
| (1 CPU) | 0.97x | 568 MB/s | - | 0.46x | 271 MB/s | - |
|
||||
| [enwik10](https://encode.su/threads/3315-enwik10-benchmark-results) | 21.07x | 9020 MB/s | 6.36% | 6.91x | 2959 MB/s | 16.95% |
|
||||
| (1 CPU) | 1.07x | 460 MB/s | - | 0.51x | 220 MB/s | - |
|
||||
|
||||
### Legend
|
||||
|
||||
* `S2 speed`: Speed of S2 compared to Snappy, using 16 cores and 1 core.
|
||||
* `S2 throughput`: Throughput of S2 in MB/s.
|
||||
* `S2 % smaller`: How many percent of the Snappy output size is S2 better.
|
||||
* `S2 "better"`: Speed when enabling "better" compression mode in S2 compared to Snappy.
|
||||
* `"better" throughput`: Speed when enabling "better" compression mode in S2 compared to Snappy.
|
||||
* `"better" % smaller`: How many percent of the Snappy output size is S2 better when using "better" compression.
|
||||
|
||||
There is a good speedup across the board when using a single thread and a significant speedup when using multiple threads.
|
||||
|
||||
Machine generated data gets by far the biggest compression boost, with size being being reduced by up to 45% of Snappy size.
|
||||
|
||||
The "better" compression mode sees a good improvement in all cases, but usually at a performance cost.
|
||||
|
||||
Incompressible content (`sharnd.out.2gb`, 2GB random data) sees the smallest speedup.
|
||||
This is likely dominated by synchronization overhead, which is confirmed by the fact that single threaded performance is higher (see above).
|
||||
|
||||
## Decompression
|
||||
|
||||
S2 attempts to create content that is also fast to decompress, except in "better" mode where the smallest representation is used.
|
||||
|
||||
S2 vs Snappy **decompression** speed. Both operating on single core:
|
||||
|
||||
| File | S2 Throughput | vs. Snappy | Better Throughput | vs. Snappy |
|
||||
|-----------------------------------------------------------------------------------------------------|---------------|------------|-------------------|------------|
|
||||
| [rawstudio-mint14.tar](https://files.klauspost.com/compress/rawstudio-mint14.7z) | 2117 MB/s | 1.14x | 1738 MB/s | 0.94x |
|
||||
| [github-june-2days-2019.json](https://files.klauspost.com/compress/github-june-2days-2019.json.zst) | 2401 MB/s | 1.25x | 2307 MB/s | 1.20x |
|
||||
| [github-ranks-backup.bin](https://files.klauspost.com/compress/github-ranks-backup.bin.zst) | 2075 MB/s | 0.98x | 1764 MB/s | 0.83x |
|
||||
| [consensus.db.10gb](https://files.klauspost.com/compress/consensus.db.10gb.zst) | 2967 MB/s | 1.05x | 2885 MB/s | 1.02x |
|
||||
| [adresser.json](https://files.klauspost.com/compress/adresser.json.zst) | 4141 MB/s | 1.07x | 4184 MB/s | 1.08x |
|
||||
| [gob-stream](https://files.klauspost.com/compress/gob-stream.7z) | 2264 MB/s | 1.12x | 2185 MB/s | 1.08x |
|
||||
| [10gb.tar](http://mattmahoney.net/dc/10gb.html) | 1525 MB/s | 1.03x | 1347 MB/s | 0.91x |
|
||||
| sharnd.out.2gb | 3813 MB/s | 0.79x | 3900 MB/s | 0.81x |
|
||||
| [enwik9](http://mattmahoney.net/dc/textdata.html) | 1246 MB/s | 1.29x | 967 MB/s | 1.00x |
|
||||
| [silesia.tar](http://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip) | 1433 MB/s | 1.12x | 1203 MB/s | 0.94x |
|
||||
| [enwik10](https://encode.su/threads/3315-enwik10-benchmark-results) | 1284 MB/s | 1.32x | 1010 MB/s | 1.04x |
|
||||
|
||||
### Legend
|
||||
|
||||
* `S2 Throughput`: Decompression speed of S2 encoded content.
|
||||
* `Better Throughput`: Decompression speed of S2 "better" encoded content.
|
||||
* `vs Snappy`: Decompression speed of S2 "better" mode compared to Snappy and absolute speed.
|
||||
|
||||
|
||||
While the decompression code hasn't changed, there is a significant speedup in decompression speed.
|
||||
S2 prefers longer matches and will typically only find matches that are 6 bytes or longer.
|
||||
While this reduces compression a bit, it improves decompression speed.
|
||||
|
||||
The "better" compression mode will actively look for shorter matches, which is why it has a decompression speed quite similar to Snappy.
|
||||
|
||||
Without assembly decompression is also very fast; single goroutine decompression speed. No assembly:
|
||||
|
||||
| File | S2 Throughput | S2 throughput |
|
||||
|--------------------------------|--------------|---------------|
|
||||
| consensus.db.10gb.s2 | 1.84x | 2289.8 MB/s |
|
||||
| 10gb.tar.s2 | 1.30x | 867.07 MB/s |
|
||||
| rawstudio-mint14.tar.s2 | 1.66x | 1329.65 MB/s |
|
||||
| github-june-2days-2019.json.s2 | 2.36x | 1831.59 MB/s |
|
||||
| github-ranks-backup.bin.s2 | 1.73x | 1390.7 MB/s |
|
||||
| enwik9.s2 | 1.67x | 681.53 MB/s |
|
||||
| adresser.json.s2 | 3.41x | 4230.53 MB/s |
|
||||
| silesia.tar.s2 | 1.52x | 811.58 |
|
||||
|
||||
Even though S2 typically compresses better than Snappy, decompression speed is always better.
|
||||
|
||||
## Block compression
|
||||
|
||||
|
||||
When compressing blocks no concurrent compression is performed just as Snappy.
|
||||
This is because blocks are for smaller payloads and generally will not benefit from concurrent compression.
|
||||
|
||||
An important change is that incompressible blocks will not be more than at most 10 bytes bigger than the input.
|
||||
In rare, worst case scenario Snappy blocks could be significantly bigger than the input.
|
||||
|
||||
### Mixed content blocks
|
||||
|
||||
The most reliable is a wide dataset.
|
||||
For this we use [`webdevdata.org-2015-01-07-subset`](https://files.klauspost.com/compress/webdevdata.org-2015-01-07-4GB-subset.7z),
|
||||
53927 files, total input size: 4,014,735,833 bytes. Single goroutine used.
|
||||
|
||||
| * | Input | Output | Reduction | MB/s |
|
||||
|-------------------|------------|------------|-----------|--------|
|
||||
| S2 | 4014735833 | 1059723369 | 73.60% | **934.34** |
|
||||
| S2 Better | 4014735833 | 969670507 | 75.85% | 532.70 |
|
||||
| S2 Best | 4014735833 | 906625668 | **77.85%** | 46.84 |
|
||||
| Snappy | 4014735833 | 1128706759 | 71.89% | 762.59 |
|
||||
| S2, Snappy Output | 4014735833 | 1093821420 | 72.75% | 908.60 |
|
||||
| LZ4 | 4014735833 | 1079259294 | 73.12% | 526.94 |
|
||||
|
||||
S2 delivers both the best single threaded throughput with regular mode and the best compression rate with "best".
|
||||
"Better" mode provides the same compression speed as LZ4 with better compression ratio.
|
||||
|
||||
When outputting Snappy compatible output it still delivers better throughput (150MB/s more) and better compression.
|
||||
|
||||
As can be seen from the other benchmarks decompression should also be easier on the S2 generated output.
|
||||
|
||||
Though they cannot be compared due to different decompression speeds here are the speed/size comparisons for
|
||||
other Go compressors:
|
||||
|
||||
| * | Input | Output | Reduction | MB/s |
|
||||
|-------------------|------------|------------|-----------|--------|
|
||||
| Zstd Fastest (Go) | 4014735833 | 794608518 | 80.21% | 236.04 |
|
||||
| Zstd Best (Go) | 4014735833 | 704603356 | 82.45% | 35.63 |
|
||||
| Deflate (Go) l1 | 4014735833 | 871294239 | 78.30% | 214.04 |
|
||||
| Deflate (Go) l9 | 4014735833 | 730389060 | 81.81% | 41.17 |
|
||||
|
||||
### Standard block compression
|
||||
|
||||
Benchmarking single block performance is subject to a lot more variation since it only tests a limited number of file patterns.
|
||||
So individual benchmarks should only be seen as a guideline and the overall picture is more important.
|
||||
|
||||
These micro-benchmarks are with data in cache and trained branch predictors. For a more realistic benchmark see the mixed content above.
|
||||
|
||||
Block compression. Parallel benchmark running on 16 cores, 16 goroutines.
|
||||
|
||||
AMD64 assembly is use for both S2 and Snappy.
|
||||
|
||||
| Absolute Perf | Snappy size | S2 Size | Snappy Speed | S2 Speed | Snappy dec | S2 dec |
|
||||
|-----------------------|-------------|---------|--------------|-------------|-------------|-------------|
|
||||
| html | 22843 | 21111 | 16246 MB/s | 17438 MB/s | 40972 MB/s | 49263 MB/s |
|
||||
| urls.10K | 335492 | 287326 | 7943 MB/s | 9693 MB/s | 22523 MB/s | 26484 MB/s |
|
||||
| fireworks.jpeg | 123034 | 123100 | 349544 MB/s | 273889 MB/s | 718321 MB/s | 827552 MB/s |
|
||||
| fireworks.jpeg (200B) | 146 | 155 | 8869 MB/s | 17773 MB/s | 33691 MB/s | 52421 MB/s |
|
||||
| paper-100k.pdf | 85304 | 84459 | 167546 MB/s | 101263 MB/s | 326905 MB/s | 291944 MB/s |
|
||||
| html_x_4 | 92234 | 21113 | 15194 MB/s | 50670 MB/s | 30843 MB/s | 32217 MB/s |
|
||||
| alice29.txt | 88034 | 85975 | 5936 MB/s | 6139 MB/s | 12882 MB/s | 20044 MB/s |
|
||||
| asyoulik.txt | 77503 | 79650 | 5517 MB/s | 6366 MB/s | 12735 MB/s | 22806 MB/s |
|
||||
| lcet10.txt | 234661 | 220670 | 6235 MB/s | 6067 MB/s | 14519 MB/s | 18697 MB/s |
|
||||
| plrabn12.txt | 319267 | 317985 | 5159 MB/s | 5726 MB/s | 11923 MB/s | 19901 MB/s |
|
||||
| geo.protodata | 23335 | 18690 | 21220 MB/s | 26529 MB/s | 56271 MB/s | 62540 MB/s |
|
||||
| kppkn.gtb | 69526 | 65312 | 9732 MB/s | 8559 MB/s | 18491 MB/s | 18969 MB/s |
|
||||
| alice29.txt (128B) | 80 | 82 | 6691 MB/s | 15489 MB/s | 31883 MB/s | 38874 MB/s |
|
||||
| alice29.txt (1000B) | 774 | 774 | 12204 MB/s | 13000 MB/s | 48056 MB/s | 52341 MB/s |
|
||||
| alice29.txt (10000B) | 6648 | 6933 | 10044 MB/s | 12806 MB/s | 32378 MB/s | 46322 MB/s |
|
||||
| alice29.txt (20000B) | 12686 | 13574 | 7733 MB/s | 11210 MB/s | 30566 MB/s | 58969 MB/s |
|
||||
|
||||
|
||||
| Relative Perf | Snappy size | S2 size improved | S2 Speed | S2 Dec Speed |
|
||||
|-----------------------|-------------|------------------|----------|--------------|
|
||||
| html | 22.31% | 7.58% | 1.07x | 1.20x |
|
||||
| urls.10K | 47.78% | 14.36% | 1.22x | 1.18x |
|
||||
| fireworks.jpeg | 99.95% | -0.05% | 0.78x | 1.15x |
|
||||
| fireworks.jpeg (200B) | 73.00% | -6.16% | 2.00x | 1.56x |
|
||||
| paper-100k.pdf | 83.30% | 0.99% | 0.60x | 0.89x |
|
||||
| html_x_4 | 22.52% | 77.11% | 3.33x | 1.04x |
|
||||
| alice29.txt | 57.88% | 2.34% | 1.03x | 1.56x |
|
||||
| asyoulik.txt | 61.91% | -2.77% | 1.15x | 1.79x |
|
||||
| lcet10.txt | 54.99% | 5.96% | 0.97x | 1.29x |
|
||||
| plrabn12.txt | 66.26% | 0.40% | 1.11x | 1.67x |
|
||||
| geo.protodata | 19.68% | 19.91% | 1.25x | 1.11x |
|
||||
| kppkn.gtb | 37.72% | 6.06% | 0.88x | 1.03x |
|
||||
| alice29.txt (128B) | 62.50% | -2.50% | 2.31x | 1.22x |
|
||||
| alice29.txt (1000B) | 77.40% | 0.00% | 1.07x | 1.09x |
|
||||
| alice29.txt (10000B) | 66.48% | -4.29% | 1.27x | 1.43x |
|
||||
| alice29.txt (20000B) | 63.43% | -7.00% | 1.45x | 1.93x |
|
||||
|
||||
Speed is generally at or above Snappy. Small blocks gets a significant speedup, although at the expense of size.
|
||||
|
||||
Decompression speed is better than Snappy, except in one case.
|
||||
|
||||
Since payloads are very small the variance in terms of size is rather big, so they should only be seen as a general guideline.
|
||||
|
||||
Size is on average around Snappy, but varies on content type.
|
||||
In cases where compression is worse, it usually is compensated by a speed boost.
|
||||
|
||||
|
||||
### Better compression
|
||||
|
||||
Benchmarking single block performance is subject to a lot more variation since it only tests a limited number of file patterns.
|
||||
So individual benchmarks should only be seen as a guideline and the overall picture is more important.
|
||||
|
||||
| Absolute Perf | Snappy size | Better Size | Snappy Speed | Better Speed | Snappy dec | Better dec |
|
||||
|-----------------------|-------------|-------------|--------------|--------------|-------------|-------------|
|
||||
| html | 22843 | 19833 | 16246 MB/s | 7731 MB/s | 40972 MB/s | 40292 MB/s |
|
||||
| urls.10K | 335492 | 253529 | 7943 MB/s | 3980 MB/s | 22523 MB/s | 20981 MB/s |
|
||||
| fireworks.jpeg | 123034 | 123100 | 349544 MB/s | 9760 MB/s | 718321 MB/s | 823698 MB/s |
|
||||
| fireworks.jpeg (200B) | 146 | 142 | 8869 MB/s | 594 MB/s | 33691 MB/s | 30101 MB/s |
|
||||
| paper-100k.pdf | 85304 | 82915 | 167546 MB/s | 7470 MB/s | 326905 MB/s | 198869 MB/s |
|
||||
| html_x_4 | 92234 | 19841 | 15194 MB/s | 23403 MB/s | 30843 MB/s | 30937 MB/s |
|
||||
| alice29.txt | 88034 | 73218 | 5936 MB/s | 2945 MB/s | 12882 MB/s | 16611 MB/s |
|
||||
| asyoulik.txt | 77503 | 66844 | 5517 MB/s | 2739 MB/s | 12735 MB/s | 14975 MB/s |
|
||||
| lcet10.txt | 234661 | 190589 | 6235 MB/s | 3099 MB/s | 14519 MB/s | 16634 MB/s |
|
||||
| plrabn12.txt | 319267 | 270828 | 5159 MB/s | 2600 MB/s | 11923 MB/s | 13382 MB/s |
|
||||
| geo.protodata | 23335 | 18278 | 21220 MB/s | 11208 MB/s | 56271 MB/s | 57961 MB/s |
|
||||
| kppkn.gtb | 69526 | 61851 | 9732 MB/s | 4556 MB/s | 18491 MB/s | 16524 MB/s |
|
||||
| alice29.txt (128B) | 80 | 81 | 6691 MB/s | 529 MB/s | 31883 MB/s | 34225 MB/s |
|
||||
| alice29.txt (1000B) | 774 | 748 | 12204 MB/s | 1943 MB/s | 48056 MB/s | 42068 MB/s |
|
||||
| alice29.txt (10000B) | 6648 | 6234 | 10044 MB/s | 2949 MB/s | 32378 MB/s | 28813 MB/s |
|
||||
| alice29.txt (20000B) | 12686 | 11584 | 7733 MB/s | 2822 MB/s | 30566 MB/s | 27315 MB/s |
|
||||
|
||||
|
||||
| Relative Perf | Snappy size | Better size | Better Speed | Better dec |
|
||||
|-----------------------|-------------|-------------|--------------|------------|
|
||||
| html | 22.31% | 13.18% | 0.48x | 0.98x |
|
||||
| urls.10K | 47.78% | 24.43% | 0.50x | 0.93x |
|
||||
| fireworks.jpeg | 99.95% | -0.05% | 0.03x | 1.15x |
|
||||
| fireworks.jpeg (200B) | 73.00% | 2.74% | 0.07x | 0.89x |
|
||||
| paper-100k.pdf | 83.30% | 2.80% | 0.07x | 0.61x |
|
||||
| html_x_4 | 22.52% | 78.49% | 0.04x | 1.00x |
|
||||
| alice29.txt | 57.88% | 16.83% | 1.54x | 1.29x |
|
||||
| asyoulik.txt | 61.91% | 13.75% | 0.50x | 1.18x |
|
||||
| lcet10.txt | 54.99% | 18.78% | 0.50x | 1.15x |
|
||||
| plrabn12.txt | 66.26% | 15.17% | 0.50x | 1.12x |
|
||||
| geo.protodata | 19.68% | 21.67% | 0.50x | 1.03x |
|
||||
| kppkn.gtb | 37.72% | 11.04% | 0.53x | 0.89x |
|
||||
| alice29.txt (128B) | 62.50% | -1.25% | 0.47x | 1.07x |
|
||||
| alice29.txt (1000B) | 77.40% | 3.36% | 0.08x | 0.88x |
|
||||
| alice29.txt (10000B) | 66.48% | 6.23% | 0.16x | 0.89x |
|
||||
| alice29.txt (20000B) | 63.43% | 8.69% | 0.29x | 0.89x |
|
||||
|
||||
Except for the mostly incompressible JPEG image compression is better and usually in the
|
||||
double digits in terms of percentage reduction over Snappy.
|
||||
|
||||
The PDF sample shows a significant slowdown compared to Snappy, as this mode tries harder
|
||||
to compress the data. Very small blocks are also not favorable for better compression, so throughput is way down.
|
||||
|
||||
This mode aims to provide better compression at the expense of performance and achieves that
|
||||
without a huge performance penalty, except on very small blocks.
|
||||
|
||||
Decompression speed suffers a little compared to the regular S2 mode,
|
||||
but still manages to be close to Snappy in spite of increased compression.
|
||||
|
||||
# Best compression mode
|
||||
|
||||
S2 offers a "best" compression mode.
|
||||
|
||||
This will compress as much as possible with little regard to CPU usage.
|
||||
|
||||
Mainly for offline compression, but where decompression speed should still
|
||||
be high and compatible with other S2 compressed data.
|
||||
|
||||
Some examples compared on 16 core CPU, amd64 assembly used:
|
||||
|
||||
```
|
||||
* enwik10
|
||||
Default... 10000000000 -> 4761467548 [47.61%]; 1.098s, 8685.6MB/s
|
||||
Better... 10000000000 -> 4219438251 [42.19%]; 1.925s, 4954.2MB/s
|
||||
Best... 10000000000 -> 3627364337 [36.27%]; 43.051s, 221.5MB/s
|
||||
|
||||
* github-june-2days-2019.json
|
||||
Default... 6273951764 -> 1043196283 [16.63%]; 431ms, 13882.3MB/s
|
||||
Better... 6273951764 -> 949146808 [15.13%]; 547ms, 10938.4MB/s
|
||||
Best... 6273951764 -> 832855506 [13.27%]; 9.455s, 632.8MB/s
|
||||
|
||||
* nyc-taxi-data-10M.csv
|
||||
Default... 3325605752 -> 1095998837 [32.96%]; 324ms, 9788.7MB/s
|
||||
Better... 3325605752 -> 954776589 [28.71%]; 491ms, 6459.4MB/s
|
||||
Best... 3325605752 -> 779098746 [23.43%]; 8.29s, 382.6MB/s
|
||||
|
||||
* 10gb.tar
|
||||
Default... 10065157632 -> 5916578242 [58.78%]; 1.028s, 9337.4MB/s
|
||||
Better... 10065157632 -> 5649207485 [56.13%]; 1.597s, 6010.6MB/s
|
||||
Best... 10065157632 -> 5208719802 [51.75%]; 32.78s, 292.8MB/
|
||||
|
||||
* consensus.db.10gb
|
||||
Default... 10737418240 -> 4562648848 [42.49%]; 882ms, 11610.0MB/s
|
||||
Better... 10737418240 -> 4542428129 [42.30%]; 1.533s, 6679.7MB/s
|
||||
Best... 10737418240 -> 4244773384 [39.53%]; 42.96s, 238.4MB/s
|
||||
```
|
||||
|
||||
Decompression speed should be around the same as using the 'better' compression mode.
|
||||
|
||||
# Snappy Compatibility
|
||||
|
||||
S2 now offers full compatibility with Snappy.
|
||||
|
||||
This means that the efficient encoders of S2 can be used to generate fully Snappy compatible output.
|
||||
|
||||
There is a [snappy](https://github.com/klauspost/compress/tree/master/snappy) package that can be used by
|
||||
simply changing imports from `github.com/golang/snappy` to `github.com/klauspost/compress/snappy`.
|
||||
This uses "better" mode for all operations.
|
||||
If you would like more control, you can use the s2 package as described below:
|
||||
|
||||
## Blocks
|
||||
|
||||
Snappy compatible blocks can be generated with the S2 encoder.
|
||||
Compression and speed is typically a bit better `MaxEncodedLen` is also smaller for smaller memory usage. Replace
|
||||
|
||||
| Snappy | S2 replacement |
|
||||
|----------------------------|-------------------------|
|
||||
| snappy.Encode(...) | s2.EncodeSnappy(...) |
|
||||
| snappy.MaxEncodedLen(...) | s2.MaxEncodedLen(...) |
|
||||
|
||||
`s2.EncodeSnappy` can be replaced with `s2.EncodeSnappyBetter` or `s2.EncodeSnappyBest` to get more efficiently compressed snappy compatible output.
|
||||
|
||||
`s2.ConcatBlocks` is compatible with snappy blocks.
|
||||
|
||||
Comparison of [`webdevdata.org-2015-01-07-subset`](https://files.klauspost.com/compress/webdevdata.org-2015-01-07-4GB-subset.7z),
|
||||
53927 files, total input size: 4,014,735,833 bytes. amd64, single goroutine used:
|
||||
|
||||
| Encoder | Size | MB/s | Reduction |
|
||||
|-----------------------|------------|--------|------------
|
||||
| snappy.Encode | 1128706759 | 725.59 | 71.89% |
|
||||
| s2.EncodeSnappy | 1093823291 | 899.16 | 72.75% |
|
||||
| s2.EncodeSnappyBetter | 1001158548 | 578.49 | 75.06% |
|
||||
| s2.EncodeSnappyBest | 944507998 | 66.00 | 76.47% |
|
||||
|
||||
## Streams
|
||||
|
||||
For streams, replace `enc = snappy.NewBufferedWriter(w)` with `enc = s2.NewWriter(w, s2.WriterSnappyCompat())`.
|
||||
All other options are available, but note that block size limit is different for snappy.
|
||||
|
||||
Comparison of different streams, AMD Ryzen 3950x, 16 cores. Size and throughput:
|
||||
|
||||
| File | snappy.NewWriter | S2 Snappy | S2 Snappy, Better | S2 Snappy, Best |
|
||||
|-----------------------------|--------------------------|---------------------------|--------------------------|-------------------------|
|
||||
| nyc-taxi-data-10M.csv | 1316042016 - 517.54MB/s | 1307003093 - 8406.29MB/s | 1174534014 - 4984.35MB/s | 1115904679 - 177.81MB/s |
|
||||
| enwik10 | 5088294643 - 433.45MB/s | 5175840939 - 8454.52MB/s | 4560784526 - 4403.10MB/s | 4340299103 - 159.71MB/s |
|
||||
| 10gb.tar | 6056946612 - 703.25MB/s | 6208571995 - 9035.75MB/s | 5741646126 - 2402.08MB/s | 5548973895 - 171.17MB/s |
|
||||
| github-june-2days-2019.json | 1525176492 - 908.11MB/s | 1476519054 - 12625.93MB/s | 1400547532 - 6163.61MB/s | 1321887137 - 200.71MB/s |
|
||||
| consensus.db.10gb | 5412897703 - 1054.38MB/s | 5354073487 - 12634.82MB/s | 5335069899 - 2472.23MB/s | 5201000954 - 166.32MB/s |
|
||||
|
||||
# Decompression
|
||||
|
||||
All decompression functions map directly to equivalent s2 functions.
|
||||
|
||||
| Snappy | S2 replacement |
|
||||
|------------------------|--------------------|
|
||||
| snappy.Decode(...) | s2.Decode(...) |
|
||||
| snappy.DecodedLen(...) | s2.DecodedLen(...) |
|
||||
| snappy.NewReader(...) | s2.NewReader(...) |
|
||||
|
||||
Features like [quick forward skipping without decompression](https://pkg.go.dev/github.com/klauspost/compress/s2#Reader.Skip)
|
||||
are also available for Snappy streams.
|
||||
|
||||
If you know you are only decompressing snappy streams, setting [`ReaderMaxBlockSize(64<<10)`](https://pkg.go.dev/github.com/klauspost/compress/s2#ReaderMaxBlockSize)
|
||||
on your Reader will reduce memory consumption.
|
||||
|
||||
# Concatenating blocks and streams.
|
||||
|
||||
Concatenating streams will concatenate the output of both without recompressing them.
|
||||
While this is inefficient in terms of compression it might be usable in certain scenarios.
|
||||
The 10 byte 'stream identifier' of the second stream can optionally be stripped, but it is not a requirement.
|
||||
|
||||
Blocks can be concatenated using the `ConcatBlocks` function.
|
||||
|
||||
Snappy blocks/streams can safely be concatenated with S2 blocks and streams.
|
||||
|
||||
# Format Extensions
|
||||
|
||||
* Frame [Stream identifier](https://github.com/google/snappy/blob/master/framing_format.txt#L68) changed from `sNaPpY` to `S2sTwO`.
|
||||
* [Framed compressed blocks](https://github.com/google/snappy/blob/master/format_description.txt) can be up to 4MB (up from 64KB).
|
||||
* Compressed blocks can have an offset of `0`, which indicates to repeat the last seen offset.
|
||||
|
||||
Repeat offsets must be encoded as a [2.2.1. Copy with 1-byte offset (01)](https://github.com/google/snappy/blob/master/format_description.txt#L89), where the offset is 0.
|
||||
|
||||
The length is specified by reading the 3-bit length specified in the tag and decode using this table:
|
||||
|
||||
| Length | Actual Length |
|
||||
|--------|----------------------|
|
||||
| 0 | 4 |
|
||||
| 1 | 5 |
|
||||
| 2 | 6 |
|
||||
| 3 | 7 |
|
||||
| 4 | 8 |
|
||||
| 5 | 8 + read 1 byte |
|
||||
| 6 | 260 + read 2 bytes |
|
||||
| 7 | 65540 + read 3 bytes |
|
||||
|
||||
This allows any repeat offset + length to be represented by 2 to 5 bytes.
|
||||
|
||||
Lengths are stored as little endian values.
|
||||
|
||||
The first copy of a block cannot be a repeat offset and the offset is not carried across blocks in streams.
|
||||
|
||||
Default streaming block size is 1MB.
|
||||
|
||||
# LICENSE
|
||||
|
||||
This code is based on the [Snappy-Go](https://github.com/golang/snappy) implementation.
|
||||
|
||||
Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.
|
||||
565
vendor/github.com/klauspost/compress/s2/decode.go
сгенерированный
поставляемый
Обычный файл
565
vendor/github.com/klauspost/compress/s2/decode.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,565 @@
|
||||
// Copyright 2011 The Snappy-Go Authors. All rights reserved.
|
||||
// Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrCorrupt reports that the input is invalid.
|
||||
ErrCorrupt = errors.New("s2: corrupt input")
|
||||
// ErrCRC reports that the input failed CRC validation (streams only)
|
||||
ErrCRC = errors.New("s2: corrupt input, crc mismatch")
|
||||
// ErrTooLarge reports that the uncompressed length is too large.
|
||||
ErrTooLarge = errors.New("s2: decoded block is too large")
|
||||
// ErrUnsupported reports that the input isn't supported.
|
||||
ErrUnsupported = errors.New("s2: unsupported input")
|
||||
)
|
||||
|
||||
// DecodedLen returns the length of the decoded block.
|
||||
func DecodedLen(src []byte) (int, error) {
|
||||
v, _, err := decodedLen(src)
|
||||
return v, err
|
||||
}
|
||||
|
||||
// decodedLen returns the length of the decoded block and the number of bytes
|
||||
// that the length header occupied.
|
||||
func decodedLen(src []byte) (blockLen, headerLen int, err error) {
|
||||
v, n := binary.Uvarint(src)
|
||||
if n <= 0 || v > 0xffffffff {
|
||||
return 0, 0, ErrCorrupt
|
||||
}
|
||||
|
||||
const wordSize = 32 << (^uint(0) >> 32 & 1)
|
||||
if wordSize == 32 && v > 0x7fffffff {
|
||||
return 0, 0, ErrTooLarge
|
||||
}
|
||||
return int(v), n, nil
|
||||
}
|
||||
|
||||
const (
|
||||
decodeErrCodeCorrupt = 1
|
||||
)
|
||||
|
||||
// Decode returns the decoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire decoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
func Decode(dst, src []byte) ([]byte, error) {
|
||||
dLen, s, err := decodedLen(src)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if dLen <= cap(dst) {
|
||||
dst = dst[:dLen]
|
||||
} else {
|
||||
dst = make([]byte, dLen)
|
||||
}
|
||||
if s2Decode(dst, src[s:]) != 0 {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
// NewReader returns a new Reader that decompresses from r, using the framing
|
||||
// format described at
|
||||
// https://github.com/google/snappy/blob/master/framing_format.txt with S2 changes.
|
||||
func NewReader(r io.Reader, opts ...ReaderOption) *Reader {
|
||||
nr := Reader{
|
||||
r: r,
|
||||
maxBlock: maxBlockSize,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
if err := opt(&nr); err != nil {
|
||||
nr.err = err
|
||||
return &nr
|
||||
}
|
||||
}
|
||||
nr.maxBufSize = MaxEncodedLen(nr.maxBlock) + checksumSize
|
||||
if nr.lazyBuf > 0 {
|
||||
nr.buf = make([]byte, MaxEncodedLen(nr.lazyBuf)+checksumSize)
|
||||
} else {
|
||||
nr.buf = make([]byte, MaxEncodedLen(defaultBlockSize)+checksumSize)
|
||||
}
|
||||
nr.paramsOK = true
|
||||
return &nr
|
||||
}
|
||||
|
||||
// ReaderOption is an option for creating a decoder.
|
||||
type ReaderOption func(*Reader) error
|
||||
|
||||
// ReaderMaxBlockSize allows to control allocations if the stream
|
||||
// has been compressed with a smaller WriterBlockSize, or with the default 1MB.
|
||||
// Blocks must be this size or smaller to decompress,
|
||||
// otherwise the decoder will return ErrUnsupported.
|
||||
//
|
||||
// For streams compressed with Snappy this can safely be set to 64KB (64 << 10).
|
||||
//
|
||||
// Default is the maximum limit of 4MB.
|
||||
func ReaderMaxBlockSize(blockSize int) ReaderOption {
|
||||
return func(r *Reader) error {
|
||||
if blockSize > maxBlockSize || blockSize <= 0 {
|
||||
return errors.New("s2: block size too large. Must be <= 4MB and > 0")
|
||||
}
|
||||
if r.lazyBuf == 0 && blockSize < defaultBlockSize {
|
||||
r.lazyBuf = blockSize
|
||||
}
|
||||
r.maxBlock = blockSize
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// ReaderAllocBlock allows to control upfront stream allocations
|
||||
// and not allocate for frames bigger than this initially.
|
||||
// If frames bigger than this is seen a bigger buffer will be allocated.
|
||||
//
|
||||
// Default is 1MB, which is default output size.
|
||||
func ReaderAllocBlock(blockSize int) ReaderOption {
|
||||
return func(r *Reader) error {
|
||||
if blockSize > maxBlockSize || blockSize < 1024 {
|
||||
return errors.New("s2: invalid ReaderAllocBlock. Must be <= 4MB and >= 1024")
|
||||
}
|
||||
r.lazyBuf = blockSize
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Reader is an io.Reader that can read Snappy-compressed bytes.
|
||||
type Reader struct {
|
||||
r io.Reader
|
||||
err error
|
||||
decoded []byte
|
||||
buf []byte
|
||||
// decoded[i:j] contains decoded bytes that have not yet been passed on.
|
||||
i, j int
|
||||
// maximum block size allowed.
|
||||
maxBlock int
|
||||
// maximum expected buffer size.
|
||||
maxBufSize int
|
||||
// alloc a buffer this size if > 0.
|
||||
lazyBuf int
|
||||
readHeader bool
|
||||
paramsOK bool
|
||||
snappyFrame bool
|
||||
}
|
||||
|
||||
// ensureBufferSize will ensure that the buffer can take at least n bytes.
|
||||
// If false is returned the buffer exceeds maximum allowed size.
|
||||
func (r *Reader) ensureBufferSize(n int) bool {
|
||||
if len(r.buf) >= n {
|
||||
return true
|
||||
}
|
||||
if n > r.maxBufSize {
|
||||
r.err = ErrCorrupt
|
||||
return false
|
||||
}
|
||||
// Realloc buffer.
|
||||
r.buf = make([]byte, n)
|
||||
return true
|
||||
}
|
||||
|
||||
// Reset discards any buffered data, resets all state, and switches the Snappy
|
||||
// reader to read from r. This permits reusing a Reader rather than allocating
|
||||
// a new one.
|
||||
func (r *Reader) Reset(reader io.Reader) {
|
||||
if !r.paramsOK {
|
||||
return
|
||||
}
|
||||
r.r = reader
|
||||
r.err = nil
|
||||
r.i = 0
|
||||
r.j = 0
|
||||
r.readHeader = false
|
||||
}
|
||||
|
||||
func (r *Reader) readFull(p []byte, allowEOF bool) (ok bool) {
|
||||
if _, r.err = io.ReadFull(r.r, p); r.err != nil {
|
||||
if r.err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) {
|
||||
r.err = ErrCorrupt
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// skipN will skip n bytes.
|
||||
// If the supplied reader supports seeking that is used.
|
||||
// tmp is used as a temporary buffer for reading.
|
||||
// The supplied slice does not need to be the size of the read.
|
||||
func (r *Reader) skipN(tmp []byte, n int, allowEOF bool) (ok bool) {
|
||||
if rs, ok := r.r.(io.ReadSeeker); ok {
|
||||
_, err := rs.Seek(int64(n), io.SeekCurrent)
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
if err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) {
|
||||
r.err = ErrCorrupt
|
||||
return false
|
||||
}
|
||||
}
|
||||
for n > 0 {
|
||||
if n < len(tmp) {
|
||||
tmp = tmp[:n]
|
||||
}
|
||||
if _, r.err = io.ReadFull(r.r, tmp); r.err != nil {
|
||||
if r.err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) {
|
||||
r.err = ErrCorrupt
|
||||
}
|
||||
return false
|
||||
}
|
||||
n -= len(tmp)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Read satisfies the io.Reader interface.
|
||||
func (r *Reader) Read(p []byte) (int, error) {
|
||||
if r.err != nil {
|
||||
return 0, r.err
|
||||
}
|
||||
for {
|
||||
if r.i < r.j {
|
||||
n := copy(p, r.decoded[r.i:r.j])
|
||||
r.i += n
|
||||
return n, nil
|
||||
}
|
||||
if !r.readFull(r.buf[:4], true) {
|
||||
return 0, r.err
|
||||
}
|
||||
chunkType := r.buf[0]
|
||||
if !r.readHeader {
|
||||
if chunkType != chunkTypeStreamIdentifier {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
r.readHeader = true
|
||||
}
|
||||
chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16
|
||||
|
||||
// The chunk types are specified at
|
||||
// https://github.com/google/snappy/blob/master/framing_format.txt
|
||||
switch chunkType {
|
||||
case chunkTypeCompressedData:
|
||||
// Section 4.2. Compressed data (chunk type 0x00).
|
||||
if chunkLen < checksumSize {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
if !r.ensureBufferSize(chunkLen) {
|
||||
if r.err == nil {
|
||||
r.err = ErrUnsupported
|
||||
}
|
||||
return 0, r.err
|
||||
}
|
||||
buf := r.buf[:chunkLen]
|
||||
if !r.readFull(buf, false) {
|
||||
return 0, r.err
|
||||
}
|
||||
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
|
||||
buf = buf[checksumSize:]
|
||||
|
||||
n, err := DecodedLen(buf)
|
||||
if err != nil {
|
||||
r.err = err
|
||||
return 0, r.err
|
||||
}
|
||||
if r.snappyFrame && n > maxSnappyBlockSize {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
|
||||
if n > len(r.decoded) {
|
||||
if n > r.maxBlock {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
r.decoded = make([]byte, n)
|
||||
}
|
||||
if _, err := Decode(r.decoded, buf); err != nil {
|
||||
r.err = err
|
||||
return 0, r.err
|
||||
}
|
||||
if crc(r.decoded[:n]) != checksum {
|
||||
r.err = ErrCRC
|
||||
return 0, r.err
|
||||
}
|
||||
r.i, r.j = 0, n
|
||||
continue
|
||||
|
||||
case chunkTypeUncompressedData:
|
||||
// Section 4.3. Uncompressed data (chunk type 0x01).
|
||||
if chunkLen < checksumSize {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
if !r.ensureBufferSize(chunkLen) {
|
||||
if r.err == nil {
|
||||
r.err = ErrUnsupported
|
||||
}
|
||||
return 0, r.err
|
||||
}
|
||||
buf := r.buf[:checksumSize]
|
||||
if !r.readFull(buf, false) {
|
||||
return 0, r.err
|
||||
}
|
||||
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
|
||||
// Read directly into r.decoded instead of via r.buf.
|
||||
n := chunkLen - checksumSize
|
||||
if r.snappyFrame && n > maxSnappyBlockSize {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
if n > len(r.decoded) {
|
||||
if n > r.maxBlock {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
r.decoded = make([]byte, n)
|
||||
}
|
||||
if !r.readFull(r.decoded[:n], false) {
|
||||
return 0, r.err
|
||||
}
|
||||
if crc(r.decoded[:n]) != checksum {
|
||||
r.err = ErrCRC
|
||||
return 0, r.err
|
||||
}
|
||||
r.i, r.j = 0, n
|
||||
continue
|
||||
|
||||
case chunkTypeStreamIdentifier:
|
||||
// Section 4.1. Stream identifier (chunk type 0xff).
|
||||
if chunkLen != len(magicBody) {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
if !r.readFull(r.buf[:len(magicBody)], false) {
|
||||
return 0, r.err
|
||||
}
|
||||
if string(r.buf[:len(magicBody)]) != magicBody {
|
||||
if string(r.buf[:len(magicBody)]) != magicBodySnappy {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
} else {
|
||||
r.snappyFrame = true
|
||||
}
|
||||
} else {
|
||||
r.snappyFrame = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if chunkType <= 0x7f {
|
||||
// Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f).
|
||||
r.err = ErrUnsupported
|
||||
return 0, r.err
|
||||
}
|
||||
// Section 4.4 Padding (chunk type 0xfe).
|
||||
// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
|
||||
if chunkLen > maxBlockSize {
|
||||
r.err = ErrUnsupported
|
||||
return 0, r.err
|
||||
}
|
||||
|
||||
if !r.skipN(r.buf, chunkLen, false) {
|
||||
return 0, r.err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Skip will skip n bytes forward in the decompressed output.
|
||||
// For larger skips this consumes less CPU and is faster than reading output and discarding it.
|
||||
// CRC is not checked on skipped blocks.
|
||||
// io.ErrUnexpectedEOF is returned if the stream ends before all bytes have been skipped.
|
||||
// If a decoding error is encountered subsequent calls to Read will also fail.
|
||||
func (r *Reader) Skip(n int64) error {
|
||||
if n < 0 {
|
||||
return errors.New("attempted negative skip")
|
||||
}
|
||||
if r.err != nil {
|
||||
return r.err
|
||||
}
|
||||
|
||||
for n > 0 {
|
||||
if r.i < r.j {
|
||||
// Skip in buffer.
|
||||
// decoded[i:j] contains decoded bytes that have not yet been passed on.
|
||||
left := int64(r.j - r.i)
|
||||
if left >= n {
|
||||
r.i += int(n)
|
||||
return nil
|
||||
}
|
||||
n -= int64(r.j - r.i)
|
||||
r.i, r.j = 0, 0
|
||||
}
|
||||
|
||||
// Buffer empty; read blocks until we have content.
|
||||
if !r.readFull(r.buf[:4], true) {
|
||||
if r.err == io.EOF {
|
||||
r.err = io.ErrUnexpectedEOF
|
||||
}
|
||||
return r.err
|
||||
}
|
||||
chunkType := r.buf[0]
|
||||
if !r.readHeader {
|
||||
if chunkType != chunkTypeStreamIdentifier {
|
||||
r.err = ErrCorrupt
|
||||
return r.err
|
||||
}
|
||||
r.readHeader = true
|
||||
}
|
||||
chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16
|
||||
|
||||
// The chunk types are specified at
|
||||
// https://github.com/google/snappy/blob/master/framing_format.txt
|
||||
switch chunkType {
|
||||
case chunkTypeCompressedData:
|
||||
// Section 4.2. Compressed data (chunk type 0x00).
|
||||
if chunkLen < checksumSize {
|
||||
r.err = ErrCorrupt
|
||||
return r.err
|
||||
}
|
||||
if !r.ensureBufferSize(chunkLen) {
|
||||
if r.err == nil {
|
||||
r.err = ErrUnsupported
|
||||
}
|
||||
return r.err
|
||||
}
|
||||
buf := r.buf[:chunkLen]
|
||||
if !r.readFull(buf, false) {
|
||||
return r.err
|
||||
}
|
||||
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
|
||||
buf = buf[checksumSize:]
|
||||
|
||||
dLen, err := DecodedLen(buf)
|
||||
if err != nil {
|
||||
r.err = err
|
||||
return r.err
|
||||
}
|
||||
if dLen > r.maxBlock {
|
||||
r.err = ErrCorrupt
|
||||
return r.err
|
||||
}
|
||||
// Check if destination is within this block
|
||||
if int64(dLen) > n {
|
||||
if len(r.decoded) < dLen {
|
||||
r.decoded = make([]byte, dLen)
|
||||
}
|
||||
if _, err := Decode(r.decoded, buf); err != nil {
|
||||
r.err = err
|
||||
return r.err
|
||||
}
|
||||
if crc(r.decoded[:dLen]) != checksum {
|
||||
r.err = ErrCorrupt
|
||||
return r.err
|
||||
}
|
||||
} else {
|
||||
// Skip block completely
|
||||
n -= int64(dLen)
|
||||
dLen = 0
|
||||
}
|
||||
r.i, r.j = 0, dLen
|
||||
continue
|
||||
case chunkTypeUncompressedData:
|
||||
// Section 4.3. Uncompressed data (chunk type 0x01).
|
||||
if chunkLen < checksumSize {
|
||||
r.err = ErrCorrupt
|
||||
return r.err
|
||||
}
|
||||
if !r.ensureBufferSize(chunkLen) {
|
||||
if r.err != nil {
|
||||
r.err = ErrUnsupported
|
||||
}
|
||||
return r.err
|
||||
}
|
||||
buf := r.buf[:checksumSize]
|
||||
if !r.readFull(buf, false) {
|
||||
return r.err
|
||||
}
|
||||
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
|
||||
// Read directly into r.decoded instead of via r.buf.
|
||||
n2 := chunkLen - checksumSize
|
||||
if n2 > len(r.decoded) {
|
||||
if n2 > r.maxBlock {
|
||||
r.err = ErrCorrupt
|
||||
return r.err
|
||||
}
|
||||
r.decoded = make([]byte, n2)
|
||||
}
|
||||
if !r.readFull(r.decoded[:n2], false) {
|
||||
return r.err
|
||||
}
|
||||
if int64(n2) < n {
|
||||
if crc(r.decoded[:n2]) != checksum {
|
||||
r.err = ErrCorrupt
|
||||
return r.err
|
||||
}
|
||||
}
|
||||
r.i, r.j = 0, n2
|
||||
continue
|
||||
case chunkTypeStreamIdentifier:
|
||||
// Section 4.1. Stream identifier (chunk type 0xff).
|
||||
if chunkLen != len(magicBody) {
|
||||
r.err = ErrCorrupt
|
||||
return r.err
|
||||
}
|
||||
if !r.readFull(r.buf[:len(magicBody)], false) {
|
||||
return r.err
|
||||
}
|
||||
if string(r.buf[:len(magicBody)]) != magicBody {
|
||||
if string(r.buf[:len(magicBody)]) != magicBodySnappy {
|
||||
r.err = ErrCorrupt
|
||||
return r.err
|
||||
}
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if chunkType <= 0x7f {
|
||||
// Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f).
|
||||
r.err = ErrUnsupported
|
||||
return r.err
|
||||
}
|
||||
if chunkLen > maxBlockSize {
|
||||
r.err = ErrUnsupported
|
||||
return r.err
|
||||
}
|
||||
// Section 4.4 Padding (chunk type 0xfe).
|
||||
// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
|
||||
if !r.skipN(r.buf, chunkLen, false) {
|
||||
return r.err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReadByte satisfies the io.ByteReader interface.
|
||||
func (r *Reader) ReadByte() (byte, error) {
|
||||
if r.err != nil {
|
||||
return 0, r.err
|
||||
}
|
||||
if r.i < r.j {
|
||||
c := r.decoded[r.i]
|
||||
r.i++
|
||||
return c, nil
|
||||
}
|
||||
var tmp [1]byte
|
||||
for i := 0; i < 10; i++ {
|
||||
n, err := r.Read(tmp[:])
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if n == 1 {
|
||||
return tmp[0], nil
|
||||
}
|
||||
}
|
||||
return 0, io.ErrNoProgress
|
||||
}
|
||||
568
vendor/github.com/klauspost/compress/s2/decode_amd64.s
сгенерированный
поставляемый
Обычный файл
568
vendor/github.com/klauspost/compress/s2/decode_amd64.s
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,568 @@
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build !appengine
|
||||
// +build gc
|
||||
// +build !noasm
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
#define R_TMP0 AX
|
||||
#define R_TMP1 BX
|
||||
#define R_LEN CX
|
||||
#define R_OFF DX
|
||||
#define R_SRC SI
|
||||
#define R_DST DI
|
||||
#define R_DBASE R8
|
||||
#define R_DLEN R9
|
||||
#define R_DEND R10
|
||||
#define R_SBASE R11
|
||||
#define R_SLEN R12
|
||||
#define R_SEND R13
|
||||
#define R_TMP2 R14
|
||||
#define R_TMP3 R15
|
||||
|
||||
// The asm code generally follows the pure Go code in decode_other.go, except
|
||||
// where marked with a "!!!".
|
||||
|
||||
// func decode(dst, src []byte) int
|
||||
//
|
||||
// All local variables fit into registers. The non-zero stack size is only to
|
||||
// spill registers and push args when issuing a CALL. The register allocation:
|
||||
// - R_TMP0 scratch
|
||||
// - R_TMP1 scratch
|
||||
// - R_LEN length or x (shared)
|
||||
// - R_OFF offset
|
||||
// - R_SRC &src[s]
|
||||
// - R_DST &dst[d]
|
||||
// + R_DBASE dst_base
|
||||
// + R_DLEN dst_len
|
||||
// + R_DEND dst_base + dst_len
|
||||
// + R_SBASE src_base
|
||||
// + R_SLEN src_len
|
||||
// + R_SEND src_base + src_len
|
||||
// - R_TMP2 used by doCopy
|
||||
// - R_TMP3 used by doCopy
|
||||
//
|
||||
// The registers R_DBASE-R_SEND (marked with a "+") are set at the start of the
|
||||
// function, and after a CALL returns, and are not otherwise modified.
|
||||
//
|
||||
// The d variable is implicitly R_DST - R_DBASE, and len(dst)-d is R_DEND - R_DST.
|
||||
// The s variable is implicitly R_SRC - R_SBASE, and len(src)-s is R_SEND - R_SRC.
|
||||
TEXT ·s2Decode(SB), NOSPLIT, $48-56
|
||||
// Initialize R_SRC, R_DST and R_DBASE-R_SEND.
|
||||
MOVQ dst_base+0(FP), R_DBASE
|
||||
MOVQ dst_len+8(FP), R_DLEN
|
||||
MOVQ R_DBASE, R_DST
|
||||
MOVQ R_DBASE, R_DEND
|
||||
ADDQ R_DLEN, R_DEND
|
||||
MOVQ src_base+24(FP), R_SBASE
|
||||
MOVQ src_len+32(FP), R_SLEN
|
||||
MOVQ R_SBASE, R_SRC
|
||||
MOVQ R_SBASE, R_SEND
|
||||
ADDQ R_SLEN, R_SEND
|
||||
XORQ R_OFF, R_OFF
|
||||
|
||||
loop:
|
||||
// for s < len(src)
|
||||
CMPQ R_SRC, R_SEND
|
||||
JEQ end
|
||||
|
||||
// R_LEN = uint32(src[s])
|
||||
//
|
||||
// switch src[s] & 0x03
|
||||
MOVBLZX (R_SRC), R_LEN
|
||||
MOVL R_LEN, R_TMP1
|
||||
ANDL $3, R_TMP1
|
||||
CMPL R_TMP1, $1
|
||||
JAE tagCopy
|
||||
|
||||
// ----------------------------------------
|
||||
// The code below handles literal tags.
|
||||
|
||||
// case tagLiteral:
|
||||
// x := uint32(src[s] >> 2)
|
||||
// switch
|
||||
SHRL $2, R_LEN
|
||||
CMPL R_LEN, $60
|
||||
JAE tagLit60Plus
|
||||
|
||||
// case x < 60:
|
||||
// s++
|
||||
INCQ R_SRC
|
||||
|
||||
doLit:
|
||||
// This is the end of the inner "switch", when we have a literal tag.
|
||||
//
|
||||
// We assume that R_LEN == x and x fits in a uint32, where x is the variable
|
||||
// used in the pure Go decode_other.go code.
|
||||
|
||||
// length = int(x) + 1
|
||||
//
|
||||
// Unlike the pure Go code, we don't need to check if length <= 0 because
|
||||
// R_LEN can hold 64 bits, so the increment cannot overflow.
|
||||
INCQ R_LEN
|
||||
|
||||
// Prepare to check if copying length bytes will run past the end of dst or
|
||||
// src.
|
||||
//
|
||||
// R_TMP0 = len(dst) - d
|
||||
// R_TMP1 = len(src) - s
|
||||
MOVQ R_DEND, R_TMP0
|
||||
SUBQ R_DST, R_TMP0
|
||||
MOVQ R_SEND, R_TMP1
|
||||
SUBQ R_SRC, R_TMP1
|
||||
|
||||
// !!! Try a faster technique for short (16 or fewer bytes) copies.
|
||||
//
|
||||
// if length > 16 || len(dst)-d < 16 || len(src)-s < 16 {
|
||||
// goto callMemmove // Fall back on calling runtime·memmove.
|
||||
// }
|
||||
//
|
||||
// The C++ snappy code calls this TryFastAppend. It also checks len(src)-s
|
||||
// against 21 instead of 16, because it cannot assume that all of its input
|
||||
// is contiguous in memory and so it needs to leave enough source bytes to
|
||||
// read the next tag without refilling buffers, but Go's Decode assumes
|
||||
// contiguousness (the src argument is a []byte).
|
||||
CMPQ R_LEN, $16
|
||||
JGT callMemmove
|
||||
CMPQ R_TMP0, $16
|
||||
JLT callMemmove
|
||||
CMPQ R_TMP1, $16
|
||||
JLT callMemmove
|
||||
|
||||
// !!! Implement the copy from src to dst as a 16-byte load and store.
|
||||
// (Decode's documentation says that dst and src must not overlap.)
|
||||
//
|
||||
// This always copies 16 bytes, instead of only length bytes, but that's
|
||||
// OK. If the input is a valid Snappy encoding then subsequent iterations
|
||||
// will fix up the overrun. Otherwise, Decode returns a nil []byte (and a
|
||||
// non-nil error), so the overrun will be ignored.
|
||||
//
|
||||
// Note that on amd64, it is legal and cheap to issue unaligned 8-byte or
|
||||
// 16-byte loads and stores. This technique probably wouldn't be as
|
||||
// effective on architectures that are fussier about alignment.
|
||||
MOVOU 0(R_SRC), X0
|
||||
MOVOU X0, 0(R_DST)
|
||||
|
||||
// d += length
|
||||
// s += length
|
||||
ADDQ R_LEN, R_DST
|
||||
ADDQ R_LEN, R_SRC
|
||||
JMP loop
|
||||
|
||||
callMemmove:
|
||||
// if length > len(dst)-d || length > len(src)-s { etc }
|
||||
CMPQ R_LEN, R_TMP0
|
||||
JGT errCorrupt
|
||||
CMPQ R_LEN, R_TMP1
|
||||
JGT errCorrupt
|
||||
|
||||
// copy(dst[d:], src[s:s+length])
|
||||
//
|
||||
// This means calling runtime·memmove(&dst[d], &src[s], length), so we push
|
||||
// R_DST, R_SRC and R_LEN as arguments. Coincidentally, we also need to spill those
|
||||
// three registers to the stack, to save local variables across the CALL.
|
||||
MOVQ R_DST, 0(SP)
|
||||
MOVQ R_SRC, 8(SP)
|
||||
MOVQ R_LEN, 16(SP)
|
||||
MOVQ R_DST, 24(SP)
|
||||
MOVQ R_SRC, 32(SP)
|
||||
MOVQ R_LEN, 40(SP)
|
||||
MOVQ R_OFF, 48(SP)
|
||||
CALL runtime·memmove(SB)
|
||||
|
||||
// Restore local variables: unspill registers from the stack and
|
||||
// re-calculate R_DBASE-R_SEND.
|
||||
MOVQ 24(SP), R_DST
|
||||
MOVQ 32(SP), R_SRC
|
||||
MOVQ 40(SP), R_LEN
|
||||
MOVQ 48(SP), R_OFF
|
||||
MOVQ dst_base+0(FP), R_DBASE
|
||||
MOVQ dst_len+8(FP), R_DLEN
|
||||
MOVQ R_DBASE, R_DEND
|
||||
ADDQ R_DLEN, R_DEND
|
||||
MOVQ src_base+24(FP), R_SBASE
|
||||
MOVQ src_len+32(FP), R_SLEN
|
||||
MOVQ R_SBASE, R_SEND
|
||||
ADDQ R_SLEN, R_SEND
|
||||
|
||||
// d += length
|
||||
// s += length
|
||||
ADDQ R_LEN, R_DST
|
||||
ADDQ R_LEN, R_SRC
|
||||
JMP loop
|
||||
|
||||
tagLit60Plus:
|
||||
// !!! This fragment does the
|
||||
//
|
||||
// s += x - 58; if uint(s) > uint(len(src)) { etc }
|
||||
//
|
||||
// checks. In the asm version, we code it once instead of once per switch case.
|
||||
ADDQ R_LEN, R_SRC
|
||||
SUBQ $58, R_SRC
|
||||
CMPQ R_SRC, R_SEND
|
||||
JA errCorrupt
|
||||
|
||||
// case x == 60:
|
||||
CMPL R_LEN, $61
|
||||
JEQ tagLit61
|
||||
JA tagLit62Plus
|
||||
|
||||
// x = uint32(src[s-1])
|
||||
MOVBLZX -1(R_SRC), R_LEN
|
||||
JMP doLit
|
||||
|
||||
tagLit61:
|
||||
// case x == 61:
|
||||
// x = uint32(src[s-2]) | uint32(src[s-1])<<8
|
||||
MOVWLZX -2(R_SRC), R_LEN
|
||||
JMP doLit
|
||||
|
||||
tagLit62Plus:
|
||||
CMPL R_LEN, $62
|
||||
JA tagLit63
|
||||
|
||||
// case x == 62:
|
||||
// x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
|
||||
// We read one byte, safe to read one back, since we are just reading tag.
|
||||
// x = binary.LittleEndian.Uint32(src[s-1:]) >> 8
|
||||
MOVL -4(R_SRC), R_LEN
|
||||
SHRL $8, R_LEN
|
||||
JMP doLit
|
||||
|
||||
tagLit63:
|
||||
// case x == 63:
|
||||
// x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
|
||||
MOVL -4(R_SRC), R_LEN
|
||||
JMP doLit
|
||||
|
||||
// The code above handles literal tags.
|
||||
// ----------------------------------------
|
||||
// The code below handles copy tags.
|
||||
|
||||
tagCopy4:
|
||||
// case tagCopy4:
|
||||
// s += 5
|
||||
ADDQ $5, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
CMPQ R_SRC, R_SEND
|
||||
JA errCorrupt
|
||||
|
||||
// length = 1 + int(src[s-5])>>2
|
||||
SHRQ $2, R_LEN
|
||||
INCQ R_LEN
|
||||
|
||||
// offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
|
||||
MOVLQZX -4(R_SRC), R_OFF
|
||||
JMP doCopy
|
||||
|
||||
tagCopy2:
|
||||
// case tagCopy2:
|
||||
// s += 3
|
||||
ADDQ $3, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
CMPQ R_SRC, R_SEND
|
||||
JA errCorrupt
|
||||
|
||||
// length = 1 + int(src[s-3])>>2
|
||||
SHRQ $2, R_LEN
|
||||
INCQ R_LEN
|
||||
|
||||
// offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
|
||||
MOVWQZX -2(R_SRC), R_OFF
|
||||
JMP doCopy
|
||||
|
||||
tagCopy:
|
||||
// We have a copy tag. We assume that:
|
||||
// - R_TMP1 == src[s] & 0x03
|
||||
// - R_LEN == src[s]
|
||||
CMPQ R_TMP1, $2
|
||||
JEQ tagCopy2
|
||||
JA tagCopy4
|
||||
|
||||
// case tagCopy1:
|
||||
// s += 2
|
||||
ADDQ $2, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
CMPQ R_SRC, R_SEND
|
||||
JA errCorrupt
|
||||
|
||||
// offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
|
||||
// length = 4 + int(src[s-2])>>2&0x7
|
||||
MOVBQZX -1(R_SRC), R_TMP1
|
||||
MOVQ R_LEN, R_TMP0
|
||||
SHRQ $2, R_LEN
|
||||
ANDQ $0xe0, R_TMP0
|
||||
ANDQ $7, R_LEN
|
||||
SHLQ $3, R_TMP0
|
||||
ADDQ $4, R_LEN
|
||||
ORQ R_TMP1, R_TMP0
|
||||
|
||||
// check if repeat code, ZF set by ORQ.
|
||||
JZ repeatCode
|
||||
|
||||
// This is a regular copy, transfer our temporary value to R_OFF (length)
|
||||
MOVQ R_TMP0, R_OFF
|
||||
JMP doCopy
|
||||
|
||||
// This is a repeat code.
|
||||
repeatCode:
|
||||
// If length < 9, reuse last offset, with the length already calculated.
|
||||
CMPQ R_LEN, $9
|
||||
JL doCopyRepeat
|
||||
|
||||
// Read additional bytes for length.
|
||||
JE repeatLen1
|
||||
|
||||
// Rare, so the extra branch shouldn't hurt too much.
|
||||
CMPQ R_LEN, $10
|
||||
JE repeatLen2
|
||||
JMP repeatLen3
|
||||
|
||||
// Read repeat lengths.
|
||||
repeatLen1:
|
||||
// s ++
|
||||
ADDQ $1, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
CMPQ R_SRC, R_SEND
|
||||
JA errCorrupt
|
||||
|
||||
// length = src[s-1] + 8
|
||||
MOVBQZX -1(R_SRC), R_LEN
|
||||
ADDL $8, R_LEN
|
||||
JMP doCopyRepeat
|
||||
|
||||
repeatLen2:
|
||||
// s +=2
|
||||
ADDQ $2, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
CMPQ R_SRC, R_SEND
|
||||
JA errCorrupt
|
||||
|
||||
// length = uint32(src[s-2]) | (uint32(src[s-1])<<8) + (1 << 8)
|
||||
MOVWQZX -2(R_SRC), R_LEN
|
||||
ADDL $260, R_LEN
|
||||
JMP doCopyRepeat
|
||||
|
||||
repeatLen3:
|
||||
// s +=3
|
||||
ADDQ $3, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
CMPQ R_SRC, R_SEND
|
||||
JA errCorrupt
|
||||
|
||||
// length = uint32(src[s-3]) | (uint32(src[s-2])<<8) | (uint32(src[s-1])<<16) + (1 << 16)
|
||||
// Read one byte further back (just part of the tag, shifted out)
|
||||
MOVL -4(R_SRC), R_LEN
|
||||
SHRL $8, R_LEN
|
||||
ADDL $65540, R_LEN
|
||||
JMP doCopyRepeat
|
||||
|
||||
doCopy:
|
||||
// This is the end of the outer "switch", when we have a copy tag.
|
||||
//
|
||||
// We assume that:
|
||||
// - R_LEN == length && R_LEN > 0
|
||||
// - R_OFF == offset
|
||||
|
||||
// if d < offset { etc }
|
||||
MOVQ R_DST, R_TMP1
|
||||
SUBQ R_DBASE, R_TMP1
|
||||
CMPQ R_TMP1, R_OFF
|
||||
JLT errCorrupt
|
||||
|
||||
// Repeat values can skip the test above, since any offset > 0 will be in dst.
|
||||
doCopyRepeat:
|
||||
// if offset <= 0 { etc }
|
||||
CMPQ R_OFF, $0
|
||||
JLE errCorrupt
|
||||
|
||||
// if length > len(dst)-d { etc }
|
||||
MOVQ R_DEND, R_TMP1
|
||||
SUBQ R_DST, R_TMP1
|
||||
CMPQ R_LEN, R_TMP1
|
||||
JGT errCorrupt
|
||||
|
||||
// forwardCopy(dst[d:d+length], dst[d-offset:]); d += length
|
||||
//
|
||||
// Set:
|
||||
// - R_TMP2 = len(dst)-d
|
||||
// - R_TMP3 = &dst[d-offset]
|
||||
MOVQ R_DEND, R_TMP2
|
||||
SUBQ R_DST, R_TMP2
|
||||
MOVQ R_DST, R_TMP3
|
||||
SUBQ R_OFF, R_TMP3
|
||||
|
||||
// !!! Try a faster technique for short (16 or fewer bytes) forward copies.
|
||||
//
|
||||
// First, try using two 8-byte load/stores, similar to the doLit technique
|
||||
// above. Even if dst[d:d+length] and dst[d-offset:] can overlap, this is
|
||||
// still OK if offset >= 8. Note that this has to be two 8-byte load/stores
|
||||
// and not one 16-byte load/store, and the first store has to be before the
|
||||
// second load, due to the overlap if offset is in the range [8, 16).
|
||||
//
|
||||
// if length > 16 || offset < 8 || len(dst)-d < 16 {
|
||||
// goto slowForwardCopy
|
||||
// }
|
||||
// copy 16 bytes
|
||||
// d += length
|
||||
CMPQ R_LEN, $16
|
||||
JGT slowForwardCopy
|
||||
CMPQ R_OFF, $8
|
||||
JLT slowForwardCopy
|
||||
CMPQ R_TMP2, $16
|
||||
JLT slowForwardCopy
|
||||
MOVQ 0(R_TMP3), R_TMP0
|
||||
MOVQ R_TMP0, 0(R_DST)
|
||||
MOVQ 8(R_TMP3), R_TMP1
|
||||
MOVQ R_TMP1, 8(R_DST)
|
||||
ADDQ R_LEN, R_DST
|
||||
JMP loop
|
||||
|
||||
slowForwardCopy:
|
||||
// !!! If the forward copy is longer than 16 bytes, or if offset < 8, we
|
||||
// can still try 8-byte load stores, provided we can overrun up to 10 extra
|
||||
// bytes. As above, the overrun will be fixed up by subsequent iterations
|
||||
// of the outermost loop.
|
||||
//
|
||||
// The C++ snappy code calls this technique IncrementalCopyFastPath. Its
|
||||
// commentary says:
|
||||
//
|
||||
// ----
|
||||
//
|
||||
// The main part of this loop is a simple copy of eight bytes at a time
|
||||
// until we've copied (at least) the requested amount of bytes. However,
|
||||
// if d and d-offset are less than eight bytes apart (indicating a
|
||||
// repeating pattern of length < 8), we first need to expand the pattern in
|
||||
// order to get the correct results. For instance, if the buffer looks like
|
||||
// this, with the eight-byte <d-offset> and <d> patterns marked as
|
||||
// intervals:
|
||||
//
|
||||
// abxxxxxxxxxxxx
|
||||
// [------] d-offset
|
||||
// [------] d
|
||||
//
|
||||
// a single eight-byte copy from <d-offset> to <d> will repeat the pattern
|
||||
// once, after which we can move <d> two bytes without moving <d-offset>:
|
||||
//
|
||||
// ababxxxxxxxxxx
|
||||
// [------] d-offset
|
||||
// [------] d
|
||||
//
|
||||
// and repeat the exercise until the two no longer overlap.
|
||||
//
|
||||
// This allows us to do very well in the special case of one single byte
|
||||
// repeated many times, without taking a big hit for more general cases.
|
||||
//
|
||||
// The worst case of extra writing past the end of the match occurs when
|
||||
// offset == 1 and length == 1; the last copy will read from byte positions
|
||||
// [0..7] and write to [4..11], whereas it was only supposed to write to
|
||||
// position 1. Thus, ten excess bytes.
|
||||
//
|
||||
// ----
|
||||
//
|
||||
// That "10 byte overrun" worst case is confirmed by Go's
|
||||
// TestSlowForwardCopyOverrun, which also tests the fixUpSlowForwardCopy
|
||||
// and finishSlowForwardCopy algorithm.
|
||||
//
|
||||
// if length > len(dst)-d-10 {
|
||||
// goto verySlowForwardCopy
|
||||
// }
|
||||
SUBQ $10, R_TMP2
|
||||
CMPQ R_LEN, R_TMP2
|
||||
JGT verySlowForwardCopy
|
||||
|
||||
// We want to keep the offset, so we use R_TMP2 from here.
|
||||
MOVQ R_OFF, R_TMP2
|
||||
|
||||
makeOffsetAtLeast8:
|
||||
// !!! As above, expand the pattern so that offset >= 8 and we can use
|
||||
// 8-byte load/stores.
|
||||
//
|
||||
// for offset < 8 {
|
||||
// copy 8 bytes from dst[d-offset:] to dst[d:]
|
||||
// length -= offset
|
||||
// d += offset
|
||||
// offset += offset
|
||||
// // The two previous lines together means that d-offset, and therefore
|
||||
// // R_TMP3, is unchanged.
|
||||
// }
|
||||
CMPQ R_TMP2, $8
|
||||
JGE fixUpSlowForwardCopy
|
||||
MOVQ (R_TMP3), R_TMP1
|
||||
MOVQ R_TMP1, (R_DST)
|
||||
SUBQ R_TMP2, R_LEN
|
||||
ADDQ R_TMP2, R_DST
|
||||
ADDQ R_TMP2, R_TMP2
|
||||
JMP makeOffsetAtLeast8
|
||||
|
||||
fixUpSlowForwardCopy:
|
||||
// !!! Add length (which might be negative now) to d (implied by R_DST being
|
||||
// &dst[d]) so that d ends up at the right place when we jump back to the
|
||||
// top of the loop. Before we do that, though, we save R_DST to R_TMP0 so that, if
|
||||
// length is positive, copying the remaining length bytes will write to the
|
||||
// right place.
|
||||
MOVQ R_DST, R_TMP0
|
||||
ADDQ R_LEN, R_DST
|
||||
|
||||
finishSlowForwardCopy:
|
||||
// !!! Repeat 8-byte load/stores until length <= 0. Ending with a negative
|
||||
// length means that we overrun, but as above, that will be fixed up by
|
||||
// subsequent iterations of the outermost loop.
|
||||
CMPQ R_LEN, $0
|
||||
JLE loop
|
||||
MOVQ (R_TMP3), R_TMP1
|
||||
MOVQ R_TMP1, (R_TMP0)
|
||||
ADDQ $8, R_TMP3
|
||||
ADDQ $8, R_TMP0
|
||||
SUBQ $8, R_LEN
|
||||
JMP finishSlowForwardCopy
|
||||
|
||||
verySlowForwardCopy:
|
||||
// verySlowForwardCopy is a simple implementation of forward copy. In C
|
||||
// parlance, this is a do/while loop instead of a while loop, since we know
|
||||
// that length > 0. In Go syntax:
|
||||
//
|
||||
// for {
|
||||
// dst[d] = dst[d - offset]
|
||||
// d++
|
||||
// length--
|
||||
// if length == 0 {
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
MOVB (R_TMP3), R_TMP1
|
||||
MOVB R_TMP1, (R_DST)
|
||||
INCQ R_TMP3
|
||||
INCQ R_DST
|
||||
DECQ R_LEN
|
||||
JNZ verySlowForwardCopy
|
||||
JMP loop
|
||||
|
||||
// The code above handles copy tags.
|
||||
// ----------------------------------------
|
||||
|
||||
end:
|
||||
// This is the end of the "for s < len(src)".
|
||||
//
|
||||
// if d != len(dst) { etc }
|
||||
CMPQ R_DST, R_DEND
|
||||
JNE errCorrupt
|
||||
|
||||
// return 0
|
||||
MOVQ $0, ret+48(FP)
|
||||
RET
|
||||
|
||||
errCorrupt:
|
||||
// return decodeErrCodeCorrupt
|
||||
MOVQ $1, ret+48(FP)
|
||||
RET
|
||||
574
vendor/github.com/klauspost/compress/s2/decode_arm64.s
сгенерированный
поставляемый
Обычный файл
574
vendor/github.com/klauspost/compress/s2/decode_arm64.s
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,574 @@
|
||||
// Copyright 2020 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build !appengine
|
||||
// +build gc
|
||||
// +build !noasm
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
#define R_TMP0 R2
|
||||
#define R_TMP1 R3
|
||||
#define R_LEN R4
|
||||
#define R_OFF R5
|
||||
#define R_SRC R6
|
||||
#define R_DST R7
|
||||
#define R_DBASE R8
|
||||
#define R_DLEN R9
|
||||
#define R_DEND R10
|
||||
#define R_SBASE R11
|
||||
#define R_SLEN R12
|
||||
#define R_SEND R13
|
||||
#define R_TMP2 R14
|
||||
#define R_TMP3 R15
|
||||
|
||||
// TEST_SRC will check if R_SRC is <= SRC_END
|
||||
#define TEST_SRC() \
|
||||
CMP R_SEND, R_SRC \
|
||||
BGT errCorrupt
|
||||
|
||||
// MOVD R_SRC, R_TMP1
|
||||
// SUB R_SBASE, R_TMP1, R_TMP1
|
||||
// CMP R_SLEN, R_TMP1
|
||||
// BGT errCorrupt
|
||||
|
||||
// The asm code generally follows the pure Go code in decode_other.go, except
|
||||
// where marked with a "!!!".
|
||||
|
||||
// func decode(dst, src []byte) int
|
||||
//
|
||||
// All local variables fit into registers. The non-zero stack size is only to
|
||||
// spill registers and push args when issuing a CALL. The register allocation:
|
||||
// - R_TMP0 scratch
|
||||
// - R_TMP1 scratch
|
||||
// - R_LEN length or x
|
||||
// - R_OFF offset
|
||||
// - R_SRC &src[s]
|
||||
// - R_DST &dst[d]
|
||||
// + R_DBASE dst_base
|
||||
// + R_DLEN dst_len
|
||||
// + R_DEND dst_base + dst_len
|
||||
// + R_SBASE src_base
|
||||
// + R_SLEN src_len
|
||||
// + R_SEND src_base + src_len
|
||||
// - R_TMP2 used by doCopy
|
||||
// - R_TMP3 used by doCopy
|
||||
//
|
||||
// The registers R_DBASE-R_SEND (marked with a "+") are set at the start of the
|
||||
// function, and after a CALL returns, and are not otherwise modified.
|
||||
//
|
||||
// The d variable is implicitly R_DST - R_DBASE, and len(dst)-d is R_DEND - R_DST.
|
||||
// The s variable is implicitly R_SRC - R_SBASE, and len(src)-s is R_SEND - R_SRC.
|
||||
TEXT ·s2Decode(SB), NOSPLIT, $56-64
|
||||
// Initialize R_SRC, R_DST and R_DBASE-R_SEND.
|
||||
MOVD dst_base+0(FP), R_DBASE
|
||||
MOVD dst_len+8(FP), R_DLEN
|
||||
MOVD R_DBASE, R_DST
|
||||
MOVD R_DBASE, R_DEND
|
||||
ADD R_DLEN, R_DEND, R_DEND
|
||||
MOVD src_base+24(FP), R_SBASE
|
||||
MOVD src_len+32(FP), R_SLEN
|
||||
MOVD R_SBASE, R_SRC
|
||||
MOVD R_SBASE, R_SEND
|
||||
ADD R_SLEN, R_SEND, R_SEND
|
||||
MOVD $0, R_OFF
|
||||
|
||||
loop:
|
||||
// for s < len(src)
|
||||
CMP R_SEND, R_SRC
|
||||
BEQ end
|
||||
|
||||
// R_LEN = uint32(src[s])
|
||||
//
|
||||
// switch src[s] & 0x03
|
||||
MOVBU (R_SRC), R_LEN
|
||||
MOVW R_LEN, R_TMP1
|
||||
ANDW $3, R_TMP1
|
||||
MOVW $1, R1
|
||||
CMPW R1, R_TMP1
|
||||
BGE tagCopy
|
||||
|
||||
// ----------------------------------------
|
||||
// The code below handles literal tags.
|
||||
|
||||
// case tagLiteral:
|
||||
// x := uint32(src[s] >> 2)
|
||||
// switch
|
||||
MOVW $60, R1
|
||||
LSRW $2, R_LEN, R_LEN
|
||||
CMPW R_LEN, R1
|
||||
BLS tagLit60Plus
|
||||
|
||||
// case x < 60:
|
||||
// s++
|
||||
ADD $1, R_SRC, R_SRC
|
||||
|
||||
doLit:
|
||||
// This is the end of the inner "switch", when we have a literal tag.
|
||||
//
|
||||
// We assume that R_LEN == x and x fits in a uint32, where x is the variable
|
||||
// used in the pure Go decode_other.go code.
|
||||
|
||||
// length = int(x) + 1
|
||||
//
|
||||
// Unlike the pure Go code, we don't need to check if length <= 0 because
|
||||
// R_LEN can hold 64 bits, so the increment cannot overflow.
|
||||
ADD $1, R_LEN, R_LEN
|
||||
|
||||
// Prepare to check if copying length bytes will run past the end of dst or
|
||||
// src.
|
||||
//
|
||||
// R_TMP0 = len(dst) - d
|
||||
// R_TMP1 = len(src) - s
|
||||
MOVD R_DEND, R_TMP0
|
||||
SUB R_DST, R_TMP0, R_TMP0
|
||||
MOVD R_SEND, R_TMP1
|
||||
SUB R_SRC, R_TMP1, R_TMP1
|
||||
|
||||
// !!! Try a faster technique for short (16 or fewer bytes) copies.
|
||||
//
|
||||
// if length > 16 || len(dst)-d < 16 || len(src)-s < 16 {
|
||||
// goto callMemmove // Fall back on calling runtime·memmove.
|
||||
// }
|
||||
//
|
||||
// The C++ snappy code calls this TryFastAppend. It also checks len(src)-s
|
||||
// against 21 instead of 16, because it cannot assume that all of its input
|
||||
// is contiguous in memory and so it needs to leave enough source bytes to
|
||||
// read the next tag without refilling buffers, but Go's Decode assumes
|
||||
// contiguousness (the src argument is a []byte).
|
||||
CMP $16, R_LEN
|
||||
BGT callMemmove
|
||||
CMP $16, R_TMP0
|
||||
BLT callMemmove
|
||||
CMP $16, R_TMP1
|
||||
BLT callMemmove
|
||||
|
||||
// !!! Implement the copy from src to dst as a 16-byte load and store.
|
||||
// (Decode's documentation says that dst and src must not overlap.)
|
||||
//
|
||||
// This always copies 16 bytes, instead of only length bytes, but that's
|
||||
// OK. If the input is a valid Snappy encoding then subsequent iterations
|
||||
// will fix up the overrun. Otherwise, Decode returns a nil []byte (and a
|
||||
// non-nil error), so the overrun will be ignored.
|
||||
//
|
||||
// Note that on arm64, it is legal and cheap to issue unaligned 8-byte or
|
||||
// 16-byte loads and stores. This technique probably wouldn't be as
|
||||
// effective on architectures that are fussier about alignment.
|
||||
LDP 0(R_SRC), (R_TMP2, R_TMP3)
|
||||
STP (R_TMP2, R_TMP3), 0(R_DST)
|
||||
|
||||
// d += length
|
||||
// s += length
|
||||
ADD R_LEN, R_DST, R_DST
|
||||
ADD R_LEN, R_SRC, R_SRC
|
||||
B loop
|
||||
|
||||
callMemmove:
|
||||
// if length > len(dst)-d || length > len(src)-s { etc }
|
||||
CMP R_TMP0, R_LEN
|
||||
BGT errCorrupt
|
||||
CMP R_TMP1, R_LEN
|
||||
BGT errCorrupt
|
||||
|
||||
// copy(dst[d:], src[s:s+length])
|
||||
//
|
||||
// This means calling runtime·memmove(&dst[d], &src[s], length), so we push
|
||||
// R_DST, R_SRC and R_LEN as arguments. Coincidentally, we also need to spill those
|
||||
// three registers to the stack, to save local variables across the CALL.
|
||||
MOVD R_DST, 8(RSP)
|
||||
MOVD R_SRC, 16(RSP)
|
||||
MOVD R_LEN, 24(RSP)
|
||||
MOVD R_DST, 32(RSP)
|
||||
MOVD R_SRC, 40(RSP)
|
||||
MOVD R_LEN, 48(RSP)
|
||||
MOVD R_OFF, 56(RSP)
|
||||
CALL runtime·memmove(SB)
|
||||
|
||||
// Restore local variables: unspill registers from the stack and
|
||||
// re-calculate R_DBASE-R_SEND.
|
||||
MOVD 32(RSP), R_DST
|
||||
MOVD 40(RSP), R_SRC
|
||||
MOVD 48(RSP), R_LEN
|
||||
MOVD 56(RSP), R_OFF
|
||||
MOVD dst_base+0(FP), R_DBASE
|
||||
MOVD dst_len+8(FP), R_DLEN
|
||||
MOVD R_DBASE, R_DEND
|
||||
ADD R_DLEN, R_DEND, R_DEND
|
||||
MOVD src_base+24(FP), R_SBASE
|
||||
MOVD src_len+32(FP), R_SLEN
|
||||
MOVD R_SBASE, R_SEND
|
||||
ADD R_SLEN, R_SEND, R_SEND
|
||||
|
||||
// d += length
|
||||
// s += length
|
||||
ADD R_LEN, R_DST, R_DST
|
||||
ADD R_LEN, R_SRC, R_SRC
|
||||
B loop
|
||||
|
||||
tagLit60Plus:
|
||||
// !!! This fragment does the
|
||||
//
|
||||
// s += x - 58; if uint(s) > uint(len(src)) { etc }
|
||||
//
|
||||
// checks. In the asm version, we code it once instead of once per switch case.
|
||||
ADD R_LEN, R_SRC, R_SRC
|
||||
SUB $58, R_SRC, R_SRC
|
||||
TEST_SRC()
|
||||
|
||||
// case x == 60:
|
||||
MOVW $61, R1
|
||||
CMPW R1, R_LEN
|
||||
BEQ tagLit61
|
||||
BGT tagLit62Plus
|
||||
|
||||
// x = uint32(src[s-1])
|
||||
MOVBU -1(R_SRC), R_LEN
|
||||
B doLit
|
||||
|
||||
tagLit61:
|
||||
// case x == 61:
|
||||
// x = uint32(src[s-2]) | uint32(src[s-1])<<8
|
||||
MOVHU -2(R_SRC), R_LEN
|
||||
B doLit
|
||||
|
||||
tagLit62Plus:
|
||||
CMPW $62, R_LEN
|
||||
BHI tagLit63
|
||||
|
||||
// case x == 62:
|
||||
// x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
|
||||
MOVHU -3(R_SRC), R_LEN
|
||||
MOVBU -1(R_SRC), R_TMP1
|
||||
ORR R_TMP1<<16, R_LEN
|
||||
B doLit
|
||||
|
||||
tagLit63:
|
||||
// case x == 63:
|
||||
// x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
|
||||
MOVWU -4(R_SRC), R_LEN
|
||||
B doLit
|
||||
|
||||
// The code above handles literal tags.
|
||||
// ----------------------------------------
|
||||
// The code below handles copy tags.
|
||||
|
||||
tagCopy4:
|
||||
// case tagCopy4:
|
||||
// s += 5
|
||||
ADD $5, R_SRC, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
MOVD R_SRC, R_TMP1
|
||||
SUB R_SBASE, R_TMP1, R_TMP1
|
||||
CMP R_SLEN, R_TMP1
|
||||
BGT errCorrupt
|
||||
|
||||
// length = 1 + int(src[s-5])>>2
|
||||
MOVD $1, R1
|
||||
ADD R_LEN>>2, R1, R_LEN
|
||||
|
||||
// offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
|
||||
MOVWU -4(R_SRC), R_OFF
|
||||
B doCopy
|
||||
|
||||
tagCopy2:
|
||||
// case tagCopy2:
|
||||
// s += 3
|
||||
ADD $3, R_SRC, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
TEST_SRC()
|
||||
|
||||
// length = 1 + int(src[s-3])>>2
|
||||
MOVD $1, R1
|
||||
ADD R_LEN>>2, R1, R_LEN
|
||||
|
||||
// offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
|
||||
MOVHU -2(R_SRC), R_OFF
|
||||
B doCopy
|
||||
|
||||
tagCopy:
|
||||
// We have a copy tag. We assume that:
|
||||
// - R_TMP1 == src[s] & 0x03
|
||||
// - R_LEN == src[s]
|
||||
CMP $2, R_TMP1
|
||||
BEQ tagCopy2
|
||||
BGT tagCopy4
|
||||
|
||||
// case tagCopy1:
|
||||
// s += 2
|
||||
ADD $2, R_SRC, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
TEST_SRC()
|
||||
|
||||
// offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
|
||||
// Calculate offset in R_TMP0 in case it is a repeat.
|
||||
MOVD R_LEN, R_TMP0
|
||||
AND $0xe0, R_TMP0
|
||||
MOVBU -1(R_SRC), R_TMP1
|
||||
ORR R_TMP0<<3, R_TMP1, R_TMP0
|
||||
|
||||
// length = 4 + int(src[s-2])>>2&0x7
|
||||
MOVD $7, R1
|
||||
AND R_LEN>>2, R1, R_LEN
|
||||
ADD $4, R_LEN, R_LEN
|
||||
|
||||
// check if repeat code with offset 0.
|
||||
CMP $0, R_TMP0
|
||||
BEQ repeatCode
|
||||
|
||||
// This is a regular copy, transfer our temporary value to R_OFF (offset)
|
||||
MOVD R_TMP0, R_OFF
|
||||
B doCopy
|
||||
|
||||
// This is a repeat code.
|
||||
repeatCode:
|
||||
// If length < 9, reuse last offset, with the length already calculated.
|
||||
CMP $9, R_LEN
|
||||
BLT doCopyRepeat
|
||||
BEQ repeatLen1
|
||||
CMP $10, R_LEN
|
||||
BEQ repeatLen2
|
||||
|
||||
repeatLen3:
|
||||
// s +=3
|
||||
ADD $3, R_SRC, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
TEST_SRC()
|
||||
|
||||
// length = uint32(src[s-3]) | (uint32(src[s-2])<<8) | (uint32(src[s-1])<<16) + 65540
|
||||
MOVBU -1(R_SRC), R_TMP0
|
||||
MOVHU -3(R_SRC), R_LEN
|
||||
ORR R_TMP0<<16, R_LEN, R_LEN
|
||||
ADD $65540, R_LEN, R_LEN
|
||||
B doCopyRepeat
|
||||
|
||||
repeatLen2:
|
||||
// s +=2
|
||||
ADD $2, R_SRC, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
TEST_SRC()
|
||||
|
||||
// length = uint32(src[s-2]) | (uint32(src[s-1])<<8) + 260
|
||||
MOVHU -2(R_SRC), R_LEN
|
||||
ADD $260, R_LEN, R_LEN
|
||||
B doCopyRepeat
|
||||
|
||||
repeatLen1:
|
||||
// s +=1
|
||||
ADD $1, R_SRC, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
TEST_SRC()
|
||||
|
||||
// length = src[s-1] + 8
|
||||
MOVBU -1(R_SRC), R_LEN
|
||||
ADD $8, R_LEN, R_LEN
|
||||
B doCopyRepeat
|
||||
|
||||
doCopy:
|
||||
// This is the end of the outer "switch", when we have a copy tag.
|
||||
//
|
||||
// We assume that:
|
||||
// - R_LEN == length && R_LEN > 0
|
||||
// - R_OFF == offset
|
||||
|
||||
// if d < offset { etc }
|
||||
MOVD R_DST, R_TMP1
|
||||
SUB R_DBASE, R_TMP1, R_TMP1
|
||||
CMP R_OFF, R_TMP1
|
||||
BLT errCorrupt
|
||||
|
||||
// Repeat values can skip the test above, since any offset > 0 will be in dst.
|
||||
doCopyRepeat:
|
||||
|
||||
// if offset <= 0 { etc }
|
||||
CMP $0, R_OFF
|
||||
BLE errCorrupt
|
||||
|
||||
// if length > len(dst)-d { etc }
|
||||
MOVD R_DEND, R_TMP1
|
||||
SUB R_DST, R_TMP1, R_TMP1
|
||||
CMP R_TMP1, R_LEN
|
||||
BGT errCorrupt
|
||||
|
||||
// forwardCopy(dst[d:d+length], dst[d-offset:]); d += length
|
||||
//
|
||||
// Set:
|
||||
// - R_TMP2 = len(dst)-d
|
||||
// - R_TMP3 = &dst[d-offset]
|
||||
MOVD R_DEND, R_TMP2
|
||||
SUB R_DST, R_TMP2, R_TMP2
|
||||
MOVD R_DST, R_TMP3
|
||||
SUB R_OFF, R_TMP3, R_TMP3
|
||||
|
||||
// !!! Try a faster technique for short (16 or fewer bytes) forward copies.
|
||||
//
|
||||
// First, try using two 8-byte load/stores, similar to the doLit technique
|
||||
// above. Even if dst[d:d+length] and dst[d-offset:] can overlap, this is
|
||||
// still OK if offset >= 8. Note that this has to be two 8-byte load/stores
|
||||
// and not one 16-byte load/store, and the first store has to be before the
|
||||
// second load, due to the overlap if offset is in the range [8, 16).
|
||||
//
|
||||
// if length > 16 || offset < 8 || len(dst)-d < 16 {
|
||||
// goto slowForwardCopy
|
||||
// }
|
||||
// copy 16 bytes
|
||||
// d += length
|
||||
CMP $16, R_LEN
|
||||
BGT slowForwardCopy
|
||||
CMP $8, R_OFF
|
||||
BLT slowForwardCopy
|
||||
CMP $16, R_TMP2
|
||||
BLT slowForwardCopy
|
||||
MOVD 0(R_TMP3), R_TMP0
|
||||
MOVD R_TMP0, 0(R_DST)
|
||||
MOVD 8(R_TMP3), R_TMP1
|
||||
MOVD R_TMP1, 8(R_DST)
|
||||
ADD R_LEN, R_DST, R_DST
|
||||
B loop
|
||||
|
||||
slowForwardCopy:
|
||||
// !!! If the forward copy is longer than 16 bytes, or if offset < 8, we
|
||||
// can still try 8-byte load stores, provided we can overrun up to 10 extra
|
||||
// bytes. As above, the overrun will be fixed up by subsequent iterations
|
||||
// of the outermost loop.
|
||||
//
|
||||
// The C++ snappy code calls this technique IncrementalCopyFastPath. Its
|
||||
// commentary says:
|
||||
//
|
||||
// ----
|
||||
//
|
||||
// The main part of this loop is a simple copy of eight bytes at a time
|
||||
// until we've copied (at least) the requested amount of bytes. However,
|
||||
// if d and d-offset are less than eight bytes apart (indicating a
|
||||
// repeating pattern of length < 8), we first need to expand the pattern in
|
||||
// order to get the correct results. For instance, if the buffer looks like
|
||||
// this, with the eight-byte <d-offset> and <d> patterns marked as
|
||||
// intervals:
|
||||
//
|
||||
// abxxxxxxxxxxxx
|
||||
// [------] d-offset
|
||||
// [------] d
|
||||
//
|
||||
// a single eight-byte copy from <d-offset> to <d> will repeat the pattern
|
||||
// once, after which we can move <d> two bytes without moving <d-offset>:
|
||||
//
|
||||
// ababxxxxxxxxxx
|
||||
// [------] d-offset
|
||||
// [------] d
|
||||
//
|
||||
// and repeat the exercise until the two no longer overlap.
|
||||
//
|
||||
// This allows us to do very well in the special case of one single byte
|
||||
// repeated many times, without taking a big hit for more general cases.
|
||||
//
|
||||
// The worst case of extra writing past the end of the match occurs when
|
||||
// offset == 1 and length == 1; the last copy will read from byte positions
|
||||
// [0..7] and write to [4..11], whereas it was only supposed to write to
|
||||
// position 1. Thus, ten excess bytes.
|
||||
//
|
||||
// ----
|
||||
//
|
||||
// That "10 byte overrun" worst case is confirmed by Go's
|
||||
// TestSlowForwardCopyOverrun, which also tests the fixUpSlowForwardCopy
|
||||
// and finishSlowForwardCopy algorithm.
|
||||
//
|
||||
// if length > len(dst)-d-10 {
|
||||
// goto verySlowForwardCopy
|
||||
// }
|
||||
SUB $10, R_TMP2, R_TMP2
|
||||
CMP R_TMP2, R_LEN
|
||||
BGT verySlowForwardCopy
|
||||
|
||||
// We want to keep the offset, so we use R_TMP2 from here.
|
||||
MOVD R_OFF, R_TMP2
|
||||
|
||||
makeOffsetAtLeast8:
|
||||
// !!! As above, expand the pattern so that offset >= 8 and we can use
|
||||
// 8-byte load/stores.
|
||||
//
|
||||
// for offset < 8 {
|
||||
// copy 8 bytes from dst[d-offset:] to dst[d:]
|
||||
// length -= offset
|
||||
// d += offset
|
||||
// offset += offset
|
||||
// // The two previous lines together means that d-offset, and therefore
|
||||
// // R_TMP3, is unchanged.
|
||||
// }
|
||||
CMP $8, R_TMP2
|
||||
BGE fixUpSlowForwardCopy
|
||||
MOVD (R_TMP3), R_TMP1
|
||||
MOVD R_TMP1, (R_DST)
|
||||
SUB R_TMP2, R_LEN, R_LEN
|
||||
ADD R_TMP2, R_DST, R_DST
|
||||
ADD R_TMP2, R_TMP2, R_TMP2
|
||||
B makeOffsetAtLeast8
|
||||
|
||||
fixUpSlowForwardCopy:
|
||||
// !!! Add length (which might be negative now) to d (implied by R_DST being
|
||||
// &dst[d]) so that d ends up at the right place when we jump back to the
|
||||
// top of the loop. Before we do that, though, we save R_DST to R_TMP0 so that, if
|
||||
// length is positive, copying the remaining length bytes will write to the
|
||||
// right place.
|
||||
MOVD R_DST, R_TMP0
|
||||
ADD R_LEN, R_DST, R_DST
|
||||
|
||||
finishSlowForwardCopy:
|
||||
// !!! Repeat 8-byte load/stores until length <= 0. Ending with a negative
|
||||
// length means that we overrun, but as above, that will be fixed up by
|
||||
// subsequent iterations of the outermost loop.
|
||||
MOVD $0, R1
|
||||
CMP R1, R_LEN
|
||||
BLE loop
|
||||
MOVD (R_TMP3), R_TMP1
|
||||
MOVD R_TMP1, (R_TMP0)
|
||||
ADD $8, R_TMP3, R_TMP3
|
||||
ADD $8, R_TMP0, R_TMP0
|
||||
SUB $8, R_LEN, R_LEN
|
||||
B finishSlowForwardCopy
|
||||
|
||||
verySlowForwardCopy:
|
||||
// verySlowForwardCopy is a simple implementation of forward copy. In C
|
||||
// parlance, this is a do/while loop instead of a while loop, since we know
|
||||
// that length > 0. In Go syntax:
|
||||
//
|
||||
// for {
|
||||
// dst[d] = dst[d - offset]
|
||||
// d++
|
||||
// length--
|
||||
// if length == 0 {
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
MOVB (R_TMP3), R_TMP1
|
||||
MOVB R_TMP1, (R_DST)
|
||||
ADD $1, R_TMP3, R_TMP3
|
||||
ADD $1, R_DST, R_DST
|
||||
SUB $1, R_LEN, R_LEN
|
||||
CBNZ R_LEN, verySlowForwardCopy
|
||||
B loop
|
||||
|
||||
// The code above handles copy tags.
|
||||
// ----------------------------------------
|
||||
|
||||
end:
|
||||
// This is the end of the "for s < len(src)".
|
||||
//
|
||||
// if d != len(dst) { etc }
|
||||
CMP R_DEND, R_DST
|
||||
BNE errCorrupt
|
||||
|
||||
// return 0
|
||||
MOVD $0, ret+48(FP)
|
||||
RET
|
||||
|
||||
errCorrupt:
|
||||
// return decodeErrCodeCorrupt
|
||||
MOVD $1, R_TMP0
|
||||
MOVD R_TMP0, ret+48(FP)
|
||||
RET
|
||||
17
vendor/github.com/klauspost/compress/s2/decode_asm.go
сгенерированный
поставляемый
Обычный файл
17
vendor/github.com/klauspost/compress/s2/decode_asm.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,17 @@
|
||||
// Copyright 2016 The Snappy-Go Authors. All rights reserved.
|
||||
// Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build (amd64 || arm64) && !appengine && gc && !noasm
|
||||
// +build amd64 arm64
|
||||
// +build !appengine
|
||||
// +build gc
|
||||
// +build !noasm
|
||||
|
||||
package s2
|
||||
|
||||
// decode has the same semantics as in decode_other.go.
|
||||
//
|
||||
//go:noescape
|
||||
func s2Decode(dst, src []byte) int
|
||||
267
vendor/github.com/klauspost/compress/s2/decode_other.go
сгенерированный
поставляемый
Обычный файл
267
vendor/github.com/klauspost/compress/s2/decode_other.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,267 @@
|
||||
// Copyright 2016 The Snappy-Go Authors. All rights reserved.
|
||||
// Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build (!amd64 && !arm64) || appengine || !gc || noasm
|
||||
// +build !amd64,!arm64 appengine !gc noasm
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// decode writes the decoding of src to dst. It assumes that the varint-encoded
|
||||
// length of the decompressed bytes has already been read, and that len(dst)
|
||||
// equals that length.
|
||||
//
|
||||
// It returns 0 on success or a decodeErrCodeXxx error code on failure.
|
||||
func s2Decode(dst, src []byte) int {
|
||||
const debug = false
|
||||
if debug {
|
||||
fmt.Println("Starting decode, dst len:", len(dst))
|
||||
}
|
||||
var d, s, length int
|
||||
offset := 0
|
||||
|
||||
// As long as we can read at least 5 bytes...
|
||||
for s < len(src)-5 {
|
||||
switch src[s] & 0x03 {
|
||||
case tagLiteral:
|
||||
x := uint32(src[s] >> 2)
|
||||
switch {
|
||||
case x < 60:
|
||||
s++
|
||||
case x == 60:
|
||||
s += 2
|
||||
x = uint32(src[s-1])
|
||||
case x == 61:
|
||||
s += 3
|
||||
x = uint32(src[s-2]) | uint32(src[s-1])<<8
|
||||
case x == 62:
|
||||
s += 4
|
||||
x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
|
||||
case x == 63:
|
||||
s += 5
|
||||
x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
|
||||
}
|
||||
length = int(x) + 1
|
||||
if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) {
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("literals, length:", length, "d-after:", d+length)
|
||||
}
|
||||
|
||||
copy(dst[d:], src[s:s+length])
|
||||
d += length
|
||||
s += length
|
||||
continue
|
||||
|
||||
case tagCopy1:
|
||||
s += 2
|
||||
length = int(src[s-2]) >> 2 & 0x7
|
||||
toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
|
||||
if toffset == 0 {
|
||||
if debug {
|
||||
fmt.Print("(repeat) ")
|
||||
}
|
||||
// keep last offset
|
||||
switch length {
|
||||
case 5:
|
||||
s += 1
|
||||
length = int(uint32(src[s-1])) + 4
|
||||
case 6:
|
||||
s += 2
|
||||
length = int(uint32(src[s-2])|(uint32(src[s-1])<<8)) + (1 << 8)
|
||||
case 7:
|
||||
s += 3
|
||||
length = int(uint32(src[s-3])|(uint32(src[s-2])<<8)|(uint32(src[s-1])<<16)) + (1 << 16)
|
||||
default: // 0-> 4
|
||||
}
|
||||
} else {
|
||||
offset = toffset
|
||||
}
|
||||
length += 4
|
||||
case tagCopy2:
|
||||
s += 3
|
||||
length = 1 + int(src[s-3])>>2
|
||||
offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
|
||||
|
||||
case tagCopy4:
|
||||
s += 5
|
||||
length = 1 + int(src[s-5])>>2
|
||||
offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
|
||||
}
|
||||
|
||||
if offset <= 0 || d < offset || length > len(dst)-d {
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
|
||||
if debug {
|
||||
fmt.Println("copy, length:", length, "offset:", offset, "d-after:", d+length)
|
||||
}
|
||||
|
||||
// Copy from an earlier sub-slice of dst to a later sub-slice.
|
||||
// If no overlap, use the built-in copy:
|
||||
if offset > length {
|
||||
copy(dst[d:d+length], dst[d-offset:])
|
||||
d += length
|
||||
continue
|
||||
}
|
||||
|
||||
// Unlike the built-in copy function, this byte-by-byte copy always runs
|
||||
// forwards, even if the slices overlap. Conceptually, this is:
|
||||
//
|
||||
// d += forwardCopy(dst[d:d+length], dst[d-offset:])
|
||||
//
|
||||
// We align the slices into a and b and show the compiler they are the same size.
|
||||
// This allows the loop to run without bounds checks.
|
||||
a := dst[d : d+length]
|
||||
b := dst[d-offset:]
|
||||
b = b[:len(a)]
|
||||
for i := range a {
|
||||
a[i] = b[i]
|
||||
}
|
||||
d += length
|
||||
}
|
||||
|
||||
// Remaining with extra checks...
|
||||
for s < len(src) {
|
||||
switch src[s] & 0x03 {
|
||||
case tagLiteral:
|
||||
x := uint32(src[s] >> 2)
|
||||
switch {
|
||||
case x < 60:
|
||||
s++
|
||||
case x == 60:
|
||||
s += 2
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-1])
|
||||
case x == 61:
|
||||
s += 3
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-2]) | uint32(src[s-1])<<8
|
||||
case x == 62:
|
||||
s += 4
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
|
||||
case x == 63:
|
||||
s += 5
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
|
||||
}
|
||||
length = int(x) + 1
|
||||
if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) {
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("literals, length:", length, "d-after:", d+length)
|
||||
}
|
||||
|
||||
copy(dst[d:], src[s:s+length])
|
||||
d += length
|
||||
s += length
|
||||
continue
|
||||
|
||||
case tagCopy1:
|
||||
s += 2
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(src[s-2]) >> 2 & 0x7
|
||||
toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
|
||||
if toffset == 0 {
|
||||
if debug {
|
||||
fmt.Print("(repeat) ")
|
||||
}
|
||||
// keep last offset
|
||||
switch length {
|
||||
case 5:
|
||||
s += 1
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(uint32(src[s-1])) + 4
|
||||
case 6:
|
||||
s += 2
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(uint32(src[s-2])|(uint32(src[s-1])<<8)) + (1 << 8)
|
||||
case 7:
|
||||
s += 3
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(uint32(src[s-3])|(uint32(src[s-2])<<8)|(uint32(src[s-1])<<16)) + (1 << 16)
|
||||
default: // 0-> 4
|
||||
}
|
||||
} else {
|
||||
offset = toffset
|
||||
}
|
||||
length += 4
|
||||
case tagCopy2:
|
||||
s += 3
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = 1 + int(src[s-3])>>2
|
||||
offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
|
||||
|
||||
case tagCopy4:
|
||||
s += 5
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = 1 + int(src[s-5])>>2
|
||||
offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
|
||||
}
|
||||
|
||||
if offset <= 0 || d < offset || length > len(dst)-d {
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
|
||||
if debug {
|
||||
fmt.Println("copy, length:", length, "offset:", offset, "d-after:", d+length)
|
||||
}
|
||||
|
||||
// Copy from an earlier sub-slice of dst to a later sub-slice.
|
||||
// If no overlap, use the built-in copy:
|
||||
if offset > length {
|
||||
copy(dst[d:d+length], dst[d-offset:])
|
||||
d += length
|
||||
continue
|
||||
}
|
||||
|
||||
// Unlike the built-in copy function, this byte-by-byte copy always runs
|
||||
// forwards, even if the slices overlap. Conceptually, this is:
|
||||
//
|
||||
// d += forwardCopy(dst[d:d+length], dst[d-offset:])
|
||||
//
|
||||
// We align the slices into a and b and show the compiler they are the same size.
|
||||
// This allows the loop to run without bounds checks.
|
||||
a := dst[d : d+length]
|
||||
b := dst[d-offset:]
|
||||
b = b[:len(a)]
|
||||
for i := range a {
|
||||
a[i] = b[i]
|
||||
}
|
||||
d += length
|
||||
}
|
||||
|
||||
if d != len(dst) {
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
return 0
|
||||
}
|
||||
1172
vendor/github.com/klauspost/compress/s2/encode.go
сгенерированный
поставляемый
Обычный файл
1172
vendor/github.com/klauspost/compress/s2/encode.go
сгенерированный
поставляемый
Обычный файл
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
456
vendor/github.com/klauspost/compress/s2/encode_all.go
сгенерированный
поставляемый
Обычный файл
456
vendor/github.com/klauspost/compress/s2/encode_all.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,456 @@
|
||||
// Copyright 2016 The Snappy-Go Authors. All rights reserved.
|
||||
// Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
func load32(b []byte, i int) uint32 {
|
||||
return binary.LittleEndian.Uint32(b[i:])
|
||||
}
|
||||
|
||||
func load64(b []byte, i int) uint64 {
|
||||
return binary.LittleEndian.Uint64(b[i:])
|
||||
}
|
||||
|
||||
// hash6 returns the hash of the lowest 6 bytes of u to fit in a hash table with h bits.
|
||||
// Preferably h should be a constant and should always be <64.
|
||||
func hash6(u uint64, h uint8) uint32 {
|
||||
const prime6bytes = 227718039650203
|
||||
return uint32(((u << (64 - 48)) * prime6bytes) >> ((64 - h) & 63))
|
||||
}
|
||||
|
||||
func encodeGo(dst, src []byte) []byte {
|
||||
if n := MaxEncodedLen(len(src)); n < 0 {
|
||||
panic(ErrTooLarge)
|
||||
} else if len(dst) < n {
|
||||
dst = make([]byte, n)
|
||||
}
|
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
d := binary.PutUvarint(dst, uint64(len(src)))
|
||||
|
||||
if len(src) == 0 {
|
||||
return dst[:d]
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
d += emitLiteral(dst[d:], src)
|
||||
return dst[:d]
|
||||
}
|
||||
n := encodeBlockGo(dst[d:], src)
|
||||
if n > 0 {
|
||||
d += n
|
||||
return dst[:d]
|
||||
}
|
||||
// Not compressible
|
||||
d += emitLiteral(dst[d:], src)
|
||||
return dst[:d]
|
||||
}
|
||||
|
||||
// encodeBlockGo encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockGo(dst, src []byte) (d int) {
|
||||
// Initialize the hash table.
|
||||
const (
|
||||
tableBits = 14
|
||||
maxTableSize = 1 << tableBits
|
||||
|
||||
debug = false
|
||||
)
|
||||
|
||||
var table [maxTableSize]uint32
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := len(src) - inputMargin
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - len(src)>>5 - 5
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 1
|
||||
cv := load64(src, s)
|
||||
|
||||
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
|
||||
repeat := 1
|
||||
|
||||
for {
|
||||
candidate := 0
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS := s + (s-nextEmit)>>6 + 4
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
hash0 := hash6(cv, tableBits)
|
||||
hash1 := hash6(cv>>8, tableBits)
|
||||
candidate = int(table[hash0])
|
||||
candidate2 := int(table[hash1])
|
||||
table[hash0] = uint32(s)
|
||||
table[hash1] = uint32(s + 1)
|
||||
hash2 := hash6(cv>>16, tableBits)
|
||||
|
||||
// Check repeat at offset checkRep.
|
||||
const checkRep = 1
|
||||
if uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) {
|
||||
base := s + checkRep
|
||||
// Extend back
|
||||
for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
|
||||
i--
|
||||
base--
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
|
||||
// Extend forward
|
||||
candidate := s - repeat + 4 + checkRep
|
||||
s += 4 + checkRep
|
||||
for s <= sLimit {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
if debug {
|
||||
// Validate match.
|
||||
if s <= candidate {
|
||||
panic("s <= candidate")
|
||||
}
|
||||
a := src[base:s]
|
||||
b := src[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
if nextEmit > 0 {
|
||||
// same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset.
|
||||
d += emitRepeat(dst[d:], repeat, s-base)
|
||||
} else {
|
||||
// First match, cannot be repeat.
|
||||
d += emitCopy(dst[d:], repeat, s-base)
|
||||
}
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
if uint32(cv) == load32(src, candidate) {
|
||||
break
|
||||
}
|
||||
candidate = int(table[hash2])
|
||||
if uint32(cv>>8) == load32(src, candidate2) {
|
||||
table[hash2] = uint32(s + 2)
|
||||
candidate = candidate2
|
||||
s++
|
||||
break
|
||||
}
|
||||
table[hash2] = uint32(s + 2)
|
||||
if uint32(cv>>16) == load32(src, candidate) {
|
||||
s += 2
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards.
|
||||
// The top bytes will be rechecked to get the full match.
|
||||
for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] {
|
||||
candidate--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
d += emitLiteral(dst[d:], src[nextEmit:s])
|
||||
|
||||
// Call emitCopy, and then see if another emitCopy could be our next
|
||||
// move. Repeat until we find no match for the input immediately after
|
||||
// what was consumed by the last emitCopy call.
|
||||
//
|
||||
// If we exit this loop normally then we need to call emitLiteral next,
|
||||
// though we don't yet know how big the literal will be. We handle that
|
||||
// by proceeding to the next iteration of the main loop. We also can
|
||||
// exit this loop via goto if we get close to exhausting the input.
|
||||
for {
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
base := s
|
||||
repeat = base - candidate
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidate += 4
|
||||
for s <= len(src)-8 {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopy(dst[d:], repeat, s-base)
|
||||
if debug {
|
||||
// Validate match.
|
||||
if s <= candidate {
|
||||
panic("s <= candidate")
|
||||
}
|
||||
a := src[base:s]
|
||||
b := src[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Check for an immediate match, otherwise start search at s+1
|
||||
x := load64(src, s-2)
|
||||
m2Hash := hash6(x, tableBits)
|
||||
currHash := hash6(x>>16, tableBits)
|
||||
candidate = int(table[currHash])
|
||||
table[m2Hash] = uint32(s - 2)
|
||||
table[currHash] = uint32(s)
|
||||
if debug && s == candidate {
|
||||
panic("s == candidate")
|
||||
}
|
||||
if uint32(x>>16) != load32(src, candidate) {
|
||||
cv = load64(src, s+1)
|
||||
s++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func encodeBlockSnappyGo(dst, src []byte) (d int) {
|
||||
// Initialize the hash table.
|
||||
const (
|
||||
tableBits = 14
|
||||
maxTableSize = 1 << tableBits
|
||||
)
|
||||
|
||||
var table [maxTableSize]uint32
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := len(src) - inputMargin
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - len(src)>>5 - 5
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 1
|
||||
cv := load64(src, s)
|
||||
|
||||
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
|
||||
repeat := 1
|
||||
|
||||
for {
|
||||
candidate := 0
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS := s + (s-nextEmit)>>6 + 4
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
hash0 := hash6(cv, tableBits)
|
||||
hash1 := hash6(cv>>8, tableBits)
|
||||
candidate = int(table[hash0])
|
||||
candidate2 := int(table[hash1])
|
||||
table[hash0] = uint32(s)
|
||||
table[hash1] = uint32(s + 1)
|
||||
hash2 := hash6(cv>>16, tableBits)
|
||||
|
||||
// Check repeat at offset checkRep.
|
||||
const checkRep = 1
|
||||
if uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) {
|
||||
base := s + checkRep
|
||||
// Extend back
|
||||
for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
|
||||
i--
|
||||
base--
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
|
||||
// Extend forward
|
||||
candidate := s - repeat + 4 + checkRep
|
||||
s += 4 + checkRep
|
||||
for s <= sLimit {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopyNoRepeat(dst[d:], repeat, s-base)
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
if uint32(cv) == load32(src, candidate) {
|
||||
break
|
||||
}
|
||||
candidate = int(table[hash2])
|
||||
if uint32(cv>>8) == load32(src, candidate2) {
|
||||
table[hash2] = uint32(s + 2)
|
||||
candidate = candidate2
|
||||
s++
|
||||
break
|
||||
}
|
||||
table[hash2] = uint32(s + 2)
|
||||
if uint32(cv>>16) == load32(src, candidate) {
|
||||
s += 2
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards
|
||||
for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] {
|
||||
candidate--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
d += emitLiteral(dst[d:], src[nextEmit:s])
|
||||
|
||||
// Call emitCopy, and then see if another emitCopy could be our next
|
||||
// move. Repeat until we find no match for the input immediately after
|
||||
// what was consumed by the last emitCopy call.
|
||||
//
|
||||
// If we exit this loop normally then we need to call emitLiteral next,
|
||||
// though we don't yet know how big the literal will be. We handle that
|
||||
// by proceeding to the next iteration of the main loop. We also can
|
||||
// exit this loop via goto if we get close to exhausting the input.
|
||||
for {
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
base := s
|
||||
repeat = base - candidate
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidate += 4
|
||||
for s <= len(src)-8 {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopyNoRepeat(dst[d:], repeat, s-base)
|
||||
if false {
|
||||
// Validate match.
|
||||
a := src[base:s]
|
||||
b := src[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Check for an immediate match, otherwise start search at s+1
|
||||
x := load64(src, s-2)
|
||||
m2Hash := hash6(x, tableBits)
|
||||
currHash := hash6(x>>16, tableBits)
|
||||
candidate = int(table[currHash])
|
||||
table[m2Hash] = uint32(s - 2)
|
||||
table[currHash] = uint32(s)
|
||||
if uint32(x>>16) != load32(src, candidate) {
|
||||
cv = load64(src, s+1)
|
||||
s++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
}
|
||||
142
vendor/github.com/klauspost/compress/s2/encode_amd64.go
сгенерированный
поставляемый
Обычный файл
142
vendor/github.com/klauspost/compress/s2/encode_amd64.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,142 @@
|
||||
//go:build !appengine && !noasm && gc
|
||||
// +build !appengine,!noasm,gc
|
||||
|
||||
package s2
|
||||
|
||||
// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlock(dst, src []byte) (d int) {
|
||||
const (
|
||||
// Use 12 bit table when less than...
|
||||
limit12B = 16 << 10
|
||||
// Use 10 bit table when less than...
|
||||
limit10B = 4 << 10
|
||||
// Use 8 bit table when less than...
|
||||
limit8B = 512
|
||||
)
|
||||
|
||||
if len(src) >= 4<<20 {
|
||||
return encodeBlockAsm(dst, src)
|
||||
}
|
||||
if len(src) >= limit12B {
|
||||
return encodeBlockAsm4MB(dst, src)
|
||||
}
|
||||
if len(src) >= limit10B {
|
||||
return encodeBlockAsm12B(dst, src)
|
||||
}
|
||||
if len(src) >= limit8B {
|
||||
return encodeBlockAsm10B(dst, src)
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
return encodeBlockAsm8B(dst, src)
|
||||
}
|
||||
|
||||
// encodeBlockBetter encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBetter(dst, src []byte) (d int) {
|
||||
const (
|
||||
// Use 12 bit table when less than...
|
||||
limit12B = 16 << 10
|
||||
// Use 10 bit table when less than...
|
||||
limit10B = 4 << 10
|
||||
// Use 8 bit table when less than...
|
||||
limit8B = 512
|
||||
)
|
||||
|
||||
if len(src) > 4<<20 {
|
||||
return encodeBetterBlockAsm(dst, src)
|
||||
}
|
||||
if len(src) >= limit12B {
|
||||
return encodeBetterBlockAsm4MB(dst, src)
|
||||
}
|
||||
if len(src) >= limit10B {
|
||||
return encodeBetterBlockAsm12B(dst, src)
|
||||
}
|
||||
if len(src) >= limit8B {
|
||||
return encodeBetterBlockAsm10B(dst, src)
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
return encodeBetterBlockAsm8B(dst, src)
|
||||
}
|
||||
|
||||
// encodeBlockSnappy encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockSnappy(dst, src []byte) (d int) {
|
||||
const (
|
||||
// Use 12 bit table when less than...
|
||||
limit12B = 16 << 10
|
||||
// Use 10 bit table when less than...
|
||||
limit10B = 4 << 10
|
||||
// Use 8 bit table when less than...
|
||||
limit8B = 512
|
||||
)
|
||||
if len(src) >= 64<<10 {
|
||||
return encodeSnappyBlockAsm(dst, src)
|
||||
}
|
||||
if len(src) >= limit12B {
|
||||
return encodeSnappyBlockAsm64K(dst, src)
|
||||
}
|
||||
if len(src) >= limit10B {
|
||||
return encodeSnappyBlockAsm12B(dst, src)
|
||||
}
|
||||
if len(src) >= limit8B {
|
||||
return encodeSnappyBlockAsm10B(dst, src)
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
return encodeSnappyBlockAsm8B(dst, src)
|
||||
}
|
||||
|
||||
// encodeBlockSnappy encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBetterSnappy(dst, src []byte) (d int) {
|
||||
const (
|
||||
// Use 12 bit table when less than...
|
||||
limit12B = 16 << 10
|
||||
// Use 10 bit table when less than...
|
||||
limit10B = 4 << 10
|
||||
// Use 8 bit table when less than...
|
||||
limit8B = 512
|
||||
)
|
||||
if len(src) >= 64<<10 {
|
||||
return encodeSnappyBetterBlockAsm(dst, src)
|
||||
}
|
||||
if len(src) >= limit12B {
|
||||
return encodeSnappyBetterBlockAsm64K(dst, src)
|
||||
}
|
||||
if len(src) >= limit10B {
|
||||
return encodeSnappyBetterBlockAsm12B(dst, src)
|
||||
}
|
||||
if len(src) >= limit8B {
|
||||
return encodeSnappyBetterBlockAsm10B(dst, src)
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
return encodeSnappyBetterBlockAsm8B(dst, src)
|
||||
}
|
||||
604
vendor/github.com/klauspost/compress/s2/encode_best.go
сгенерированный
поставляемый
Обычный файл
604
vendor/github.com/klauspost/compress/s2/encode_best.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,604 @@
|
||||
// Copyright 2016 The Snappy-Go Authors. All rights reserved.
|
||||
// Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
// encodeBlockBest encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBest(dst, src []byte) (d int) {
|
||||
// Initialize the hash tables.
|
||||
const (
|
||||
// Long hash matches.
|
||||
lTableBits = 19
|
||||
maxLTableSize = 1 << lTableBits
|
||||
|
||||
// Short hash matches.
|
||||
sTableBits = 16
|
||||
maxSTableSize = 1 << sTableBits
|
||||
|
||||
inputMargin = 8 + 2
|
||||
)
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := len(src) - inputMargin
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
|
||||
var lTable [maxLTableSize]uint64
|
||||
var sTable [maxSTableSize]uint64
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - 5
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 1
|
||||
cv := load64(src, s)
|
||||
|
||||
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
|
||||
repeat := 1
|
||||
const lowbitMask = 0xffffffff
|
||||
getCur := func(x uint64) int {
|
||||
return int(x & lowbitMask)
|
||||
}
|
||||
getPrev := func(x uint64) int {
|
||||
return int(x >> 32)
|
||||
}
|
||||
const maxSkip = 64
|
||||
|
||||
for {
|
||||
type match struct {
|
||||
offset int
|
||||
s int
|
||||
length int
|
||||
score int
|
||||
rep bool
|
||||
}
|
||||
var best match
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS := (s-nextEmit)>>8 + 1
|
||||
if nextS > maxSkip {
|
||||
nextS = s + maxSkip
|
||||
} else {
|
||||
nextS += s
|
||||
}
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
hashL := hash8(cv, lTableBits)
|
||||
hashS := hash4(cv, sTableBits)
|
||||
candidateL := lTable[hashL]
|
||||
candidateS := sTable[hashS]
|
||||
|
||||
score := func(m match) int {
|
||||
// Matches that are longer forward are penalized since we must emit it as a literal.
|
||||
score := m.length - m.s
|
||||
if nextEmit == m.s {
|
||||
// If we do not have to emit literals, we save 1 byte
|
||||
score++
|
||||
}
|
||||
offset := m.s - m.offset
|
||||
if m.rep {
|
||||
return score - emitRepeatSize(offset, m.length)
|
||||
}
|
||||
return score - emitCopySize(offset, m.length)
|
||||
}
|
||||
|
||||
matchAt := func(offset, s int, first uint32, rep bool) match {
|
||||
if best.length != 0 && best.s-best.offset == s-offset {
|
||||
// Don't retest if we have the same offset.
|
||||
return match{offset: offset, s: s}
|
||||
}
|
||||
if load32(src, offset) != first {
|
||||
return match{offset: offset, s: s}
|
||||
}
|
||||
m := match{offset: offset, s: s, length: 4 + offset, rep: rep}
|
||||
s += 4
|
||||
for s <= sLimit {
|
||||
if diff := load64(src, s) ^ load64(src, m.length); diff != 0 {
|
||||
m.length += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
m.length += 8
|
||||
}
|
||||
m.length -= offset
|
||||
m.score = score(m)
|
||||
if m.score <= -m.s {
|
||||
// Eliminate if no savings, we might find a better one.
|
||||
m.length = 0
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
bestOf := func(a, b match) match {
|
||||
if b.length == 0 {
|
||||
return a
|
||||
}
|
||||
if a.length == 0 {
|
||||
return b
|
||||
}
|
||||
as := a.score + b.s
|
||||
bs := b.score + a.s
|
||||
if as >= bs {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
best = bestOf(matchAt(getCur(candidateL), s, uint32(cv), false), matchAt(getPrev(candidateL), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getCur(candidateS), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(candidateS), s, uint32(cv), false))
|
||||
|
||||
{
|
||||
best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8), true))
|
||||
if best.length > 0 {
|
||||
// s+1
|
||||
nextShort := sTable[hash4(cv>>8, sTableBits)]
|
||||
s := s + 1
|
||||
cv := load64(src, s)
|
||||
nextLong := lTable[hash8(cv, lTableBits)]
|
||||
best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv), false))
|
||||
// Repeat at + 2
|
||||
best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8), true))
|
||||
|
||||
// s+2
|
||||
if true {
|
||||
nextShort = sTable[hash4(cv>>8, sTableBits)]
|
||||
s++
|
||||
cv = load64(src, s)
|
||||
nextLong = lTable[hash8(cv, lTableBits)]
|
||||
best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv), false))
|
||||
}
|
||||
// Search for a match at best match end, see if that is better.
|
||||
if sAt := best.s + best.length; sAt < sLimit {
|
||||
sBack := best.s
|
||||
backL := best.length
|
||||
// Load initial values
|
||||
cv = load64(src, sBack)
|
||||
// Search for mismatch
|
||||
next := lTable[hash8(load64(src, sAt), lTableBits)]
|
||||
//next := sTable[hash4(load64(src, sAt), sTableBits)]
|
||||
|
||||
if checkAt := getCur(next) - backL; checkAt > 0 {
|
||||
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
|
||||
}
|
||||
if checkAt := getPrev(next) - backL; checkAt > 0 {
|
||||
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update table
|
||||
lTable[hashL] = uint64(s) | candidateL<<32
|
||||
sTable[hashS] = uint64(s) | candidateS<<32
|
||||
|
||||
if best.length > 0 {
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards, not needed for repeats...
|
||||
s = best.s
|
||||
if !best.rep {
|
||||
for best.offset > 0 && s > nextEmit && src[best.offset-1] == src[s-1] {
|
||||
best.offset--
|
||||
best.length++
|
||||
s--
|
||||
}
|
||||
}
|
||||
if false && best.offset >= s {
|
||||
panic(fmt.Errorf("t %d >= s %d", best.offset, s))
|
||||
}
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
base := s
|
||||
offset := s - best.offset
|
||||
|
||||
s += best.length
|
||||
|
||||
if offset > 65535 && s-base <= 5 && !best.rep {
|
||||
// Bail if the match is equal or worse to the encoding.
|
||||
s = best.s + 1
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
if best.rep {
|
||||
if nextEmit > 0 {
|
||||
// same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset.
|
||||
d += emitRepeat(dst[d:], offset, best.length)
|
||||
} else {
|
||||
// First match, cannot be repeat.
|
||||
d += emitCopy(dst[d:], offset, best.length)
|
||||
}
|
||||
} else {
|
||||
d += emitCopy(dst[d:], offset, best.length)
|
||||
}
|
||||
repeat = offset
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Fill tables...
|
||||
for i := best.s + 1; i < s; i++ {
|
||||
cv0 := load64(src, i)
|
||||
long0 := hash8(cv0, lTableBits)
|
||||
short0 := hash4(cv0, sTableBits)
|
||||
lTable[long0] = uint64(i) | lTable[long0]<<32
|
||||
sTable[short0] = uint64(i) | sTable[short0]<<32
|
||||
}
|
||||
cv = load64(src, s)
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// encodeBlockBestSnappy encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBestSnappy(dst, src []byte) (d int) {
|
||||
// Initialize the hash tables.
|
||||
const (
|
||||
// Long hash matches.
|
||||
lTableBits = 19
|
||||
maxLTableSize = 1 << lTableBits
|
||||
|
||||
// Short hash matches.
|
||||
sTableBits = 16
|
||||
maxSTableSize = 1 << sTableBits
|
||||
|
||||
inputMargin = 8 + 2
|
||||
)
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := len(src) - inputMargin
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
|
||||
var lTable [maxLTableSize]uint64
|
||||
var sTable [maxSTableSize]uint64
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - 5
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 1
|
||||
cv := load64(src, s)
|
||||
|
||||
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
|
||||
repeat := 1
|
||||
const lowbitMask = 0xffffffff
|
||||
getCur := func(x uint64) int {
|
||||
return int(x & lowbitMask)
|
||||
}
|
||||
getPrev := func(x uint64) int {
|
||||
return int(x >> 32)
|
||||
}
|
||||
const maxSkip = 64
|
||||
|
||||
for {
|
||||
type match struct {
|
||||
offset int
|
||||
s int
|
||||
length int
|
||||
score int
|
||||
}
|
||||
var best match
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS := (s-nextEmit)>>8 + 1
|
||||
if nextS > maxSkip {
|
||||
nextS = s + maxSkip
|
||||
} else {
|
||||
nextS += s
|
||||
}
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
hashL := hash8(cv, lTableBits)
|
||||
hashS := hash4(cv, sTableBits)
|
||||
candidateL := lTable[hashL]
|
||||
candidateS := sTable[hashS]
|
||||
|
||||
score := func(m match) int {
|
||||
// Matches that are longer forward are penalized since we must emit it as a literal.
|
||||
score := m.length - m.s
|
||||
if nextEmit == m.s {
|
||||
// If we do not have to emit literals, we save 1 byte
|
||||
score++
|
||||
}
|
||||
offset := m.s - m.offset
|
||||
|
||||
return score - emitCopySize(offset, m.length)
|
||||
}
|
||||
|
||||
matchAt := func(offset, s int, first uint32) match {
|
||||
if best.length != 0 && best.s-best.offset == s-offset {
|
||||
// Don't retest if we have the same offset.
|
||||
return match{offset: offset, s: s}
|
||||
}
|
||||
if load32(src, offset) != first {
|
||||
return match{offset: offset, s: s}
|
||||
}
|
||||
m := match{offset: offset, s: s, length: 4 + offset}
|
||||
s += 4
|
||||
for s <= sLimit {
|
||||
if diff := load64(src, s) ^ load64(src, m.length); diff != 0 {
|
||||
m.length += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
m.length += 8
|
||||
}
|
||||
m.length -= offset
|
||||
m.score = score(m)
|
||||
if m.score <= -m.s {
|
||||
// Eliminate if no savings, we might find a better one.
|
||||
m.length = 0
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
bestOf := func(a, b match) match {
|
||||
if b.length == 0 {
|
||||
return a
|
||||
}
|
||||
if a.length == 0 {
|
||||
return b
|
||||
}
|
||||
as := a.score + b.s
|
||||
bs := b.score + a.s
|
||||
if as >= bs {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
best = bestOf(matchAt(getCur(candidateL), s, uint32(cv)), matchAt(getPrev(candidateL), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getCur(candidateS), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getPrev(candidateS), s, uint32(cv)))
|
||||
|
||||
{
|
||||
best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8)))
|
||||
if best.length > 0 {
|
||||
// s+1
|
||||
nextShort := sTable[hash4(cv>>8, sTableBits)]
|
||||
s := s + 1
|
||||
cv := load64(src, s)
|
||||
nextLong := lTable[hash8(cv, lTableBits)]
|
||||
best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv)))
|
||||
// Repeat at + 2
|
||||
best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8)))
|
||||
|
||||
// s+2
|
||||
if true {
|
||||
nextShort = sTable[hash4(cv>>8, sTableBits)]
|
||||
s++
|
||||
cv = load64(src, s)
|
||||
nextLong = lTable[hash8(cv, lTableBits)]
|
||||
best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv)))
|
||||
}
|
||||
// Search for a match at best match end, see if that is better.
|
||||
if sAt := best.s + best.length; sAt < sLimit {
|
||||
sBack := best.s
|
||||
backL := best.length
|
||||
// Load initial values
|
||||
cv = load64(src, sBack)
|
||||
// Search for mismatch
|
||||
next := lTable[hash8(load64(src, sAt), lTableBits)]
|
||||
//next := sTable[hash4(load64(src, sAt), sTableBits)]
|
||||
|
||||
if checkAt := getCur(next) - backL; checkAt > 0 {
|
||||
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv)))
|
||||
}
|
||||
if checkAt := getPrev(next) - backL; checkAt > 0 {
|
||||
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update table
|
||||
lTable[hashL] = uint64(s) | candidateL<<32
|
||||
sTable[hashS] = uint64(s) | candidateS<<32
|
||||
|
||||
if best.length > 0 {
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards, not needed for repeats...
|
||||
s = best.s
|
||||
if true {
|
||||
for best.offset > 0 && s > nextEmit && src[best.offset-1] == src[s-1] {
|
||||
best.offset--
|
||||
best.length++
|
||||
s--
|
||||
}
|
||||
}
|
||||
if false && best.offset >= s {
|
||||
panic(fmt.Errorf("t %d >= s %d", best.offset, s))
|
||||
}
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
base := s
|
||||
offset := s - best.offset
|
||||
|
||||
s += best.length
|
||||
|
||||
if offset > 65535 && s-base <= 5 {
|
||||
// Bail if the match is equal or worse to the encoding.
|
||||
s = best.s + 1
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
d += emitCopyNoRepeat(dst[d:], offset, best.length)
|
||||
repeat = offset
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Fill tables...
|
||||
for i := best.s + 1; i < s; i++ {
|
||||
cv0 := load64(src, i)
|
||||
long0 := hash8(cv0, lTableBits)
|
||||
short0 := hash4(cv0, sTableBits)
|
||||
lTable[long0] = uint64(i) | lTable[long0]<<32
|
||||
sTable[short0] = uint64(i) | sTable[short0]<<32
|
||||
}
|
||||
cv = load64(src, s)
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// emitCopySize returns the size to encode the offset+length
|
||||
//
|
||||
// It assumes that:
|
||||
// 1 <= offset && offset <= math.MaxUint32
|
||||
// 4 <= length && length <= 1 << 24
|
||||
func emitCopySize(offset, length int) int {
|
||||
if offset >= 65536 {
|
||||
i := 0
|
||||
if length > 64 {
|
||||
length -= 64
|
||||
if length >= 4 {
|
||||
// Emit remaining as repeats
|
||||
return 5 + emitRepeatSize(offset, length)
|
||||
}
|
||||
i = 5
|
||||
}
|
||||
if length == 0 {
|
||||
return i
|
||||
}
|
||||
return i + 5
|
||||
}
|
||||
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 {
|
||||
// Emit remaining as repeats, at least 4 bytes remain.
|
||||
return 3 + emitRepeatSize(offset, length-60)
|
||||
}
|
||||
if length >= 12 || offset >= 2048 {
|
||||
return 3
|
||||
}
|
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
return 2
|
||||
}
|
||||
|
||||
// emitRepeatSize returns the number of bytes required to encode a repeat.
|
||||
// Length must be at least 4 and < 1<<24
|
||||
func emitRepeatSize(offset, length int) int {
|
||||
// Repeat offset, make length cheaper
|
||||
if length <= 4+4 || (length < 8+4 && offset < 2048) {
|
||||
return 2
|
||||
}
|
||||
if length < (1<<8)+4+4 {
|
||||
return 3
|
||||
}
|
||||
if length < (1<<16)+(1<<8)+4 {
|
||||
return 4
|
||||
}
|
||||
const maxRepeat = (1 << 24) - 1
|
||||
length -= (1 << 16) - 4
|
||||
left := 0
|
||||
if length > maxRepeat {
|
||||
left = length - maxRepeat + 4
|
||||
length = maxRepeat - 4
|
||||
}
|
||||
if left > 0 {
|
||||
return 5 + emitRepeatSize(offset, left)
|
||||
}
|
||||
return 5
|
||||
}
|
||||
Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше
Ссылка в новой задаче
Block a user