mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

Upgrading server dependancies (#8308)

Просмотр исходного кода
Этот коммит содержится в:
Christopher Speller
2018-02-16 06:47:51 -08:00
коммит произвёл Joram Wilander
родитель b112747de7
Коммит 6d8f122a51
602 изменённых файлов: 23090 добавлений и 129178 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

187
vendor/github.com/davecgh/go-spew/spew/bypass.go сгенерированный поставляемый
Просмотреть файл

@@ -16,7 +16,9 @@
// when the code is not running on Google App Engine, compiled by GopherJS, and
// "-tags safe" is not added to the go build command line. The "disableunsafe"
// tag is deprecated and thus should not be used.
// +build !js,!appengine,!safe,!disableunsafe
// Go versions prior to 1.4 are disabled because they use a different layout
// for interfaces which make the implementation of unsafeReflectValue more complex.
// +build !js,!appengine,!safe,!disableunsafe,go1.4
package spew
@@ -34,80 +36,49 @@ const (
ptrSize = unsafe.Sizeof((*byte)(nil))
)
var (
// offsetPtr, offsetScalar, and offsetFlag are the offsets for the
// internal reflect.Value fields. These values are valid before golang
// commit ecccf07e7f9d which changed the format. The are also valid
// after commit 82f48826c6c7 which changed the format again to mirror
// the original format. Code in the init function updates these offsets
// as necessary.
offsetPtr = ptrSize
offsetScalar = uintptr(0)
offsetFlag = ptrSize * 2
type flag uintptr
// flagKindWidth and flagKindShift indicate various bits that the
// reflect package uses internally to track kind information.
//
// flagRO indicates whether or not the value field of a reflect.Value is
// read-only.
//
// flagIndir indicates whether the value field of a reflect.Value is
// the actual data or a pointer to the data.
//
// These values are valid before golang commit 90a7c3c86944 which
// changed their positions. Code in the init function updates these
// flags as necessary.
flagKindWidth = uintptr(5)
flagKindShift = flagKindWidth - 1
flagRO = uintptr(1 << 0)
flagIndir = uintptr(1 << 1)
var (
// flagRO indicates whether the value field of a reflect.Value
// is read-only.
flagRO flag
// flagAddr indicates whether the address of the reflect.Value's
// value may be taken.
flagAddr flag
)
func init() {
// Older versions of reflect.Value stored small integers directly in the
// ptr field (which is named val in the older versions). Versions
// between commits ecccf07e7f9d and 82f48826c6c7 added a new field named
// scalar for this purpose which unfortunately came before the flag
// field, so the offset of the flag field is different for those
// versions.
//
// This code constructs a new reflect.Value from a known small integer
// and checks if the size of the reflect.Value struct indicates it has
// the scalar field. When it does, the offsets are updated accordingly.
vv := reflect.ValueOf(0xf00)
if unsafe.Sizeof(vv) == (ptrSize * 4) {
offsetScalar = ptrSize * 2
offsetFlag = ptrSize * 3
}
// flagKindMask holds the bits that make up the kind
// part of the flags field. In all the supported versions,
// it is in the lower 5 bits.
const flagKindMask = flag(0x1f)
// Commit 90a7c3c86944 changed the flag positions such that the low
// order bits are the kind. This code extracts the kind from the flags
// field and ensures it's the correct type. When it's not, the flag
// order has been changed to the newer format, so the flags are updated
// accordingly.
upf := unsafe.Pointer(uintptr(unsafe.Pointer(&vv)) + offsetFlag)
upfv := *(*uintptr)(upf)
flagKindMask := uintptr((1<<flagKindWidth - 1) << flagKindShift)
if (upfv&flagKindMask)>>flagKindShift != uintptr(reflect.Int) {
flagKindShift = 0
flagRO = 1 << 5
flagIndir = 1 << 6
// Different versions of Go have used different
// bit layouts for the flags type. This table
// records the known combinations.
var okFlags = []struct {
ro, addr flag
}{{
// From Go 1.4 to 1.5
ro: 1 << 5,
addr: 1 << 7,
}, {
// Up to Go tip.
ro: 1<<5 | 1<<6,
addr: 1 << 8,
}}
// Commit adf9b30e5594 modified the flags to separate the
// flagRO flag into two bits which specifies whether or not the
// field is embedded. This causes flagIndir to move over a bit
// and means that flagRO is the combination of either of the
// original flagRO bit and the new bit.
//
// This code detects the change by extracting what used to be
// the indirect bit to ensure it's set. When it's not, the flag
// order has been changed to the newer format, so the flags are
// updated accordingly.
if upfv&flagIndir == 0 {
flagRO = 3 << 5
flagIndir = 1 << 7
}
var flagValOffset = func() uintptr {
field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag")
if !ok {
panic("reflect.Value has no flag field")
}
return field.Offset
}()
// flagField returns a pointer to the flag field of a reflect.Value.
func flagField(v *reflect.Value) *flag {
return (*flag)(unsafe.Pointer(uintptr(unsafe.Pointer(v)) + flagValOffset))
}
// unsafeReflectValue converts the passed reflect.Value into a one that bypasses
@@ -119,34 +90,56 @@ func init() {
// This allows us to check for implementations of the Stringer and error
// interfaces to be used for pretty printing ordinarily unaddressable and
// inaccessible values such as unexported struct fields.
func unsafeReflectValue(v reflect.Value) (rv reflect.Value) {
indirects := 1
vt := v.Type()
upv := unsafe.Pointer(uintptr(unsafe.Pointer(&v)) + offsetPtr)
rvf := *(*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(&v)) + offsetFlag))
if rvf&flagIndir != 0 {
vt = reflect.PtrTo(v.Type())
indirects++
} else if offsetScalar != 0 {
// The value is in the scalar field when it's not one of the
// reference types.
switch vt.Kind() {
case reflect.Uintptr:
case reflect.Chan:
case reflect.Func:
case reflect.Map:
case reflect.Ptr:
case reflect.UnsafePointer:
default:
upv = unsafe.Pointer(uintptr(unsafe.Pointer(&v)) +
offsetScalar)
func unsafeReflectValue(v reflect.Value) reflect.Value {
if !v.IsValid() || (v.CanInterface() && v.CanAddr()) {
return v
}
flagFieldPtr := flagField(&v)
*flagFieldPtr &^= flagRO
*flagFieldPtr |= flagAddr
return v
}
// Sanity checks against future reflect package changes
// to the type or semantics of the Value.flag field.
func init() {
field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag")
if !ok {
panic("reflect.Value has no flag field")
}
if field.Type.Kind() != reflect.TypeOf(flag(0)).Kind() {
panic("reflect.Value flag field has changed kind")
}
type t0 int
var t struct {
A t0
// t0 will have flagEmbedRO set.
t0
// a will have flagStickyRO set
a t0
}
vA := reflect.ValueOf(t).FieldByName("A")
va := reflect.ValueOf(t).FieldByName("a")
vt0 := reflect.ValueOf(t).FieldByName("t0")
// Infer flagRO from the difference between the flags
// for the (otherwise identical) fields in t.
flagPublic := *flagField(&vA)
flagWithRO := *flagField(&va) | *flagField(&vt0)
flagRO = flagPublic ^ flagWithRO
// Infer flagAddr from the difference between a value
// taken from a pointer and not.
vPtrA := reflect.ValueOf(&t).Elem().FieldByName("A")
flagNoPtr := *flagField(&vA)
flagPtr := *flagField(&vPtrA)
flagAddr = flagNoPtr ^ flagPtr
// Check that the inferred flags tally with one of the known versions.
for _, f := range okFlags {
if flagRO == f.ro && flagAddr == f.addr {
return
}
}
pv := reflect.NewAt(vt, upv)
rv = pv
for i := 0; i < indirects; i++ {
rv = rv.Elem()
}
return rv
panic("reflect.Value read-only flag has changed semantics")
}

2
vendor/github.com/davecgh/go-spew/spew/bypasssafe.go сгенерированный поставляемый
Просмотреть файл

@@ -16,7 +16,7 @@
// when the code is running on Google App Engine, compiled by GopherJS, or
// "-tags safe" is added to the go build command line. The "disableunsafe"
// tag is deprecated and thus should not be used.
// +build js appengine safe disableunsafe
// +build js appengine safe disableunsafe !go1.4
package spew

2
vendor/github.com/davecgh/go-spew/spew/dump_test.go сгенерированный поставляемый
Просмотреть файл

@@ -768,7 +768,7 @@ func addUintptrDumpTests() {
func addUnsafePointerDumpTests() {
// Null pointer.
v := unsafe.Pointer(uintptr(0))
v := unsafe.Pointer(nil)
nv := (*unsafe.Pointer)(nil)
pv := &v
vAddr := fmt.Sprintf("%p", pv)

2
vendor/github.com/davecgh/go-spew/spew/format_test.go сгенерированный поставляемый
Просмотреть файл

@@ -1083,7 +1083,7 @@ func addUintptrFormatterTests() {
func addUnsafePointerFormatterTests() {
// Null pointer.
v := unsafe.Pointer(uintptr(0))
v := unsafe.Pointer(nil)
nv := (*unsafe.Pointer)(nil)
pv := &v
vAddr := fmt.Sprintf("%p", pv)

11
vendor/github.com/davecgh/go-spew/spew/internalunsafe_test.go сгенерированный поставляемый
Просмотреть файл

@@ -16,7 +16,7 @@
// when the code is not running on Google App Engine, compiled by GopherJS, and
// "-tags safe" is not added to the go build command line. The "disableunsafe"
// tag is deprecated and thus should not be used.
// +build !js,!appengine,!safe,!disableunsafe
// +build !js,!appengine,!safe,!disableunsafe,go1.4
/*
This test file is part of the spew package rather than than the spew_test
@@ -30,7 +30,6 @@ import (
"bytes"
"reflect"
"testing"
"unsafe"
)
// changeKind uses unsafe to intentionally change the kind of a reflect.Value to
@@ -38,13 +37,13 @@ import (
// fallback code which punts to the standard fmt library for new types that
// might get added to the language.
func changeKind(v *reflect.Value, readOnly bool) {
rvf := (*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(v)) + offsetFlag))
*rvf = *rvf | ((1<<flagKindWidth - 1) << flagKindShift)
flags := flagField(v)
if readOnly {
*rvf |= flagRO
*flags |= flagRO
} else {
*rvf &= ^uintptr(flagRO)
*flags &^= flagRO
}
*flags |= flagKindMask
}
// TestAddedReflectValue tests functionaly of the dump and formatter code which

93
vendor/github.com/go-ini/ini/file_test.go сгенерированный поставляемый
Просмотреть файл

@@ -16,6 +16,7 @@ package ini_test
import (
"bytes"
"io/ioutil"
"testing"
. "github.com/smartystreets/goconvey/convey"
@@ -253,93 +254,15 @@ func TestFile_WriteTo(t *testing.T) {
var buf bytes.Buffer
_, err = f.WriteTo(&buf)
So(err, ShouldBeNil)
So(buf.String(), ShouldEqual, `; Package name
NAME = ini
; Package version
VERSION = v1
; Package import path
IMPORT_PATH = gopkg.in/%(NAME)s.%(VERSION)s
; Information about package author
# Bio can be written in multiple lines.
[author]
; This is author name
NAME = Unknwon
E-MAIL = u@gogs.io
GITHUB = https://github.com/%(NAME)s
# Succeeding comment
BIO = """Gopher.
Coding addict.
Good man.
"""
golden := "testdata/TestFile_WriteTo.golden"
if *update {
ioutil.WriteFile(golden, buf.Bytes(), 0644)
}
[package]
CLONE_URL = https://%(IMPORT_PATH)s
[package.sub]
UNUSED_KEY = should be deleted
[features]
- = Support read/write comments of keys and sections
- = Support auto-increment of key names
- = Support load multiple files to overwrite key values
[types]
STRING = str
BOOL = true
BOOL_FALSE = false
FLOAT64 = 1.25
INT = 10
TIME = 2015-01-01T20:17:05Z
DURATION = 2h45m
UINT = 3
[array]
STRINGS = en, zh, de
FLOAT64S = 1.1, 2.2, 3.3
INTS = 1, 2, 3
UINTS = 1, 2, 3
TIMES = 2015-01-01T20:17:05Z,2015-01-01T20:17:05Z,2015-01-01T20:17:05Z
[note]
empty_lines = next line is empty
boolean_key
more = notes
; Comment before the section
; This is a comment for the section too
[comments]
; Comment before key
key = value
; This is a comment for key2
key2 = value2
key3 = "one", "two", "three"
[string escapes]
key1 = value1, value2, value3
key2 = value1\, value2
key3 = val\ue1, value2
key4 = value1\\, value\\\\2
key5 = value1\,, value2
key6 = aaa bbb\ and\ space ccc
[advance]
value with quotes = some value
value quote2 again = some value
includes comment sign = `+"`"+"my#password"+"`"+`
includes comment sign2 = `+"`"+"my;password"+"`"+`
true = 2+3=5
`+"`"+`1+1=2`+"`"+` = true
`+"`"+`6+1=7`+"`"+` = true
"""`+"`"+`5+5`+"`"+`""" = 10
`+"`"+`"6+6"`+"`"+` = 12
`+"`"+`7-2=4`+"`"+` = false
ADDRESS = """404 road,
NotFound, State, 50000"""
two_lines = how about continuation lines?
lots_of_lines = 1 2 3 4
`)
expected, err := ioutil.ReadFile(golden)
So(err, ShouldBeNil)
So(buf.String(), ShouldEqual, string(expected))
})
}

2
vendor/github.com/go-ini/ini/ini.go сгенерированный поставляемый
Просмотреть файл

@@ -32,7 +32,7 @@ const (
// Maximum allowed depth when recursively substituing variable names.
_DEPTH_VALUES = 99
_VERSION = "1.32.0"
_VERSION = "1.32.1"
)
// Version returns current package version literal.

3
vendor/github.com/go-ini/ini/ini_test.go сгенерированный поставляемый
Просмотреть файл

@@ -16,6 +16,7 @@ package ini_test
import (
"bytes"
"flag"
"io/ioutil"
"testing"
@@ -47,6 +48,8 @@ const (
_NOT_FOUND_CONF = "testdata/404.ini"
)
var update = flag.Bool("update", false, "Update .golden files")
func TestLoad(t *testing.T) {
Convey("Load from good data sources", t, func() {
f, err := ini.Load([]byte(`

86
vendor/github.com/go-ini/ini/testdata/TestFile_WriteTo.golden сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,86 @@
; Package name
NAME = ini
; Package version
VERSION = v1
; Package import path
IMPORT_PATH = gopkg.in/%(NAME)s.%(VERSION)s
; Information about package author
# Bio can be written in multiple lines.
[author]
; This is author name
NAME = Unknwon
E-MAIL = u@gogs.io
GITHUB = https://github.com/%(NAME)s
# Succeeding comment
BIO = """Gopher.
Coding addict.
Good man.
"""
[package]
CLONE_URL = https://%(IMPORT_PATH)s
[package.sub]
UNUSED_KEY = should be deleted
[features]
- = Support read/write comments of keys and sections
- = Support auto-increment of key names
- = Support load multiple files to overwrite key values
[types]
STRING = str
BOOL = true
BOOL_FALSE = false
FLOAT64 = 1.25
INT = 10
TIME = 2015-01-01T20:17:05Z
DURATION = 2h45m
UINT = 3
[array]
STRINGS = en, zh, de
FLOAT64S = 1.1, 2.2, 3.3
INTS = 1, 2, 3
UINTS = 1, 2, 3
TIMES = 2015-01-01T20:17:05Z,2015-01-01T20:17:05Z,2015-01-01T20:17:05Z
[note]
empty_lines = next line is empty
boolean_key
more = notes
; Comment before the section
; This is a comment for the section too
[comments]
; Comment before key
key = value
; This is a comment for key2
key2 = value2
key3 = "one", "two", "three"
[string escapes]
key1 = value1, value2, value3
key2 = value1\, value2
key3 = val\ue1, value2
key4 = value1\\, value\\\\2
key5 = value1\,, value2
key6 = aaa bbb\ and\ space ccc
[advance]
value with quotes = some value
value quote2 again = some value
includes comment sign = `my#password`
includes comment sign2 = `my;password`
true = 2+3=5
`1+1=2` = true
`6+1=7` = true
"""`5+5`""" = 10
`"6+6"` = 12
`7-2=4` = false
ADDRESS = """404 road,
NotFound, State, 50000"""
two_lines = how about continuation lines?
lots_of_lines = 1 2 3 4

42
vendor/github.com/go-redis/redis/cluster.go сгенерированный поставляемый
Просмотреть файл

@@ -445,6 +445,10 @@ type ClusterClient struct {
cmdsInfoOnce internal.Once
cmdsInfo map[string]*CommandInfo
process func(Cmder) error
processPipeline func([]Cmder) error
processTxPipeline func([]Cmder) error
// Reports whether slots reloading is in progress.
reloading uint32
}
@@ -458,7 +462,12 @@ func NewClusterClient(opt *ClusterOptions) *ClusterClient {
opt: opt,
nodes: newClusterNodes(opt),
}
c.setProcessor(c.Process)
c.process = c.defaultProcess
c.processPipeline = c.defaultProcessPipeline
c.processTxPipeline = c.defaultProcessTxPipeline
c.cmdable.setProcessor(c.Process)
// Add initial nodes.
for _, addr := range opt.Addrs {
@@ -628,7 +637,20 @@ func (c *ClusterClient) Close() error {
return c.nodes.Close()
}
func (c *ClusterClient) WrapProcess(
fn func(oldProcess func(Cmder) error) func(Cmder) error,
) {
c.process = fn(c.process)
}
func (c *ClusterClient) Process(cmd Cmder) error {
if c.process != nil {
return c.process(cmd)
}
return c.defaultProcess(cmd)
}
func (c *ClusterClient) defaultProcess(cmd Cmder) error {
state, err := c.state()
if err != nil {
cmd.setErr(err)
@@ -910,9 +932,9 @@ func (c *ClusterClient) reaper(idleCheckFrequency time.Duration) {
func (c *ClusterClient) Pipeline() Pipeliner {
pipe := Pipeline{
exec: c.pipelineExec,
exec: c.processPipeline,
}
pipe.setProcessor(pipe.Process)
pipe.statefulCmdable.setProcessor(pipe.Process)
return &pipe
}
@@ -920,7 +942,13 @@ func (c *ClusterClient) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) {
return c.Pipeline().Pipelined(fn)
}
func (c *ClusterClient) pipelineExec(cmds []Cmder) error {
func (c *ClusterClient) WrapProcessPipeline(
fn func(oldProcess func([]Cmder) error) func([]Cmder) error,
) {
c.processPipeline = fn(c.processPipeline)
}
func (c *ClusterClient) defaultProcessPipeline(cmds []Cmder) error {
cmdsMap, err := c.mapCmdsByNode(cmds)
if err != nil {
setCmdsErr(cmds, err)
@@ -1064,9 +1092,9 @@ func (c *ClusterClient) checkMovedErr(
// TxPipeline acts like Pipeline, but wraps queued commands with MULTI/EXEC.
func (c *ClusterClient) TxPipeline() Pipeliner {
pipe := Pipeline{
exec: c.txPipelineExec,
exec: c.processTxPipeline,
}
pipe.setProcessor(pipe.Process)
pipe.statefulCmdable.setProcessor(pipe.Process)
return &pipe
}
@@ -1074,7 +1102,7 @@ func (c *ClusterClient) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) {
return c.TxPipeline().Pipelined(fn)
}
func (c *ClusterClient) txPipelineExec(cmds []Cmder) error {
func (c *ClusterClient) defaultProcessTxPipeline(cmds []Cmder) error {
state, err := c.state()
if err != nil {
return err

4
vendor/github.com/go-redis/redis/command.go сгенерированный поставляемый
Просмотреть файл

@@ -81,9 +81,9 @@ func cmdFirstKeyPos(cmd Cmder, info *CommandInfo) int {
case "eval", "evalsha":
if cmd.stringArg(2) != "0" {
return 3
} else {
return 0
}
return 0
case "publish":
return 1
}

77
vendor/github.com/go-redis/redis/example_instrumentation_test.go сгенерированный поставляемый
Просмотреть файл

@@ -2,58 +2,47 @@ package redis_test
import (
"fmt"
"sync/atomic"
"time"
"github.com/go-redis/redis"
)
func Example_instrumentation() {
ring := redis.NewRing(&redis.RingOptions{
Addrs: map[string]string{
"shard1": ":6379",
},
cl := redis.NewClient(&redis.Options{
Addr: ":6379",
})
ring.ForEachShard(func(client *redis.Client) error {
wrapRedisProcess(client)
return nil
})
for {
ring.Ping()
}
}
func wrapRedisProcess(client *redis.Client) {
const precision = time.Microsecond
var count, avgDur uint32
go func() {
for range time.Tick(3 * time.Second) {
n := atomic.LoadUint32(&count)
dur := time.Duration(atomic.LoadUint32(&avgDur)) * precision
fmt.Printf("%s: processed=%d avg_dur=%s\n", client, n, dur)
}
}()
client.WrapProcess(func(oldProcess func(redis.Cmder) error) func(redis.Cmder) error {
cl.WrapProcess(func(old func(cmd redis.Cmder) error) func(cmd redis.Cmder) error {
return func(cmd redis.Cmder) error {
start := time.Now()
err := oldProcess(cmd)
dur := time.Since(start)
const decay = float64(1) / 100
ms := float64(dur / precision)
for {
avg := atomic.LoadUint32(&avgDur)
newAvg := uint32((1-decay)*float64(avg) + decay*ms)
if atomic.CompareAndSwapUint32(&avgDur, avg, newAvg) {
break
}
}
atomic.AddUint32(&count, 1)
fmt.Printf("starting processing: <%s>\n", cmd)
err := old(cmd)
fmt.Printf("finished processing: <%s>\n", cmd)
return err
}
})
cl.Ping()
// Output: starting processing: <ping: >
// finished processing: <ping: PONG>
}
func Example_Pipeline_instrumentation() {
client := redis.NewClient(&redis.Options{
Addr: ":6379",
})
client.WrapProcessPipeline(func(old func([]redis.Cmder) error) func([]redis.Cmder) error {
return func(cmds []redis.Cmder) error {
fmt.Printf("pipeline starting processing: %v\n", cmds)
err := old(cmds)
fmt.Printf("pipeline finished processing: %v\n", cmds)
return err
}
})
client.Pipelined(func(pipe redis.Pipeliner) error {
pipe.Ping()
pipe.Ping()
return nil
})
// Output: pipeline starting processing: [ping: ping: ]
// pipeline finished processing: [ping: PONG ping: PONG]
}

66
vendor/github.com/go-redis/redis/internal/proto/reader.go сгенерированный поставляемый
Просмотреть файл

@@ -37,25 +37,25 @@ func (r *Reader) Reset(rd io.Reader) {
r.src.Reset(rd)
}
func (p *Reader) PeekBuffered() []byte {
if n := p.src.Buffered(); n != 0 {
b, _ := p.src.Peek(n)
func (r *Reader) PeekBuffered() []byte {
if n := r.src.Buffered(); n != 0 {
b, _ := r.src.Peek(n)
return b
}
return nil
}
func (p *Reader) ReadN(n int) ([]byte, error) {
b, err := readN(p.src, p.buf, n)
func (r *Reader) ReadN(n int) ([]byte, error) {
b, err := readN(r.src, r.buf, n)
if err != nil {
return nil, err
}
p.buf = b
r.buf = b
return b, nil
}
func (p *Reader) ReadLine() ([]byte, error) {
line, isPrefix, err := p.src.ReadLine()
func (r *Reader) ReadLine() ([]byte, error) {
line, isPrefix, err := r.src.ReadLine()
if err != nil {
return nil, err
}
@@ -71,8 +71,8 @@ func (p *Reader) ReadLine() ([]byte, error) {
return line, nil
}
func (p *Reader) ReadReply(m MultiBulkParse) (interface{}, error) {
line, err := p.ReadLine()
func (r *Reader) ReadReply(m MultiBulkParse) (interface{}, error) {
line, err := r.ReadLine()
if err != nil {
return nil, err
}
@@ -85,19 +85,19 @@ func (p *Reader) ReadReply(m MultiBulkParse) (interface{}, error) {
case IntReply:
return parseInt(line[1:], 10, 64)
case StringReply:
return p.readTmpBytesValue(line)
return r.readTmpBytesValue(line)
case ArrayReply:
n, err := parseArrayLen(line)
if err != nil {
return nil, err
}
return m(p, n)
return m(r, n)
}
return nil, fmt.Errorf("redis: can't parse %.100q", line)
}
func (p *Reader) ReadIntReply() (int64, error) {
line, err := p.ReadLine()
func (r *Reader) ReadIntReply() (int64, error) {
line, err := r.ReadLine()
if err != nil {
return 0, err
}
@@ -111,8 +111,8 @@ func (p *Reader) ReadIntReply() (int64, error) {
}
}
func (p *Reader) ReadTmpBytesReply() ([]byte, error) {
line, err := p.ReadLine()
func (r *Reader) ReadTmpBytesReply() ([]byte, error) {
line, err := r.ReadLine()
if err != nil {
return nil, err
}
@@ -120,7 +120,7 @@ func (p *Reader) ReadTmpBytesReply() ([]byte, error) {
case ErrorReply:
return nil, ParseErrorReply(line)
case StringReply:
return p.readTmpBytesValue(line)
return r.readTmpBytesValue(line)
case StatusReply:
return parseStatusValue(line), nil
default:
@@ -138,24 +138,24 @@ func (r *Reader) ReadBytesReply() ([]byte, error) {
return cp, nil
}
func (p *Reader) ReadStringReply() (string, error) {
b, err := p.ReadTmpBytesReply()
func (r *Reader) ReadStringReply() (string, error) {
b, err := r.ReadTmpBytesReply()
if err != nil {
return "", err
}
return string(b), nil
}
func (p *Reader) ReadFloatReply() (float64, error) {
b, err := p.ReadTmpBytesReply()
func (r *Reader) ReadFloatReply() (float64, error) {
b, err := r.ReadTmpBytesReply()
if err != nil {
return 0, err
}
return parseFloat(b, 64)
}
func (p *Reader) ReadArrayReply(m MultiBulkParse) (interface{}, error) {
line, err := p.ReadLine()
func (r *Reader) ReadArrayReply(m MultiBulkParse) (interface{}, error) {
line, err := r.ReadLine()
if err != nil {
return nil, err
}
@@ -167,14 +167,14 @@ func (p *Reader) ReadArrayReply(m MultiBulkParse) (interface{}, error) {
if err != nil {
return nil, err
}
return m(p, n)
return m(r, n)
default:
return nil, fmt.Errorf("redis: can't parse array reply: %.100q", line)
}
}
func (p *Reader) ReadArrayLen() (int64, error) {
line, err := p.ReadLine()
func (r *Reader) ReadArrayLen() (int64, error) {
line, err := r.ReadLine()
if err != nil {
return 0, err
}
@@ -188,8 +188,8 @@ func (p *Reader) ReadArrayLen() (int64, error) {
}
}
func (p *Reader) ReadScanReply() ([]string, uint64, error) {
n, err := p.ReadArrayLen()
func (r *Reader) ReadScanReply() ([]string, uint64, error) {
n, err := r.ReadArrayLen()
if err != nil {
return nil, 0, err
}
@@ -197,19 +197,19 @@ func (p *Reader) ReadScanReply() ([]string, uint64, error) {
return nil, 0, fmt.Errorf("redis: got %d elements in scan reply, expected 2", n)
}
cursor, err := p.ReadUint()
cursor, err := r.ReadUint()
if err != nil {
return nil, 0, err
}
n, err = p.ReadArrayLen()
n, err = r.ReadArrayLen()
if err != nil {
return nil, 0, err
}
keys := make([]string, n)
for i := int64(0); i < n; i++ {
key, err := p.ReadStringReply()
key, err := r.ReadStringReply()
if err != nil {
return nil, 0, err
}
@@ -219,7 +219,7 @@ func (p *Reader) ReadScanReply() ([]string, uint64, error) {
return keys, cursor, err
}
func (p *Reader) readTmpBytesValue(line []byte) ([]byte, error) {
func (r *Reader) readTmpBytesValue(line []byte) ([]byte, error) {
if isNilReply(line) {
return nil, internal.Nil
}
@@ -229,7 +229,7 @@ func (p *Reader) readTmpBytesValue(line []byte) ([]byte, error) {
return nil, err
}
b, err := p.ReadN(replyLen + 2)
b, err := r.ReadN(replyLen + 2)
if err != nil {
return nil, err
}

10
vendor/github.com/go-redis/redis/pubsub.go сгенерированный поставляемый
Просмотреть файл

@@ -127,7 +127,7 @@ func (c *PubSub) Close() error {
return nil
}
// Subscribes the client to the specified channels. It returns
// Subscribe the client to the specified channels. It returns
// empty subscription if there are no channels.
func (c *PubSub) Subscribe(channels ...string) error {
c.mu.Lock()
@@ -137,7 +137,7 @@ func (c *PubSub) Subscribe(channels ...string) error {
return err
}
// Subscribes the client to the given patterns. It returns
// PSubscribe the client to the given patterns. It returns
// empty subscription if there are no patterns.
func (c *PubSub) PSubscribe(patterns ...string) error {
c.mu.Lock()
@@ -147,7 +147,7 @@ func (c *PubSub) PSubscribe(patterns ...string) error {
return err
}
// Unsubscribes the client from the given channels, or from all of
// Unsubscribe the client from the given channels, or from all of
// them if none is given.
func (c *PubSub) Unsubscribe(channels ...string) error {
c.mu.Lock()
@@ -157,7 +157,7 @@ func (c *PubSub) Unsubscribe(channels ...string) error {
return err
}
// Unsubscribes the client from the given patterns, or from all of
// PUnsubscribe the client from the given patterns, or from all of
// them if none is given.
func (c *PubSub) PUnsubscribe(patterns ...string) error {
c.mu.Lock()
@@ -196,7 +196,7 @@ func (c *PubSub) Ping(payload ...string) error {
return err
}
// Message received after a successful subscription to channel.
// Subscription received after a successful subscription to channel.
type Subscription struct {
// Can be "subscribe", "unsubscribe", "psubscribe" or "punsubscribe".
Kind string

110
vendor/github.com/go-redis/redis/redis.go сгенерированный поставляемый
Просмотреть файл

@@ -11,7 +11,7 @@ import (
"github.com/go-redis/redis/internal/proto"
)
// Redis nil reply returned when key does not exist.
// Nil reply redis returned when key does not exist.
const Nil = internal.Nil
func init() {
@@ -22,6 +22,12 @@ func SetLogger(logger *log.Logger) {
internal.Logger = logger
}
func (c *baseClient) init() {
c.process = c.defaultProcess
c.processPipeline = c.defaultProcessPipeline
c.processTxPipeline = c.defaultProcessTxPipeline
}
func (c *baseClient) String() string {
return fmt.Sprintf("Redis<%s db:%d>", c.getAddr(), c.opt.DB)
}
@@ -85,7 +91,8 @@ func (c *baseClient) initConn(cn *pool.Conn) error {
connPool: pool.NewSingleConnPool(cn),
},
}
conn.setProcessor(conn.Process)
conn.baseClient.init()
conn.statefulCmdable.setProcessor(conn.Process)
_, err := conn.Pipelined(func(pipe Pipeliner) error {
if c.opt.Password != "" {
@@ -117,14 +124,11 @@ func (c *baseClient) initConn(cn *pool.Conn) error {
// an input and returns the new wrapper process func. createWrapper should
// use call the old process func within the new process func.
func (c *baseClient) WrapProcess(fn func(oldProcess func(cmd Cmder) error) func(cmd Cmder) error) {
c.process = fn(c.defaultProcess)
c.process = fn(c.process)
}
func (c *baseClient) Process(cmd Cmder) error {
if c.process != nil {
return c.process(cmd)
}
return c.defaultProcess(cmd)
return c.process(cmd)
}
func (c *baseClient) defaultProcess(cmd Cmder) error {
@@ -172,9 +176,9 @@ func (c *baseClient) retryBackoff(attempt int) time.Duration {
func (c *baseClient) cmdTimeout(cmd Cmder) time.Duration {
if timeout := cmd.readTimeout(); timeout != nil {
return *timeout
} else {
return c.opt.ReadTimeout
}
return c.opt.ReadTimeout
}
// Close closes the client, releasing any open resources.
@@ -198,35 +202,48 @@ func (c *baseClient) getAddr() string {
return c.opt.Addr
}
func (c *baseClient) WrapProcessPipeline(
fn func(oldProcess func([]Cmder) error) func([]Cmder) error,
) {
c.processPipeline = fn(c.processPipeline)
c.processTxPipeline = fn(c.processTxPipeline)
}
func (c *baseClient) defaultProcessPipeline(cmds []Cmder) error {
return c.generalProcessPipeline(cmds, c.pipelineProcessCmds)
}
func (c *baseClient) defaultProcessTxPipeline(cmds []Cmder) error {
return c.generalProcessPipeline(cmds, c.txPipelineProcessCmds)
}
type pipelineProcessor func(*pool.Conn, []Cmder) (bool, error)
func (c *baseClient) pipelineExecer(p pipelineProcessor) pipelineExecer {
return func(cmds []Cmder) error {
for attempt := 0; attempt <= c.opt.MaxRetries; attempt++ {
if attempt > 0 {
time.Sleep(c.retryBackoff(attempt))
}
cn, _, err := c.getConn()
if err != nil {
setCmdsErr(cmds, err)
return err
}
canRetry, err := p(cn, cmds)
if err == nil || internal.IsRedisError(err) {
_ = c.connPool.Put(cn)
break
}
_ = c.connPool.Remove(cn)
if !canRetry || !internal.IsRetryableError(err, true) {
break
}
func (c *baseClient) generalProcessPipeline(cmds []Cmder, p pipelineProcessor) error {
for attempt := 0; attempt <= c.opt.MaxRetries; attempt++ {
if attempt > 0 {
time.Sleep(c.retryBackoff(attempt))
}
cn, _, err := c.getConn()
if err != nil {
setCmdsErr(cmds, err)
return err
}
canRetry, err := p(cn, cmds)
if err == nil || internal.IsRedisError(err) {
_ = c.connPool.Put(cn)
break
}
_ = c.connPool.Remove(cn)
if !canRetry || !internal.IsRetryableError(err, true) {
break
}
return firstCmdsErr(cmds)
}
return firstCmdsErr(cmds)
}
func (c *baseClient) pipelineProcessCmds(cn *pool.Conn, cmds []Cmder) (bool, error) {
@@ -324,14 +341,15 @@ type Client struct {
}
func newClient(opt *Options, pool pool.Pooler) *Client {
client := Client{
c := Client{
baseClient: baseClient{
opt: opt,
connPool: pool,
},
}
client.setProcessor(client.Process)
return &client
c.baseClient.init()
c.cmdable.setProcessor(c.Process)
return &c
}
// NewClient returns a client to the Redis Server specified by Options.
@@ -343,7 +361,7 @@ func NewClient(opt *Options) *Client {
func (c *Client) copy() *Client {
c2 := new(Client)
*c2 = *c
c2.setProcessor(c2.Process)
c2.cmdable.setProcessor(c2.Process)
return c2
}
@@ -366,9 +384,9 @@ func (c *Client) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) {
func (c *Client) Pipeline() Pipeliner {
pipe := Pipeline{
exec: c.pipelineExecer(c.pipelineProcessCmds),
exec: c.processPipeline,
}
pipe.setProcessor(pipe.Process)
pipe.statefulCmdable.setProcessor(pipe.Process)
return &pipe
}
@@ -379,9 +397,9 @@ func (c *Client) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) {
// TxPipeline acts like Pipeline, but wraps queued commands with MULTI/EXEC.
func (c *Client) TxPipeline() Pipeliner {
pipe := Pipeline{
exec: c.pipelineExecer(c.txPipelineProcessCmds),
exec: c.processTxPipeline,
}
pipe.setProcessor(pipe.Process)
pipe.statefulCmdable.setProcessor(pipe.Process)
return &pipe
}
@@ -430,9 +448,9 @@ func (c *Conn) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) {
func (c *Conn) Pipeline() Pipeliner {
pipe := Pipeline{
exec: c.pipelineExecer(c.pipelineProcessCmds),
exec: c.processPipeline,
}
pipe.setProcessor(pipe.Process)
pipe.statefulCmdable.setProcessor(pipe.Process)
return &pipe
}
@@ -443,8 +461,8 @@ func (c *Conn) TxPipelined(fn func(Pipeliner) error) ([]Cmder, error) {
// TxPipeline acts like Pipeline, but wraps queued commands with MULTI/EXEC.
func (c *Conn) TxPipeline() Pipeliner {
pipe := Pipeline{
exec: c.pipelineExecer(c.txPipelineProcessCmds),
exec: c.processTxPipeline,
}
pipe.setProcessor(pipe.Process)
pipe.statefulCmdable.setProcessor(pipe.Process)
return &pipe
}

5
vendor/github.com/go-redis/redis/redis_context.go сгенерированный поставляемый
Просмотреть файл

@@ -12,7 +12,10 @@ type baseClient struct {
connPool pool.Pooler
opt *Options
process func(Cmder) error
process func(Cmder) error
processPipeline func([]Cmder) error
processTxPipeline func([]Cmder) error
onClose func() error // hook called when client is closed
ctx context.Context

5
vendor/github.com/go-redis/redis/redis_no_context.go сгенерированный поставляемый
Просмотреть файл

@@ -10,6 +10,9 @@ type baseClient struct {
connPool pool.Pooler
opt *Options
process func(Cmder) error
process func(Cmder) error
processPipeline func([]Cmder) error
processTxPipeline func([]Cmder) error
onClose func() error // hook called when client is closed
}

29
vendor/github.com/go-redis/redis/ring.go сгенерированный поставляемый
Просмотреть файл

@@ -150,6 +150,8 @@ type Ring struct {
shards map[string]*ringShard
shardsList []*ringShard
processPipeline func([]Cmder) error
cmdsInfoOnce internal.Once
cmdsInfo map[string]*CommandInfo
@@ -158,7 +160,9 @@ type Ring struct {
func NewRing(opt *RingOptions) *Ring {
const nreplicas = 100
opt.init()
ring := &Ring{
opt: opt,
nreplicas: nreplicas,
@@ -166,13 +170,17 @@ func NewRing(opt *RingOptions) *Ring {
hash: consistenthash.New(nreplicas, nil),
shards: make(map[string]*ringShard),
}
ring.setProcessor(ring.Process)
ring.processPipeline = ring.defaultProcessPipeline
ring.cmdable.setProcessor(ring.Process)
for name, addr := range opt.Addrs {
clopt := opt.clientOptions()
clopt.Addr = addr
ring.addShard(name, NewClient(clopt))
}
go ring.heartbeat()
return ring
}
@@ -354,6 +362,13 @@ func (c *Ring) cmdShard(cmd Cmder) (*ringShard, error) {
return c.shardByKey(firstKey)
}
func (c *Ring) WrapProcess(fn func(oldProcess func(cmd Cmder) error) func(cmd Cmder) error) {
c.ForEachShard(func(c *Client) error {
c.WrapProcess(fn)
return nil
})
}
func (c *Ring) Process(cmd Cmder) error {
shard, err := c.cmdShard(cmd)
if err != nil {
@@ -436,9 +451,9 @@ func (c *Ring) Close() error {
func (c *Ring) Pipeline() Pipeliner {
pipe := Pipeline{
exec: c.pipelineExec,
exec: c.processPipeline,
}
pipe.setProcessor(pipe.Process)
pipe.cmdable.setProcessor(pipe.Process)
return &pipe
}
@@ -446,7 +461,13 @@ func (c *Ring) Pipelined(fn func(Pipeliner) error) ([]Cmder, error) {
return c.Pipeline().Pipelined(fn)
}
func (c *Ring) pipelineExec(cmds []Cmder) error {
func (c *Ring) WrapProcessPipeline(
fn func(oldProcess func([]Cmder) error) func([]Cmder) error,
) {
c.processPipeline = fn(c.processPipeline)
}
func (c *Ring) defaultProcessPipeline(cmds []Cmder) error {
cmdsMap := make(map[string][]Cmder)
for _, cmd := range cmds {
cmdInfo := c.cmdInfo(cmd.Name())

14
vendor/github.com/go-redis/redis/sentinel.go сгенерированный поставляемый
Просмотреть файл

@@ -76,7 +76,7 @@ func NewFailoverClient(failoverOpt *FailoverOptions) *Client {
opt: opt,
}
client := Client{
c := Client{
baseClient: baseClient{
opt: opt,
connPool: failover.Pool(),
@@ -86,9 +86,10 @@ func NewFailoverClient(failoverOpt *FailoverOptions) *Client {
},
},
}
client.setProcessor(client.Process)
c.baseClient.init()
c.setProcessor(c.Process)
return &client
return &c
}
//------------------------------------------------------------------------------
@@ -100,14 +101,15 @@ type sentinelClient struct {
func newSentinel(opt *Options) *sentinelClient {
opt.init()
client := sentinelClient{
c := sentinelClient{
baseClient: baseClient{
opt: opt,
connPool: newConnPool(opt),
},
}
client.cmdable = cmdable{client.Process}
return &client
c.baseClient.init()
c.cmdable.setProcessor(c.Process)
return &c
}
func (c *sentinelClient) PubSub() *PubSub {

11
vendor/github.com/go-redis/redis/tx.go сгенерированный поставляемый
Просмотреть файл

@@ -5,7 +5,7 @@ import (
"github.com/go-redis/redis/internal/pool"
)
// Redis transaction failed.
// TxFailedErr transaction redis failed.
const TxFailedErr = internal.RedisError("redis: transaction failed")
// Tx implements Redis transactions as described in
@@ -24,7 +24,8 @@ func (c *Client) newTx() *Tx {
connPool: pool.NewStickyConnPool(c.connPool.(*pool.ConnPool), true),
},
}
tx.setProcessor(tx.Process)
tx.baseClient.init()
tx.statefulCmdable.setProcessor(tx.Process)
return &tx
}
@@ -42,7 +43,7 @@ func (c *Client) Watch(fn func(*Tx) error, keys ...string) error {
return err
}
// close closes the transaction, releasing any open resources.
// Close closes the transaction, releasing any open resources.
func (c *Tx) Close() error {
_ = c.Unwatch().Err()
return c.baseClient.Close()
@@ -75,9 +76,9 @@ func (c *Tx) Unwatch(keys ...string) *StatusCmd {
func (c *Tx) Pipeline() Pipeliner {
pipe := Pipeline{
exec: c.pipelineExecer(c.txPipelineProcessCmds),
exec: c.processTxPipeline,
}
pipe.setProcessor(pipe.Process)
pipe.statefulCmdable.setProcessor(pipe.Process)
return &pipe
}

2
vendor/github.com/golang/protobuf/proto/extensions_test.go сгенерированный поставляемый
Просмотреть файл

@@ -478,7 +478,7 @@ func TestUnmarshalRepeatingNonRepeatedExtension(t *testing.T) {
t.Fatalf("[%s] Invalid extension", test.name)
}
if !reflect.DeepEqual(*ext, want) {
t.Errorf("[%s] Wrong value for ComplexExtension: got: %s want: %s\n", test.name, ext, want)
t.Errorf("[%s] Wrong value for ComplexExtension: got: %s want: %s\n", test.name, ext, &want)
}
}
}

6
vendor/github.com/golang/protobuf/protoc-gen-go/generator/generator.go сгенерированный поставляемый
Просмотреть файл

@@ -2029,7 +2029,11 @@ func (g *Generator) generateMessage(message *Descriptor) {
// TODO: Revisit this and consider reverting back to anonymous interfaces.
for oi := range message.OneofDecl {
dname := oneofDisc[int32(oi)]
g.P("type ", dname, " interface { ", dname, "() }")
g.P("type ", dname, " interface {")
g.In()
g.P(dname, "()")
g.Out()
g.P("}")
}
g.P()
for _, field := range message.Field {

18
vendor/github.com/gorilla/schema/.travis.yml сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,18 @@
language: go
sudo: false
matrix:
include:
- go: 1.5
- go: 1.6
- go: 1.7
- go: 1.8
- go: tip
allow_failures:
- go: tip
script:
- go get -t -v ./...
- diff -u <(echo -n) <(gofmt -d .)
- go vet $(go list ./... | grep -v /vendor/)
- go test -v -race ./...

8
vendor/github.com/rsc/letsencrypt/LICENSE → vendor/github.com/gorilla/schema/LICENSE сгенерированный поставляемый
Просмотреть файл

@@ -1,16 +1,16 @@
Copyright (c) 2009 The Go Authors. All rights reserved.
Copyright (c) 2012 Rodrigo Moraes. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.

90
vendor/github.com/gorilla/schema/README.md сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,90 @@
schema
======
[![GoDoc](https://godoc.org/github.com/gorilla/schema?status.svg)](https://godoc.org/github.com/gorilla/schema) [![Build Status](https://travis-ci.org/gorilla/schema.png?branch=master)](https://travis-ci.org/gorilla/schema)
[![Sourcegraph](https://sourcegraph.com/github.com/gorilla/schema/-/badge.svg)](https://sourcegraph.com/github.com/gorilla/schema?badge)
Package gorilla/schema converts structs to and from form values.
## Example
Here's a quick example: we parse POST form values and then decode them into a struct:
```go
// Set a Decoder instance as a package global, because it caches
// meta-data about structs, and an instance can be shared safely.
var decoder = schema.NewDecoder()
type Person struct {
Name string
Phone string
}
func MyHandler(w http.ResponseWriter, r *http.Request) {
err := r.ParseForm()
if err != nil {
// Handle error
}
var person Person
// r.PostForm is a map of our POST form values
err := decoder.Decode(&person, r.PostForm)
if err != nil {
// Handle error
}
// Do something with person.Name or person.Phone
}
```
Conversely, contents of a struct can be encoded into form values. Here's a variant of the previous example using the Encoder:
```go
var encoder = schema.NewEncoder()
func MyHttpRequest() {
person := Person{"Jane Doe", "555-5555"}
form := url.Values{}
err := encoder.Encode(person, form)
if err != nil {
// Handle error
}
// Use form values, for example, with an http client
client := new(http.Client)
res, err := client.PostForm("http://my-api.test", form)
}
```
To define custom names for fields, use a struct tag "schema". To not populate certain fields, use a dash for the name and it will be ignored:
```go
type Person struct {
Name string `schema:"name"` // custom name
Phone string `schema:"phone"` // custom name
Admin bool `schema:"-"` // this field is never set
}
```
The supported field types in the struct are:
* bool
* float variants (float32, float64)
* int variants (int, int8, int16, int32, int64)
* string
* uint variants (uint, uint8, uint16, uint32, uint64)
* struct
* a pointer to one of the above types
* a slice or a pointer to a slice of one of the above types
Unsupported types are simply ignored, however custom types can be registered to be converted.
More examples are available on the Gorilla website: http://www.gorillatoolkit.org/pkg/schema
## License
BSD licensed. See the LICENSE file for details.

264
vendor/github.com/gorilla/schema/cache.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,264 @@
// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package schema
import (
"errors"
"reflect"
"strconv"
"strings"
"sync"
)
var invalidPath = errors.New("schema: invalid path")
// newCache returns a new cache.
func newCache() *cache {
c := cache{
m: make(map[reflect.Type]*structInfo),
regconv: make(map[reflect.Type]Converter),
tag: "schema",
}
return &c
}
// cache caches meta-data about a struct.
type cache struct {
l sync.RWMutex
m map[reflect.Type]*structInfo
regconv map[reflect.Type]Converter
tag string
}
// registerConverter registers a converter function for a custom type.
func (c *cache) registerConverter(value interface{}, converterFunc Converter) {
c.regconv[reflect.TypeOf(value)] = converterFunc
}
// parsePath parses a path in dotted notation verifying that it is a valid
// path to a struct field.
//
// It returns "path parts" which contain indices to fields to be used by
// reflect.Value.FieldByString(). Multiple parts are required for slices of
// structs.
func (c *cache) parsePath(p string, t reflect.Type) ([]pathPart, error) {
var struc *structInfo
var field *fieldInfo
var index64 int64
var err error
parts := make([]pathPart, 0)
path := make([]string, 0)
keys := strings.Split(p, ".")
for i := 0; i < len(keys); i++ {
if t.Kind() != reflect.Struct {
return nil, invalidPath
}
if struc = c.get(t); struc == nil {
return nil, invalidPath
}
if field = struc.get(keys[i]); field == nil {
return nil, invalidPath
}
// Valid field. Append index.
path = append(path, field.name)
if field.ss {
// Parse a special case: slices of structs.
// i+1 must be the slice index.
//
// Now that struct can implements TextUnmarshaler interface,
// we don't need to force the struct's fields to appear in the path.
// So checking i+2 is not necessary anymore.
i++
if i+1 > len(keys) {
return nil, invalidPath
}
if index64, err = strconv.ParseInt(keys[i], 10, 0); err != nil {
return nil, invalidPath
}
parts = append(parts, pathPart{
path: path,
field: field,
index: int(index64),
})
path = make([]string, 0)
// Get the next struct type, dropping ptrs.
if field.typ.Kind() == reflect.Ptr {
t = field.typ.Elem()
} else {
t = field.typ
}
if t.Kind() == reflect.Slice {
t = t.Elem()
if t.Kind() == reflect.Ptr {
t = t.Elem()
}
}
} else if field.typ.Kind() == reflect.Ptr {
t = field.typ.Elem()
} else {
t = field.typ
}
}
// Add the remaining.
parts = append(parts, pathPart{
path: path,
field: field,
index: -1,
})
return parts, nil
}
// get returns a cached structInfo, creating it if necessary.
func (c *cache) get(t reflect.Type) *structInfo {
c.l.RLock()
info := c.m[t]
c.l.RUnlock()
if info == nil {
info = c.create(t, nil)
c.l.Lock()
c.m[t] = info
c.l.Unlock()
}
return info
}
// create creates a structInfo with meta-data about a struct.
func (c *cache) create(t reflect.Type, info *structInfo) *structInfo {
if info == nil {
info = &structInfo{fields: []*fieldInfo{}}
}
for i := 0; i < t.NumField(); i++ {
field := t.Field(i)
if field.Anonymous {
ft := field.Type
if ft.Kind() == reflect.Ptr {
ft = ft.Elem()
}
if ft.Kind() == reflect.Struct {
bef := len(info.fields)
c.create(ft, info)
for _, fi := range info.fields[bef:len(info.fields)] {
// exclude required check because duplicated to embedded field
fi.required = false
}
}
}
c.createField(field, info)
}
return info
}
// createField creates a fieldInfo for the given field.
func (c *cache) createField(field reflect.StructField, info *structInfo) {
alias, options := fieldAlias(field, c.tag)
if alias == "-" {
// Ignore this field.
return
}
// Check if the type is supported and don't cache it if not.
// First let's get the basic type.
isSlice, isStruct := false, false
ft := field.Type
if ft.Kind() == reflect.Ptr {
ft = ft.Elem()
}
if isSlice = ft.Kind() == reflect.Slice; isSlice {
ft = ft.Elem()
if ft.Kind() == reflect.Ptr {
ft = ft.Elem()
}
}
if ft.Kind() == reflect.Array {
ft = ft.Elem()
if ft.Kind() == reflect.Ptr {
ft = ft.Elem()
}
}
if isStruct = ft.Kind() == reflect.Struct; !isStruct {
if c.converter(ft) == nil && builtinConverters[ft.Kind()] == nil {
// Type is not supported.
return
}
}
info.fields = append(info.fields, &fieldInfo{
typ: field.Type,
name: field.Name,
ss: isSlice && isStruct,
alias: alias,
anon: field.Anonymous,
required: options.Contains("required"),
})
}
// converter returns the converter for a type.
func (c *cache) converter(t reflect.Type) Converter {
return c.regconv[t]
}
// ----------------------------------------------------------------------------
type structInfo struct {
fields []*fieldInfo
}
func (i *structInfo) get(alias string) *fieldInfo {
for _, field := range i.fields {
if strings.EqualFold(field.alias, alias) {
return field
}
}
return nil
}
type fieldInfo struct {
typ reflect.Type
name string // field name in the struct.
ss bool // true if this is a slice of structs.
alias string
anon bool // is an embedded field
required bool // tag option
}
type pathPart struct {
field *fieldInfo
path []string // path to the field: walks structs using field names.
index int // struct index in slices of structs.
}
// ----------------------------------------------------------------------------
// fieldAlias parses a field tag to get a field alias.
func fieldAlias(field reflect.StructField, tagName string) (alias string, options tagOptions) {
if tag := field.Tag.Get(tagName); tag != "" {
alias, options = parseTag(tag)
}
if alias == "" {
alias = field.Name
}
return alias, options
}
// tagOptions is the string following a comma in a struct field's tag, or
// the empty string. It does not include the leading comma.
type tagOptions []string
// parseTag splits a struct field's url tag into its name and comma-separated
// options.
func parseTag(tag string) (string, tagOptions) {
s := strings.Split(tag, ",")
return s[0], s[1:]
}
// Contains checks whether the tagOptions contains the specified option.
func (o tagOptions) Contains(option string) bool {
for _, s := range o {
if s == option {
return true
}
}
return false
}

145
vendor/github.com/gorilla/schema/converter.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,145 @@
// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package schema
import (
"reflect"
"strconv"
)
type Converter func(string) reflect.Value
var (
invalidValue = reflect.Value{}
boolType = reflect.Bool
float32Type = reflect.Float32
float64Type = reflect.Float64
intType = reflect.Int
int8Type = reflect.Int8
int16Type = reflect.Int16
int32Type = reflect.Int32
int64Type = reflect.Int64
stringType = reflect.String
uintType = reflect.Uint
uint8Type = reflect.Uint8
uint16Type = reflect.Uint16
uint32Type = reflect.Uint32
uint64Type = reflect.Uint64
)
// Default converters for basic types.
var builtinConverters = map[reflect.Kind]Converter{
boolType: convertBool,
float32Type: convertFloat32,
float64Type: convertFloat64,
intType: convertInt,
int8Type: convertInt8,
int16Type: convertInt16,
int32Type: convertInt32,
int64Type: convertInt64,
stringType: convertString,
uintType: convertUint,
uint8Type: convertUint8,
uint16Type: convertUint16,
uint32Type: convertUint32,
uint64Type: convertUint64,
}
func convertBool(value string) reflect.Value {
if value == "on" {
return reflect.ValueOf(true)
} else if v, err := strconv.ParseBool(value); err == nil {
return reflect.ValueOf(v)
}
return invalidValue
}
func convertFloat32(value string) reflect.Value {
if v, err := strconv.ParseFloat(value, 32); err == nil {
return reflect.ValueOf(float32(v))
}
return invalidValue
}
func convertFloat64(value string) reflect.Value {
if v, err := strconv.ParseFloat(value, 64); err == nil {
return reflect.ValueOf(v)
}
return invalidValue
}
func convertInt(value string) reflect.Value {
if v, err := strconv.ParseInt(value, 10, 0); err == nil {
return reflect.ValueOf(int(v))
}
return invalidValue
}
func convertInt8(value string) reflect.Value {
if v, err := strconv.ParseInt(value, 10, 8); err == nil {
return reflect.ValueOf(int8(v))
}
return invalidValue
}
func convertInt16(value string) reflect.Value {
if v, err := strconv.ParseInt(value, 10, 16); err == nil {
return reflect.ValueOf(int16(v))
}
return invalidValue
}
func convertInt32(value string) reflect.Value {
if v, err := strconv.ParseInt(value, 10, 32); err == nil {
return reflect.ValueOf(int32(v))
}
return invalidValue
}
func convertInt64(value string) reflect.Value {
if v, err := strconv.ParseInt(value, 10, 64); err == nil {
return reflect.ValueOf(v)
}
return invalidValue
}
func convertString(value string) reflect.Value {
return reflect.ValueOf(value)
}
func convertUint(value string) reflect.Value {
if v, err := strconv.ParseUint(value, 10, 0); err == nil {
return reflect.ValueOf(uint(v))
}
return invalidValue
}
func convertUint8(value string) reflect.Value {
if v, err := strconv.ParseUint(value, 10, 8); err == nil {
return reflect.ValueOf(uint8(v))
}
return invalidValue
}
func convertUint16(value string) reflect.Value {
if v, err := strconv.ParseUint(value, 10, 16); err == nil {
return reflect.ValueOf(uint16(v))
}
return invalidValue
}
func convertUint32(value string) reflect.Value {
if v, err := strconv.ParseUint(value, 10, 32); err == nil {
return reflect.ValueOf(uint32(v))
}
return invalidValue
}
func convertUint64(value string) reflect.Value {
if v, err := strconv.ParseUint(value, 10, 64); err == nil {
return reflect.ValueOf(v)
}
return invalidValue
}

420
vendor/github.com/gorilla/schema/decoder.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,420 @@
// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package schema
import (
"encoding"
"errors"
"fmt"
"reflect"
"strings"
)
// NewDecoder returns a new Decoder.
func NewDecoder() *Decoder {
return &Decoder{cache: newCache()}
}
// Decoder decodes values from a map[string][]string to a struct.
type Decoder struct {
cache *cache
zeroEmpty bool
ignoreUnknownKeys bool
}
// SetAliasTag changes the tag used to locate custom field aliases.
// The default tag is "schema".
func (d *Decoder) SetAliasTag(tag string) {
d.cache.tag = tag
}
// ZeroEmpty controls the behaviour when the decoder encounters empty values
// in a map.
// If z is true and a key in the map has the empty string as a value
// then the corresponding struct field is set to the zero value.
// If z is false then empty strings are ignored.
//
// The default value is false, that is empty values do not change
// the value of the struct field.
func (d *Decoder) ZeroEmpty(z bool) {
d.zeroEmpty = z
}
// IgnoreUnknownKeys controls the behaviour when the decoder encounters unknown
// keys in the map.
// If i is true and an unknown field is encountered, it is ignored. This is
// similar to how unknown keys are handled by encoding/json.
// If i is false then Decode will return an error. Note that any valid keys
// will still be decoded in to the target struct.
//
// To preserve backwards compatibility, the default value is false.
func (d *Decoder) IgnoreUnknownKeys(i bool) {
d.ignoreUnknownKeys = i
}
// RegisterConverter registers a converter function for a custom type.
func (d *Decoder) RegisterConverter(value interface{}, converterFunc Converter) {
d.cache.registerConverter(value, converterFunc)
}
// Decode decodes a map[string][]string to a struct.
//
// The first parameter must be a pointer to a struct.
//
// The second parameter is a map, typically url.Values from an HTTP request.
// Keys are "paths" in dotted notation to the struct fields and nested structs.
//
// See the package documentation for a full explanation of the mechanics.
func (d *Decoder) Decode(dst interface{}, src map[string][]string) error {
v := reflect.ValueOf(dst)
if v.Kind() != reflect.Ptr || v.Elem().Kind() != reflect.Struct {
return errors.New("schema: interface must be a pointer to struct")
}
v = v.Elem()
t := v.Type()
errors := MultiError{}
for path, values := range src {
if parts, err := d.cache.parsePath(path, t); err == nil {
if err = d.decode(v, path, parts, values); err != nil {
errors[path] = err
}
} else if !d.ignoreUnknownKeys {
errors[path] = fmt.Errorf("schema: invalid path %q", path)
}
}
if len(errors) > 0 {
return errors
}
return d.checkRequired(t, src, "")
}
// checkRequired checks whether required fields are empty
//
// check type t recursively if t has struct fields, and prefix is same as parsePath: in dotted notation
//
// src is the source map for decoding, we use it here to see if those required fields are included in src
func (d *Decoder) checkRequired(t reflect.Type, src map[string][]string, prefix string) error {
struc := d.cache.get(t)
if struc == nil {
// unexpect, cache.get never return nil
return errors.New("cache fail")
}
for _, f := range struc.fields {
if f.typ.Kind() == reflect.Struct {
err := d.checkRequired(f.typ, src, prefix+f.alias+".")
if err != nil {
if !f.anon {
return err
}
// check embedded parent field.
err2 := d.checkRequired(f.typ, src, prefix)
if err2 != nil {
return err
}
}
}
if f.required {
key := f.alias
if prefix != "" {
key = prefix + key
}
if isEmpty(f.typ, src[key]) {
return fmt.Errorf("%v is empty", key)
}
}
}
return nil
}
// isEmpty returns true if value is empty for specific type
func isEmpty(t reflect.Type, value []string) bool {
if len(value) == 0 {
return true
}
switch t.Kind() {
case boolType, float32Type, float64Type, intType, int8Type, int32Type, int64Type, stringType, uint8Type, uint16Type, uint32Type, uint64Type:
return len(value[0]) == 0
}
return false
}
// decode fills a struct field using a parsed path.
func (d *Decoder) decode(v reflect.Value, path string, parts []pathPart, values []string) error {
// Get the field walking the struct fields by index.
for _, name := range parts[0].path {
if v.Type().Kind() == reflect.Ptr {
if v.IsNil() {
v.Set(reflect.New(v.Type().Elem()))
}
v = v.Elem()
}
v = v.FieldByName(name)
}
// Don't even bother for unexported fields.
if !v.CanSet() {
return nil
}
// Dereference if needed.
t := v.Type()
if t.Kind() == reflect.Ptr {
t = t.Elem()
if v.IsNil() {
v.Set(reflect.New(t))
}
v = v.Elem()
}
// Slice of structs. Let's go recursive.
if len(parts) > 1 {
idx := parts[0].index
if v.IsNil() || v.Len() < idx+1 {
value := reflect.MakeSlice(t, idx+1, idx+1)
if v.Len() < idx+1 {
// Resize it.
reflect.Copy(value, v)
}
v.Set(value)
}
return d.decode(v.Index(idx), path, parts[1:], values)
}
// Get the converter early in case there is one for a slice type.
conv := d.cache.converter(t)
m := isTextUnmarshaler(v)
if conv == nil && t.Kind() == reflect.Slice && m.IsSlice {
var items []reflect.Value
elemT := t.Elem()
isPtrElem := elemT.Kind() == reflect.Ptr
if isPtrElem {
elemT = elemT.Elem()
}
// Try to get a converter for the element type.
conv := d.cache.converter(elemT)
if conv == nil {
conv = builtinConverters[elemT.Kind()]
if conv == nil {
// As we are not dealing with slice of structs here, we don't need to check if the type
// implements TextUnmarshaler interface
return fmt.Errorf("schema: converter not found for %v", elemT)
}
}
for key, value := range values {
if value == "" {
if d.zeroEmpty {
items = append(items, reflect.Zero(elemT))
}
} else if m.IsValid {
u := reflect.New(elemT)
if m.IsPtr {
u = reflect.New(reflect.PtrTo(elemT).Elem())
}
if err := u.Interface().(encoding.TextUnmarshaler).UnmarshalText([]byte(value)); err != nil {
return ConversionError{
Key: path,
Type: t,
Index: key,
Err: err,
}
}
if m.IsPtr {
items = append(items, u.Elem().Addr())
} else if u.Kind() == reflect.Ptr {
items = append(items, u.Elem())
} else {
items = append(items, u)
}
} else if item := conv(value); item.IsValid() {
if isPtrElem {
ptr := reflect.New(elemT)
ptr.Elem().Set(item)
item = ptr
}
if item.Type() != elemT && !isPtrElem {
item = item.Convert(elemT)
}
items = append(items, item)
} else {
if strings.Contains(value, ",") {
values := strings.Split(value, ",")
for _, value := range values {
if value == "" {
if d.zeroEmpty {
items = append(items, reflect.Zero(elemT))
}
} else if item := conv(value); item.IsValid() {
if isPtrElem {
ptr := reflect.New(elemT)
ptr.Elem().Set(item)
item = ptr
}
if item.Type() != elemT && !isPtrElem {
item = item.Convert(elemT)
}
items = append(items, item)
} else {
return ConversionError{
Key: path,
Type: elemT,
Index: key,
}
}
}
} else {
return ConversionError{
Key: path,
Type: elemT,
Index: key,
}
}
}
}
value := reflect.Append(reflect.MakeSlice(t, 0, 0), items...)
v.Set(value)
} else {
val := ""
// Use the last value provided if any values were provided
if len(values) > 0 {
val = values[len(values)-1]
}
if val == "" {
if d.zeroEmpty {
v.Set(reflect.Zero(t))
}
} else if conv != nil {
if value := conv(val); value.IsValid() {
v.Set(value.Convert(t))
} else {
return ConversionError{
Key: path,
Type: t,
Index: -1,
}
}
} else if m.IsValid {
// If the value implements the encoding.TextUnmarshaler interface
// apply UnmarshalText as the converter
if err := m.Unmarshaler.UnmarshalText([]byte(val)); err != nil {
return ConversionError{
Key: path,
Type: t,
Index: -1,
Err: err,
}
}
} else if conv := builtinConverters[t.Kind()]; conv != nil {
if value := conv(val); value.IsValid() {
v.Set(value.Convert(t))
} else {
return ConversionError{
Key: path,
Type: t,
Index: -1,
}
}
} else {
return fmt.Errorf("schema: converter not found for %v", t)
}
}
return nil
}
func isTextUnmarshaler(v reflect.Value) unmarshaler {
// Create a new unmarshaller instance
m := unmarshaler{}
// As the UnmarshalText function should be applied
// to the pointer of the type, we convert the value to pointer.
if v.CanAddr() {
v = v.Addr()
}
if m.Unmarshaler, m.IsValid = v.Interface().(encoding.TextUnmarshaler); m.IsValid {
return m
}
// if v is []T or *[]T create new T
t := v.Type()
if t.Kind() == reflect.Ptr {
t = t.Elem()
}
if t.Kind() == reflect.Slice {
// if t is a pointer slice, check if it implements encoding.TextUnmarshaler
m.IsSlice = true
if t = t.Elem(); t.Kind() == reflect.Ptr {
t = reflect.PtrTo(t.Elem())
v = reflect.Zero(t)
m.IsPtr = true
m.Unmarshaler, m.IsValid = v.Interface().(encoding.TextUnmarshaler)
return m
}
}
v = reflect.New(t)
m.Unmarshaler, m.IsValid = v.Interface().(encoding.TextUnmarshaler)
return m
}
// TextUnmarshaler helpers ----------------------------------------------------
// unmarshaller contains information about a TextUnmarshaler type
type unmarshaler struct {
Unmarshaler encoding.TextUnmarshaler
IsSlice bool
IsPtr bool
IsValid bool
}
// Errors ---------------------------------------------------------------------
// ConversionError stores information about a failed conversion.
type ConversionError struct {
Key string // key from the source map.
Type reflect.Type // expected type of elem
Index int // index for multi-value fields; -1 for single-value fields.
Err error // low-level error (when it exists)
}
func (e ConversionError) Error() string {
var output string
if e.Index < 0 {
output = fmt.Sprintf("schema: error converting value for %q", e.Key)
} else {
output = fmt.Sprintf("schema: error converting value for index %d of %q",
e.Index, e.Key)
}
if e.Err != nil {
output = fmt.Sprintf("%s. Details: %s", output, e.Err)
}
return output
}
// MultiError stores multiple decoding errors.
//
// Borrowed from the App Engine SDK.
type MultiError map[string]error
func (e MultiError) Error() string {
s := ""
for _, err := range e {
s = err.Error()
break
}
switch len(e) {
case 0:
return "(0 errors)"
case 1:
return s
case 2:
return s + " (and 1 other error)"
}
return fmt.Sprintf("%s (and %d other errors)", s, len(e)-1)
}

1693
vendor/github.com/gorilla/schema/decoder_test.go сгенерированный поставляемый Обычный файл
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

148
vendor/github.com/gorilla/schema/doc.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,148 @@
// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
Package gorilla/schema fills a struct with form values.
The basic usage is really simple. Given this struct:
type Person struct {
Name string
Phone string
}
...we can fill it passing a map to the Decode() function:
values := map[string][]string{
"Name": {"John"},
"Phone": {"999-999-999"},
}
person := new(Person)
decoder := schema.NewDecoder()
decoder.Decode(person, values)
This is just a simple example and it doesn't make a lot of sense to create
the map manually. Typically it will come from a http.Request object and
will be of type url.Values, http.Request.Form, or http.Request.MultipartForm:
func MyHandler(w http.ResponseWriter, r *http.Request) {
err := r.ParseForm()
if err != nil {
// Handle error
}
decoder := schema.NewDecoder()
// r.PostForm is a map of our POST form values
err := decoder.Decode(person, r.PostForm)
if err != nil {
// Handle error
}
// Do something with person.Name or person.Phone
}
Note: it is a good idea to set a Decoder instance as a package global,
because it caches meta-data about structs, and an instance can be shared safely:
var decoder = schema.NewDecoder()
To define custom names for fields, use a struct tag "schema". To not populate
certain fields, use a dash for the name and it will be ignored:
type Person struct {
Name string `schema:"name"` // custom name
Phone string `schema:"phone"` // custom name
Admin bool `schema:"-"` // this field is never set
}
The supported field types in the destination struct are:
* bool
* float variants (float32, float64)
* int variants (int, int8, int16, int32, int64)
* string
* uint variants (uint, uint8, uint16, uint32, uint64)
* struct
* a pointer to one of the above types
* a slice or a pointer to a slice of one of the above types
Non-supported types are simply ignored, however custom types can be registered
to be converted.
To fill nested structs, keys must use a dotted notation as the "path" for the
field. So for example, to fill the struct Person below:
type Phone struct {
Label string
Number string
}
type Person struct {
Name string
Phone Phone
}
...the source map must have the keys "Name", "Phone.Label" and "Phone.Number".
This means that an HTML form to fill a Person struct must look like this:
<form>
<input type="text" name="Name">
<input type="text" name="Phone.Label">
<input type="text" name="Phone.Number">
</form>
Single values are filled using the first value for a key from the source map.
Slices are filled using all values for a key from the source map. So to fill
a Person with multiple Phone values, like:
type Person struct {
Name string
Phones []Phone
}
...an HTML form that accepts three Phone values would look like this:
<form>
<input type="text" name="Name">
<input type="text" name="Phones.0.Label">
<input type="text" name="Phones.0.Number">
<input type="text" name="Phones.1.Label">
<input type="text" name="Phones.1.Number">
<input type="text" name="Phones.2.Label">
<input type="text" name="Phones.2.Number">
</form>
Notice that only for slices of structs the slice index is required.
This is needed for disambiguation: if the nested struct also had a slice
field, we could not translate multiple values to it if we did not use an
index for the parent struct.
There's also the possibility to create a custom type that implements the
TextUnmarshaler interface, and in this case there's no need to register
a converter, like:
type Person struct {
Emails []Email
}
type Email struct {
*mail.Address
}
func (e *Email) UnmarshalText(text []byte) (err error) {
e.Address, err = mail.ParseAddress(string(text))
return
}
...an HTML form that accepts three Email values would look like this:
<form>
<input type="email" name="Emails.0">
<input type="email" name="Emails.1">
<input type="email" name="Emails.2">
</form>
*/
package schema

195
vendor/github.com/gorilla/schema/encoder.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,195 @@
package schema
import (
"errors"
"fmt"
"reflect"
"strconv"
)
type encoderFunc func(reflect.Value) string
// Encoder encodes values from a struct into url.Values.
type Encoder struct {
cache *cache
regenc map[reflect.Type]encoderFunc
}
// NewEncoder returns a new Encoder with defaults.
func NewEncoder() *Encoder {
return &Encoder{cache: newCache(), regenc: make(map[reflect.Type]encoderFunc)}
}
// Encode encodes a struct into map[string][]string.
//
// Intended for use with url.Values.
func (e *Encoder) Encode(src interface{}, dst map[string][]string) error {
v := reflect.ValueOf(src)
return e.encode(v, dst)
}
// RegisterEncoder registers a converter for encoding a custom type.
func (e *Encoder) RegisterEncoder(value interface{}, encoder func(reflect.Value) string) {
e.regenc[reflect.TypeOf(value)] = encoder
}
// SetAliasTag changes the tag used to locate custom field aliases.
// The default tag is "schema".
func (e *Encoder) SetAliasTag(tag string) {
e.cache.tag = tag
}
// isValidStructPointer test if input value is a valid struct pointer.
func isValidStructPointer(v reflect.Value) bool {
return v.Type().Kind() == reflect.Ptr && v.Elem().IsValid() && v.Elem().Type().Kind() == reflect.Struct
}
func isZero(v reflect.Value) bool {
switch v.Kind() {
case reflect.Func:
case reflect.Map, reflect.Slice:
return v.IsNil() || v.Len() == 0
case reflect.Array:
z := true
for i := 0; i < v.Len(); i++ {
z = z && isZero(v.Index(i))
}
return z
case reflect.Struct:
z := true
for i := 0; i < v.NumField(); i++ {
z = z && isZero(v.Field(i))
}
return z
}
// Compare other types directly:
z := reflect.Zero(v.Type())
return v.Interface() == z.Interface()
}
func (e *Encoder) encode(v reflect.Value, dst map[string][]string) error {
if v.Kind() == reflect.Ptr {
v = v.Elem()
}
if v.Kind() != reflect.Struct {
return errors.New("schema: interface must be a struct")
}
t := v.Type()
errors := MultiError{}
for i := 0; i < v.NumField(); i++ {
name, opts := fieldAlias(t.Field(i), e.cache.tag)
if name == "-" {
continue
}
// Encode struct pointer types if the field is a valid pointer and a struct.
if isValidStructPointer(v.Field(i)) {
e.encode(v.Field(i).Elem(), dst)
continue
}
encFunc := typeEncoder(v.Field(i).Type(), e.regenc)
// Encode non-slice types and custom implementations immediately.
if encFunc != nil {
value := encFunc(v.Field(i))
if opts.Contains("omitempty") && isZero(v.Field(i)) {
continue
}
dst[name] = append(dst[name], value)
continue
}
if v.Field(i).Type().Kind() == reflect.Struct {
e.encode(v.Field(i), dst)
continue
}
if v.Field(i).Type().Kind() == reflect.Slice {
encFunc = typeEncoder(v.Field(i).Type().Elem(), e.regenc)
}
if encFunc == nil {
errors[v.Field(i).Type().String()] = fmt.Errorf("schema: encoder not found for %v", v.Field(i))
continue
}
// Encode a slice.
if v.Field(i).Len() == 0 && opts.Contains("omitempty") {
continue
}
dst[name] = []string{}
for j := 0; j < v.Field(i).Len(); j++ {
dst[name] = append(dst[name], encFunc(v.Field(i).Index(j)))
}
}
if len(errors) > 0 {
return errors
}
return nil
}
func typeEncoder(t reflect.Type, reg map[reflect.Type]encoderFunc) encoderFunc {
if f, ok := reg[t]; ok {
return f
}
switch t.Kind() {
case reflect.Bool:
return encodeBool
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return encodeInt
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
return encodeUint
case reflect.Float32:
return encodeFloat32
case reflect.Float64:
return encodeFloat64
case reflect.Ptr:
f := typeEncoder(t.Elem(), reg)
return func(v reflect.Value) string {
if v.IsNil() {
return "null"
}
return f(v.Elem())
}
case reflect.String:
return encodeString
default:
return nil
}
}
func encodeBool(v reflect.Value) string {
return strconv.FormatBool(v.Bool())
}
func encodeInt(v reflect.Value) string {
return strconv.FormatInt(int64(v.Int()), 10)
}
func encodeUint(v reflect.Value) string {
return strconv.FormatUint(uint64(v.Uint()), 10)
}
func encodeFloat(v reflect.Value, bits int) string {
return strconv.FormatFloat(v.Float(), 'f', 6, bits)
}
func encodeFloat32(v reflect.Value) string {
return encodeFloat(v, 32)
}
func encodeFloat64(v reflect.Value) string {
return encodeFloat(v, 64)
}
func encodeString(v reflect.Value) string {
return v.String()
}

420
vendor/github.com/gorilla/schema/encoder_test.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,420 @@
package schema
import (
"fmt"
"reflect"
"testing"
)
type E1 struct {
F01 int `schema:"f01"`
F02 int `schema:"-"`
F03 string `schema:"f03"`
F04 string `schema:"f04,omitempty"`
F05 bool `schema:"f05"`
F06 bool `schema:"f06"`
F07 *string `schema:"f07"`
F08 *int8 `schema:"f08"`
F09 float64 `schema:"f09"`
F10 func() `schema:"f10"`
F11 inner
}
type inner struct {
F12 int
}
func TestFilled(t *testing.T) {
f07 := "seven"
var f08 int8 = 8
s := &E1{
F01: 1,
F02: 2,
F03: "three",
F04: "four",
F05: true,
F06: false,
F07: &f07,
F08: &f08,
F09: 1.618,
F10: func() {},
F11: inner{12},
}
vals := make(map[string][]string)
errs := NewEncoder().Encode(s, vals)
valExists(t, "f01", "1", vals)
valNotExists(t, "f02", vals)
valExists(t, "f03", "three", vals)
valExists(t, "f05", "true", vals)
valExists(t, "f06", "false", vals)
valExists(t, "f07", "seven", vals)
valExists(t, "f08", "8", vals)
valExists(t, "f09", "1.618000", vals)
valExists(t, "F12", "12", vals)
emptyErr := MultiError{}
if errs.Error() == emptyErr.Error() {
t.Errorf("Expected error got %v", errs)
}
}
type Aa int
type E3 struct {
F01 bool `schema:"f01"`
F02 float32 `schema:"f02"`
F03 float64 `schema:"f03"`
F04 int `schema:"f04"`
F05 int8 `schema:"f05"`
F06 int16 `schema:"f06"`
F07 int32 `schema:"f07"`
F08 int64 `schema:"f08"`
F09 string `schema:"f09"`
F10 uint `schema:"f10"`
F11 uint8 `schema:"f11"`
F12 uint16 `schema:"f12"`
F13 uint32 `schema:"f13"`
F14 uint64 `schema:"f14"`
F15 Aa `schema:"f15"`
}
// Test compatibility with default decoder types.
func TestCompat(t *testing.T) {
src := &E3{
F01: true,
F02: 4.2,
F03: 4.3,
F04: -42,
F05: -43,
F06: -44,
F07: -45,
F08: -46,
F09: "foo",
F10: 42,
F11: 43,
F12: 44,
F13: 45,
F14: 46,
F15: 1,
}
dst := &E3{}
vals := make(map[string][]string)
encoder := NewEncoder()
decoder := NewDecoder()
encoder.RegisterEncoder(src.F15, func(reflect.Value) string { return "1" })
decoder.RegisterConverter(src.F15, func(string) reflect.Value { return reflect.ValueOf(1) })
err := encoder.Encode(src, vals)
if err != nil {
t.Errorf("Encoder has non-nil error: %v", err)
}
err = decoder.Decode(dst, vals)
if err != nil {
t.Errorf("Decoder has non-nil error: %v", err)
}
if *src != *dst {
t.Errorf("Decoder-Encoder compatibility: expected %v, got %v\n", src, dst)
}
}
func TestEmpty(t *testing.T) {
s := &E1{
F01: 1,
F02: 2,
F03: "three",
}
estr := "schema: encoder not found for <nil>"
vals := make(map[string][]string)
err := NewEncoder().Encode(s, vals)
if err.Error() != estr {
t.Errorf("Expected: %s, got %v", estr, err)
}
valExists(t, "f03", "three", vals)
valNotExists(t, "f04", vals)
}
func TestStruct(t *testing.T) {
estr := "schema: interface must be a struct"
vals := make(map[string][]string)
err := NewEncoder().Encode("hello world", vals)
if err.Error() != estr {
t.Errorf("Expected: %s, got %v", estr, err)
}
}
func TestSlices(t *testing.T) {
type oneAsWord int
ones := []oneAsWord{1, 2}
s1 := &struct {
ones []oneAsWord `schema:"ones"`
ints []int `schema:"ints"`
nonempty []int `schema:"nonempty"`
empty []int `schema:"empty,omitempty"`
}{ones, []int{1, 1}, []int{}, []int{}}
vals := make(map[string][]string)
encoder := NewEncoder()
encoder.RegisterEncoder(ones[0], func(v reflect.Value) string { return "one" })
err := encoder.Encode(s1, vals)
if err != nil {
t.Errorf("Encoder has non-nil error: %v", err)
}
valsExist(t, "ones", []string{"one", "one"}, vals)
valsExist(t, "ints", []string{"1", "1"}, vals)
valsExist(t, "nonempty", []string{}, vals)
valNotExists(t, "empty", vals)
}
func TestCompatSlices(t *testing.T) {
type oneAsWord int
type s1 struct {
Ones []oneAsWord `schema:"ones"`
Ints []int `schema:"ints"`
}
ones := []oneAsWord{1, 1}
src := &s1{ones, []int{1, 1}}
vals := make(map[string][]string)
dst := &s1{}
encoder := NewEncoder()
encoder.RegisterEncoder(ones[0], func(v reflect.Value) string { return "one" })
decoder := NewDecoder()
decoder.RegisterConverter(ones[0], func(s string) reflect.Value {
if s == "one" {
return reflect.ValueOf(1)
}
return reflect.ValueOf(2)
})
err := encoder.Encode(src, vals)
if err != nil {
t.Errorf("Encoder has non-nil error: %v", err)
}
err = decoder.Decode(dst, vals)
if err != nil {
t.Errorf("Dncoder has non-nil error: %v", err)
}
if len(src.Ints) != len(dst.Ints) || len(src.Ones) != len(src.Ones) {
t.Fatalf("Expected %v, got %v", src, dst)
}
for i, v := range src.Ones {
if dst.Ones[i] != v {
t.Fatalf("Expected %v, got %v", v, dst.Ones[i])
}
}
for i, v := range src.Ints {
if dst.Ints[i] != v {
t.Fatalf("Expected %v, got %v", v, dst.Ints[i])
}
}
}
func TestRegisterEncoder(t *testing.T) {
type oneAsWord int
type twoAsWord int
type oneSliceAsWord []int
s1 := &struct {
oneAsWord
twoAsWord
oneSliceAsWord
}{1, 2, []int{1, 1}}
v1 := make(map[string][]string)
encoder := NewEncoder()
encoder.RegisterEncoder(s1.oneAsWord, func(v reflect.Value) string { return "one" })
encoder.RegisterEncoder(s1.twoAsWord, func(v reflect.Value) string { return "two" })
encoder.RegisterEncoder(s1.oneSliceAsWord, func(v reflect.Value) string { return "one" })
err := encoder.Encode(s1, v1)
if err != nil {
t.Errorf("Encoder has non-nil error: %v", err)
}
valExists(t, "oneAsWord", "one", v1)
valExists(t, "twoAsWord", "two", v1)
valExists(t, "oneSliceAsWord", "one", v1)
}
func TestEncoderOrder(t *testing.T) {
type builtinEncoderSimple int
type builtinEncoderSimpleOverridden int
type builtinEncoderSlice []int
type builtinEncoderSliceOverridden []int
type builtinEncoderStruct struct{ nr int }
type builtinEncoderStructOverridden struct{ nr int }
s1 := &struct {
builtinEncoderSimple `schema:"simple"`
builtinEncoderSimpleOverridden `schema:"simple_overridden"`
builtinEncoderSlice `schema:"slice"`
builtinEncoderSliceOverridden `schema:"slice_overridden"`
builtinEncoderStruct `schema:"struct"`
builtinEncoderStructOverridden `schema:"struct_overridden"`
}{
1,
1,
[]int{2},
[]int{2},
builtinEncoderStruct{3},
builtinEncoderStructOverridden{3},
}
v1 := make(map[string][]string)
encoder := NewEncoder()
encoder.RegisterEncoder(s1.builtinEncoderSimpleOverridden, func(v reflect.Value) string { return "one" })
encoder.RegisterEncoder(s1.builtinEncoderSliceOverridden, func(v reflect.Value) string { return "two" })
encoder.RegisterEncoder(s1.builtinEncoderStructOverridden, func(v reflect.Value) string { return "three" })
err := encoder.Encode(s1, v1)
if err != nil {
t.Errorf("Encoder has non-nil error: %v", err)
}
valExists(t, "simple", "1", v1)
valExists(t, "simple_overridden", "one", v1)
valExists(t, "slice", "2", v1)
valExists(t, "slice_overridden", "two", v1)
valExists(t, "nr", "3", v1)
valExists(t, "struct_overridden", "three", v1)
}
func valExists(t *testing.T, key string, expect string, result map[string][]string) {
valsExist(t, key, []string{expect}, result)
}
func valsExist(t *testing.T, key string, expect []string, result map[string][]string) {
vals, ok := result[key]
if !ok {
t.Fatalf("Key not found. Expected: %s", key)
}
if len(expect) != len(vals) {
t.Fatalf("Expected: %v, got: %v", expect, vals)
}
for i, v := range expect {
if vals[i] != v {
t.Fatalf("Unexpected value. Expected: %v, got %v", v, vals[i])
}
}
}
func valNotExists(t *testing.T, key string, result map[string][]string) {
if val, ok := result[key]; ok {
t.Error("Key not ommited. Expected: empty; got: " + val[0] + ".")
}
}
type E4 struct {
ID string `json:"id"`
}
func TestEncoderSetAliasTag(t *testing.T) {
data := map[string][]string{}
s := E4{
ID: "foo",
}
encoder := NewEncoder()
encoder.SetAliasTag("json")
encoder.Encode(&s, data)
valExists(t, "id", "foo", data)
}
type E5 struct {
F01 int `schema:"f01,omitempty"`
F02 string `schema:"f02,omitempty"`
F03 *string `schema:"f03,omitempty"`
F04 *int8 `schema:"f04,omitempty"`
F05 float64 `schema:"f05,omitempty"`
F06 E5F06 `schema:"f06,omitempty"`
F07 E5F06 `schema:"f07,omitempty"`
F08 []string `schema:"f08,omitempty"`
F09 []string `schema:"f09,omitempty"`
}
type E5F06 struct {
F0601 string `schema:"f0601,omitempty"`
}
func TestEncoderWithOmitempty(t *testing.T) {
vals := map[string][]string{}
s := E5{
F02: "test",
F07: E5F06{
F0601: "test",
},
F09: []string{"test"},
}
encoder := NewEncoder()
encoder.Encode(&s, vals)
valNotExists(t, "f01", vals)
valExists(t, "f02", "test", vals)
valNotExists(t, "f03", vals)
valNotExists(t, "f04", vals)
valNotExists(t, "f05", vals)
valNotExists(t, "f06", vals)
valExists(t, "f0601", "test", vals)
valNotExists(t, "f08", vals)
valsExist(t, "f09", []string{"test"}, vals)
}
type E6 struct {
F01 *inner
F02 *inner
F03 *inner `schema:",omitempty"`
}
func TestStructPointer(t *testing.T) {
vals := map[string][]string{}
s := E6{
F01: &inner{2},
}
encoder := NewEncoder()
encoder.Encode(&s, vals)
valExists(t, "F12", "2", vals)
valExists(t, "F02", "null", vals)
valNotExists(t, "F03", vals)
}
func TestRegisterEncoderCustomArrayType(t *testing.T) {
type CustomInt []int
type S1 struct {
SomeInts CustomInt `schema:",omitempty"`
}
ss := []S1{
{},
{CustomInt{}},
{CustomInt{1, 2, 3}},
}
for s := range ss {
vals := map[string][]string{}
encoder := NewEncoder()
encoder.RegisterEncoder(CustomInt{}, func(value reflect.Value) string {
return fmt.Sprint(value.Interface())
})
encoder.Encode(s, vals)
t.Log(vals)
}
}

15
vendor/github.com/gorilla/websocket/conn.go сгенерированный поставляемый
Просмотреть файл

@@ -1051,8 +1051,9 @@ func (c *Conn) CloseHandler() func(code int, text string) error {
// if the close message is empty. The default close handler sends a close
// message back to the peer.
//
// The application must read the connection to process close messages as
// described in the section on Control Messages above.
// The handler function is called from the NextReader, ReadMessage and message
// reader Read methods. The application must read the connection to process
// close messages as described in the section on Control Messages above.
//
// The connection read methods return a CloseError when a close message is
// received. Most applications should handle close messages as part of their
@@ -1079,8 +1080,9 @@ func (c *Conn) PingHandler() func(appData string) error {
// The appData argument to h is the PING message application data. The default
// ping handler sends a pong to the peer.
//
// The application must read the connection to process ping messages as
// described in the section on Control Messages above.
// The handler function is called from the NextReader, ReadMessage and message
// reader Read methods. The application must read the connection to process
// ping messages as described in the section on Control Messages above.
func (c *Conn) SetPingHandler(h func(appData string) error) {
if h == nil {
h = func(message string) error {
@@ -1105,8 +1107,9 @@ func (c *Conn) PongHandler() func(appData string) error {
// The appData argument to h is the PONG message application data. The default
// pong handler does nothing.
//
// The application must read the connection to process ping messages as
// described in the section on Control Messages above.
// The handler function is called from the NextReader, ReadMessage and message
// reader Read methods. The application must read the connection to process
// pong messages as described in the section on Control Messages above.
func (c *Conn) SetPongHandler(h func(appData string) error) {
if h == nil {
h = func(string) error { return nil }

7
vendor/github.com/hashicorp/go-immutable-radix/iradix.go сгенерированный поставляемый
Просмотреть файл

@@ -338,6 +338,11 @@ func (t *Txn) delete(parent, n *Node, search []byte) (*Node, *leafNode) {
if !n.isLeaf() {
return nil, nil
}
// Copy the pointer in case we are in a transaction that already
// modified this node since the node will be reused. Any changes
// made to the node will not affect returning the original leaf
// value.
oldLeaf := n.leaf
// Remove the leaf node
nc := t.writeNode(n, true)
@@ -347,7 +352,7 @@ func (t *Txn) delete(parent, n *Node, search []byte) (*Node, *leafNode) {
if n != t.root && len(nc.edges) == 1 {
t.mergeChild(nc)
}
return nc, n.leaf
return nc, oldLeaf
}
// Look for an edge

37
vendor/github.com/hashicorp/go-immutable-radix/iradix_test.go сгенерированный поставляемый
Просмотреть файл

@@ -173,7 +173,7 @@ func TestRoot(t *testing.T) {
}
val, ok := r.Get(nil)
if !ok || val != true {
t.Fatalf("bad: %v %#v", val)
t.Fatalf("bad: %#v", val)
}
r, val, ok = r.Delete(nil)
if !ok || val != true {
@@ -1494,3 +1494,38 @@ func TestTrackMutate_cachedNodeChange(t *testing.T) {
}
}
}
func TestLenTxn(t *testing.T) {
r := New()
if r.Len() != 0 {
t.Fatalf("not starting with empty tree")
}
txn := r.Txn()
keys := []string{
"foo/bar/baz",
"foo/baz/bar",
"foo/zip/zap",
"foobar",
"nochange",
}
for _, k := range keys {
txn.Insert([]byte(k), nil)
}
r = txn.Commit()
if r.Len() != len(keys) {
t.Fatalf("bad: expected %d, got %d", len(keys), r.Len())
}
txn = r.Txn()
for _, k := range keys {
txn.Delete([]byte(k))
}
r = txn.Commit()
if r.Len() != 0 {
t.Fatalf("tree len should be zero, got %d", r.Len())
}
}

2
vendor/github.com/hashicorp/go-sockaddr/ifaddrs.go сгенерированный поставляемый
Просмотреть файл

@@ -16,7 +16,7 @@ var (
// Centralize all regexps and regexp.Copy() where necessary.
signRE *regexp.Regexp = regexp.MustCompile(`^[\s]*[+-]`)
whitespaceRE *regexp.Regexp = regexp.MustCompile(`[\s]+`)
ifNameRE *regexp.Regexp = regexp.MustCompile(`^Ethernet adapter ([^\s:]+):`)
ifNameRE *regexp.Regexp = regexp.MustCompile(`^Ethernet adapter ([^:]+):`)
ipAddrRE *regexp.Regexp = regexp.MustCompile(`^ IPv[46] Address\. \. \. \. \. \. \. \. \. \. \. : ([^\s]+)`)
)

21
vendor/github.com/hashicorp/golang-lru/2q.go сгенерированный поставляемый
Просмотреть файл

@@ -30,9 +30,9 @@ type TwoQueueCache struct {
size int
recentSize int
recent *simplelru.LRU
frequent *simplelru.LRU
recentEvict *simplelru.LRU
recent simplelru.LRUCache
frequent simplelru.LRUCache
recentEvict simplelru.LRUCache
lock sync.RWMutex
}
@@ -84,7 +84,8 @@ func New2QParams(size int, recentRatio float64, ghostRatio float64) (*TwoQueueCa
return c, nil
}
func (c *TwoQueueCache) Get(key interface{}) (interface{}, bool) {
// Get looks up a key's value from the cache.
func (c *TwoQueueCache) Get(key interface{}) (value interface{}, ok bool) {
c.lock.Lock()
defer c.lock.Unlock()
@@ -105,6 +106,7 @@ func (c *TwoQueueCache) Get(key interface{}) (interface{}, bool) {
return nil, false
}
// Add adds a value to the cache.
func (c *TwoQueueCache) Add(key, value interface{}) {
c.lock.Lock()
defer c.lock.Unlock()
@@ -160,12 +162,15 @@ func (c *TwoQueueCache) ensureSpace(recentEvict bool) {
c.frequent.RemoveOldest()
}
// Len returns the number of items in the cache.
func (c *TwoQueueCache) Len() int {
c.lock.RLock()
defer c.lock.RUnlock()
return c.recent.Len() + c.frequent.Len()
}
// Keys returns a slice of the keys in the cache.
// The frequently used keys are first in the returned slice.
func (c *TwoQueueCache) Keys() []interface{} {
c.lock.RLock()
defer c.lock.RUnlock()
@@ -174,6 +179,7 @@ func (c *TwoQueueCache) Keys() []interface{} {
return append(k1, k2...)
}
// Remove removes the provided key from the cache.
func (c *TwoQueueCache) Remove(key interface{}) {
c.lock.Lock()
defer c.lock.Unlock()
@@ -188,6 +194,7 @@ func (c *TwoQueueCache) Remove(key interface{}) {
}
}
// Purge is used to completely clear the cache.
func (c *TwoQueueCache) Purge() {
c.lock.Lock()
defer c.lock.Unlock()
@@ -196,13 +203,17 @@ func (c *TwoQueueCache) Purge() {
c.recentEvict.Purge()
}
// Contains is used to check if the cache contains a key
// without updating recency or frequency.
func (c *TwoQueueCache) Contains(key interface{}) bool {
c.lock.RLock()
defer c.lock.RUnlock()
return c.frequent.Contains(key) || c.recent.Contains(key)
}
func (c *TwoQueueCache) Peek(key interface{}) (interface{}, bool) {
// Peek is used to inspect the cache value of a key
// without updating recency or frequency.
func (c *TwoQueueCache) Peek(key interface{}) (value interface{}, ok bool) {
c.lock.RLock()
defer c.lock.RUnlock()
if val, ok := c.frequent.Peek(key); ok {

16
vendor/github.com/hashicorp/golang-lru/arc.go сгенерированный поставляемый
Просмотреть файл

@@ -18,11 +18,11 @@ type ARCCache struct {
size int // Size is the total capacity of the cache
p int // P is the dynamic preference towards T1 or T2
t1 *simplelru.LRU // T1 is the LRU for recently accessed items
b1 *simplelru.LRU // B1 is the LRU for evictions from t1
t1 simplelru.LRUCache // T1 is the LRU for recently accessed items
b1 simplelru.LRUCache // B1 is the LRU for evictions from t1
t2 *simplelru.LRU // T2 is the LRU for frequently accessed items
b2 *simplelru.LRU // B2 is the LRU for evictions from t2
t2 simplelru.LRUCache // T2 is the LRU for frequently accessed items
b2 simplelru.LRUCache // B2 is the LRU for evictions from t2
lock sync.RWMutex
}
@@ -60,11 +60,11 @@ func NewARC(size int) (*ARCCache, error) {
}
// Get looks up a key's value from the cache.
func (c *ARCCache) Get(key interface{}) (interface{}, bool) {
func (c *ARCCache) Get(key interface{}) (value interface{}, ok bool) {
c.lock.Lock()
defer c.lock.Unlock()
// Ff the value is contained in T1 (recent), then
// If the value is contained in T1 (recent), then
// promote it to T2 (frequent)
if val, ok := c.t1.Peek(key); ok {
c.t1.Remove(key)
@@ -153,7 +153,7 @@ func (c *ARCCache) Add(key, value interface{}) {
// Remove from B2
c.b2.Remove(key)
// Add the key to the frequntly used list
// Add the key to the frequently used list
c.t2.Add(key, value)
return
}
@@ -247,7 +247,7 @@ func (c *ARCCache) Contains(key interface{}) bool {
// Peek is used to inspect the cache value of a key
// without updating recency or frequency.
func (c *ARCCache) Peek(key interface{}) (interface{}, bool) {
func (c *ARCCache) Peek(key interface{}) (value interface{}, ok bool) {
c.lock.RLock()
defer c.lock.RUnlock()
if val, ok := c.t1.Peek(key); ok {

21
vendor/github.com/hashicorp/golang-lru/doc.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,21 @@
// Package lru provides three different LRU caches of varying sophistication.
//
// Cache is a simple LRU cache. It is based on the
// LRU implementation in groupcache:
// https://github.com/golang/groupcache/tree/master/lru
//
// TwoQueueCache tracks frequently used and recently used entries separately.
// This avoids a burst of accesses from taking out frequently used entries,
// at the cost of about 2x computational overhead and some extra bookkeeping.
//
// ARCCache is an adaptive replacement cache. It tracks recent evictions as
// well as recent usage in both the frequent and recent caches. Its
// computational overhead is comparable to TwoQueueCache, but the memory
// overhead is linear with the size of the cache.
//
// ARC has been patented by IBM, so do not use it if that is problematic for
// your program.
//
// All caches in this package take locks while operating, and are therefore
// thread-safe for consumers.
package lru

28
vendor/github.com/hashicorp/golang-lru/lru.go сгенерированный поставляемый
Просмотреть файл

@@ -1,6 +1,3 @@
// This package provides a simple LRU cache. It is based on the
// LRU implementation in groupcache:
// https://github.com/golang/groupcache/tree/master/lru
package lru
import (
@@ -11,11 +8,11 @@ import (
// Cache is a thread-safe fixed size LRU cache.
type Cache struct {
lru *simplelru.LRU
lru simplelru.LRUCache
lock sync.RWMutex
}
// New creates an LRU of the given size
// New creates an LRU of the given size.
func New(size int) (*Cache, error) {
return NewWithEvict(size, nil)
}
@@ -33,7 +30,7 @@ func NewWithEvict(size int, onEvicted func(key interface{}, value interface{}))
return c, nil
}
// Purge is used to completely clear the cache
// Purge is used to completely clear the cache.
func (c *Cache) Purge() {
c.lock.Lock()
c.lru.Purge()
@@ -41,30 +38,30 @@ func (c *Cache) Purge() {
}
// Add adds a value to the cache. Returns true if an eviction occurred.
func (c *Cache) Add(key, value interface{}) bool {
func (c *Cache) Add(key, value interface{}) (evicted bool) {
c.lock.Lock()
defer c.lock.Unlock()
return c.lru.Add(key, value)
}
// Get looks up a key's value from the cache.
func (c *Cache) Get(key interface{}) (interface{}, bool) {
func (c *Cache) Get(key interface{}) (value interface{}, ok bool) {
c.lock.Lock()
defer c.lock.Unlock()
return c.lru.Get(key)
}
// Check if a key is in the cache, without updating the recent-ness
// or deleting it for being stale.
// Contains checks if a key is in the cache, without updating the
// recent-ness or deleting it for being stale.
func (c *Cache) Contains(key interface{}) bool {
c.lock.RLock()
defer c.lock.RUnlock()
return c.lru.Contains(key)
}
// Returns the key value (or undefined if not found) without updating
// Peek returns the key value (or undefined if not found) without updating
// the "recently used"-ness of the key.
func (c *Cache) Peek(key interface{}) (interface{}, bool) {
func (c *Cache) Peek(key interface{}) (value interface{}, ok bool) {
c.lock.RLock()
defer c.lock.RUnlock()
return c.lru.Peek(key)
@@ -73,16 +70,15 @@ func (c *Cache) Peek(key interface{}) (interface{}, bool) {
// ContainsOrAdd checks if a key is in the cache without updating the
// recent-ness or deleting it for being stale, and if not, adds the value.
// Returns whether found and whether an eviction occurred.
func (c *Cache) ContainsOrAdd(key, value interface{}) (ok, evict bool) {
func (c *Cache) ContainsOrAdd(key, value interface{}) (ok, evicted bool) {
c.lock.Lock()
defer c.lock.Unlock()
if c.lru.Contains(key) {
return true, false
} else {
evict := c.lru.Add(key, value)
return false, evict
}
evicted = c.lru.Add(key, value)
return false, evicted
}
// Remove removes the provided key from the cache.

4
vendor/github.com/hashicorp/golang-lru/lru_test.go сгенерированный поставляемый
Просмотреть файл

@@ -72,7 +72,7 @@ func TestLRU(t *testing.T) {
if k != v {
t.Fatalf("Evict values not equal (%v!=%v)", k, v)
}
evictCounter += 1
evictCounter++
}
l, err := NewWithEvict(128, onEvicted)
if err != nil {
@@ -136,7 +136,7 @@ func TestLRU(t *testing.T) {
func TestLRUAdd(t *testing.T) {
evictCounter := 0
onEvicted := func(k interface{}, v interface{}) {
evictCounter += 1
evictCounter++
}
l, err := NewWithEvict(1, onEvicted)

17
vendor/github.com/hashicorp/golang-lru/simplelru/lru.go сгенерированный поставляемый
Просмотреть файл

@@ -36,7 +36,7 @@ func NewLRU(size int, onEvict EvictCallback) (*LRU, error) {
return c, nil
}
// Purge is used to completely clear the cache
// Purge is used to completely clear the cache.
func (c *LRU) Purge() {
for k, v := range c.items {
if c.onEvict != nil {
@@ -48,7 +48,7 @@ func (c *LRU) Purge() {
}
// Add adds a value to the cache. Returns true if an eviction occurred.
func (c *LRU) Add(key, value interface{}) bool {
func (c *LRU) Add(key, value interface{}) (evicted bool) {
// Check for existing item
if ent, ok := c.items[key]; ok {
c.evictList.MoveToFront(ent)
@@ -78,17 +78,18 @@ func (c *LRU) Get(key interface{}) (value interface{}, ok bool) {
return
}
// Check if a key is in the cache, without updating the recent-ness
// Contains checks if a key is in the cache, without updating the recent-ness
// or deleting it for being stale.
func (c *LRU) Contains(key interface{}) (ok bool) {
_, ok = c.items[key]
return ok
}
// Returns the key value (or undefined if not found) without updating
// Peek returns the key value (or undefined if not found) without updating
// the "recently used"-ness of the key.
func (c *LRU) Peek(key interface{}) (value interface{}, ok bool) {
if ent, ok := c.items[key]; ok {
var ent *list.Element
if ent, ok = c.items[key]; ok {
return ent.Value.(*entry).value, true
}
return nil, ok
@@ -96,7 +97,7 @@ func (c *LRU) Peek(key interface{}) (value interface{}, ok bool) {
// Remove removes the provided key from the cache, returning if the
// key was contained.
func (c *LRU) Remove(key interface{}) bool {
func (c *LRU) Remove(key interface{}) (present bool) {
if ent, ok := c.items[key]; ok {
c.removeElement(ent)
return true
@@ -105,7 +106,7 @@ func (c *LRU) Remove(key interface{}) bool {
}
// RemoveOldest removes the oldest item from the cache.
func (c *LRU) RemoveOldest() (interface{}, interface{}, bool) {
func (c *LRU) RemoveOldest() (key interface{}, value interface{}, ok bool) {
ent := c.evictList.Back()
if ent != nil {
c.removeElement(ent)
@@ -116,7 +117,7 @@ func (c *LRU) RemoveOldest() (interface{}, interface{}, bool) {
}
// GetOldest returns the oldest entry
func (c *LRU) GetOldest() (interface{}, interface{}, bool) {
func (c *LRU) GetOldest() (key interface{}, value interface{}, ok bool) {
ent := c.evictList.Back()
if ent != nil {
kv := ent.Value.(*entry)

37
vendor/github.com/hashicorp/golang-lru/simplelru/lru_interface.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,37 @@
package simplelru
// LRUCache is the interface for simple LRU cache.
type LRUCache interface {
// Adds a value to the cache, returns true if an eviction occurred and
// updates the "recently used"-ness of the key.
Add(key, value interface{}) bool
// Returns key's value from the cache and
// updates the "recently used"-ness of the key. #value, isFound
Get(key interface{}) (value interface{}, ok bool)
// Check if a key exsists in cache without updating the recent-ness.
Contains(key interface{}) (ok bool)
// Returns key's value without updating the "recently used"-ness of the key.
Peek(key interface{}) (value interface{}, ok bool)
// Removes a key from the cache.
Remove(key interface{}) bool
// Removes the oldest entry from cache.
RemoveOldest() (interface{}, interface{}, bool)
// Returns the oldest entry from the cache. #key, value, isFound
GetOldest() (interface{}, interface{}, bool)
// Returns a slice of the keys in the cache, from oldest to newest.
Keys() []interface{}
// Returns the number of items in the cache.
Len() int
// Clear all cache entries
Purge()
}

4
vendor/github.com/hashicorp/golang-lru/simplelru/lru_test.go сгенерированный поставляемый
Просмотреть файл

@@ -8,7 +8,7 @@ func TestLRU(t *testing.T) {
if k != v {
t.Fatalf("Evict values not equal (%v!=%v)", k, v)
}
evictCounter += 1
evictCounter++
}
l, err := NewLRU(128, onEvicted)
if err != nil {
@@ -112,7 +112,7 @@ func TestLRU_GetOldest_RemoveOldest(t *testing.T) {
func TestLRU_Add(t *testing.T) {
evictCounter := 0
onEvicted := func(k interface{}, v interface{}) {
evictCounter += 1
evictCounter++
}
l, err := NewLRU(1, onEvicted)

2
vendor/github.com/hashicorp/memberlist/README.md сгенерированный поставляемый
Просмотреть файл

@@ -65,7 +65,7 @@ For complete documentation, see the associated [Godoc](http://godoc.org/github.c
## Protocol
memberlist is based on ["SWIM: Scalable Weakly-consistent Infection-style Process Group Membership Protocol"](http://www.cs.cornell.edu/~asdas/research/dsn02-swim.pdf). However, we extend the protocol in a number of ways:
memberlist is based on ["SWIM: Scalable Weakly-consistent Infection-style Process Group Membership Protocol"](http://ieeexplore.ieee.org/document/1028914/). However, we extend the protocol in a number of ways:
* Several extensions are made to increase propagation speed and
convergence rate.

1
vendor/github.com/lib/pq/.travis.yml сгенерированный поставляемый
Просмотреть файл

@@ -1,7 +1,6 @@
language: go
go:
- 1.5.x
- 1.6.x
- 1.7.x
- 1.8.x

21
vendor/github.com/magiconair/properties/CHANGELOG.md сгенерированный поставляемый
Просмотреть файл

@@ -1,10 +1,29 @@
## Changelog
### [1.7.6](https://github.com/magiconair/properties/tree/v1.7.6) - 14 Feb 2018
* [PR #29](https://github.com/magiconair/properties/pull/29): Reworked expansion logic to handle more complex cases.
See PR for an example.
Thanks to [@yobert](https://github.com/yobert) for the fix.
### [1.7.5](https://github.com/magiconair/properties/tree/v1.7.5) - 13 Feb 2018
* [PR #28](https://github.com/magiconair/properties/pull/28): Support duplicate expansions in the same value
Values which expand the same key multiple times (e.g. `key=${a} ${a}`) will no longer fail
with a `circular reference error`.
Thanks to [@yobert](https://github.com/yobert) for the fix.
### [1.7.4](https://github.com/magiconair/properties/tree/v1.7.4) - 31 Oct 2017
* [Issue #23](https://github.com/magiconair/properties/issues/23): Ignore blank lines with whitespaces
* [PR #24](https://github.com/magiconair/properties/pull/24): Update keys when DisableExpansion is enabled
Thanks to @mgurov for the fix.
Thanks to [@mgurov](https://github.com/mgurov) for the fix.
### [1.7.3](https://github.com/magiconair/properties/tree/v1.7.3) - 10 Jul 2017

2
vendor/github.com/magiconair/properties/LICENSE сгенерированный поставляемый
Просмотреть файл

@@ -1,6 +1,6 @@
goproperties - properties file decoder for Go
Copyright (c) 2013-2014 - Frank Schroeder
Copyright (c) 2013-2018 - Frank Schroeder
All rights reserved.

50
vendor/github.com/magiconair/properties/README.md сгенерированный поставляемый
Просмотреть файл

@@ -1,7 +1,11 @@
Overview [![Build Status](https://travis-ci.org/magiconair/properties.svg?branch=master)](https://travis-ci.org/magiconair/properties)
========
[![](https://img.shields.io/github/tag/magiconair/properties.svg?style=flat-square&label=release)](https://github.com/magiconair/properties/releases)
[![Build Status](https://img.shields.io/travis/magiconair/properties.svg?branch=master&style=flat-square)](https://travis-ci.org/magiconair/properties)
[![License](https://img.shields.io/badge/License-BSD%202--Clause-orange.svg?style=flat-square)](https://raw.githubusercontent.com/magiconair/properties/master/LICENSE)
[![GoDoc](http://img.shields.io/badge/godoc-reference-5272B4.svg?style=flat-square)](http://godoc.org/github.com/magiconair/properties)
#### Current version: 1.7.4
# Overview
#### Please run `git pull --tags` to update the tags. See [below](#updated-git-tags) why.
properties is a Go library for reading and writing properties files.
@@ -27,8 +31,7 @@ details.
Read the full documentation on [GoDoc](https://godoc.org/github.com/magiconair/properties) [![GoDoc](https://godoc.org/github.com/magiconair/properties?status.png)](https://godoc.org/github.com/magiconair/properties)
Getting Started
---------------
## Getting Started
```go
import (
@@ -83,18 +86,43 @@ func main() {
```
Installation and Upgrade
------------------------
## Installation and Upgrade
```
$ go get -u github.com/magiconair/properties
```
License
-------
## License
2 clause BSD license. See [LICENSE](https://github.com/magiconair/properties/blob/master/LICENSE) file for details.
ToDo
----
## ToDo
* Dump contents with passwords and secrets obscured
## Updated Git tags
#### 13 Feb 2018
I realized that all of the git tags I had pushed before v1.7.5 were lightweight tags
and I've only recently learned that this doesn't play well with `git describe` 😞
I have replaced all lightweight tags with signed tags using this script which should
retain the commit date, name and email address. Please run `git pull --tags` to update them.
Worst case you have to reclone the repo.
```shell
#!/bin/bash
tag=$1
echo "Updating $tag"
date=$(git show ${tag}^0 --format=%aD | head -1)
email=$(git show ${tag}^0 --format=%aE | head -1)
name=$(git show ${tag}^0 --format=%aN | head -1)
GIT_COMMITTER_DATE="$date" GIT_COMMITTER_NAME="$name" GIT_COMMITTER_EMAIL="$email" git tag -s -f ${tag} ${tag}^0 -m ${tag}
```
I apologize for the inconvenience.
Frank

2
vendor/github.com/magiconair/properties/load.go сгенерированный поставляемый
Просмотреть файл

@@ -218,7 +218,7 @@ func must(p *Properties, err error) *Properties {
// with an empty string. Malformed expressions like "${ENV_VAR" will
// be reported as error.
func expandName(name string) (string, error) {
return expand(name, make(map[string]bool), "${", "}", make(map[string]string))
return expand(name, []string{}, "${", "}", make(map[string]string))
}
// Interprets a byte buffer either as an ISO-8859-1 or UTF-8 encoded string.

77
vendor/github.com/magiconair/properties/properties.go сгенерированный поставляемый
Просмотреть файл

@@ -19,6 +19,8 @@ import (
"unicode/utf8"
)
const maxExpansionDepth = 64
// ErrorHandlerFunc defines the type of function which handles failures
// of the MustXXX() functions. An error handler function must exit
// the application after handling the error.
@@ -92,7 +94,7 @@ func (p *Properties) Get(key string) (value string, ok bool) {
return "", false
}
expanded, err := p.expand(v)
expanded, err := p.expand(key, v)
// we guarantee that the expanded value is free of
// circular references and malformed expressions
@@ -525,7 +527,7 @@ func (p *Properties) Set(key, value string) (prev string, ok bool, err error) {
p.m[key] = value
// now check for a circular reference
_, err = p.expand(value)
_, err = p.expand(key, value)
if err != nil {
// revert to the previous state
@@ -696,56 +698,65 @@ outer:
// check expands all values and returns an error if a circular reference or
// a malformed expression was found.
func (p *Properties) check() error {
for _, value := range p.m {
if _, err := p.expand(value); err != nil {
for key, value := range p.m {
if _, err := p.expand(key, value); err != nil {
return err
}
}
return nil
}
func (p *Properties) expand(input string) (string, error) {
func (p *Properties) expand(key, input string) (string, error) {
// no pre/postfix -> nothing to expand
if p.Prefix == "" && p.Postfix == "" {
return input, nil
}
return expand(input, make(map[string]bool), p.Prefix, p.Postfix, p.m)
return expand(input, []string{key}, p.Prefix, p.Postfix, p.m)
}
// expand recursively expands expressions of '(prefix)key(postfix)' to their corresponding values.
// The function keeps track of the keys that were already expanded and stops if it
// detects a circular reference or a malformed expression of the form '(prefix)key'.
func expand(s string, keys map[string]bool, prefix, postfix string, values map[string]string) (string, error) {
start := strings.Index(s, prefix)
if start == -1 {
return s, nil
func expand(s string, keys []string, prefix, postfix string, values map[string]string) (string, error) {
if len(keys) > maxExpansionDepth {
return "", fmt.Errorf("expansion too deep")
}
keyStart := start + len(prefix)
keyLen := strings.Index(s[keyStart:], postfix)
if keyLen == -1 {
return "", fmt.Errorf("malformed expression")
for {
start := strings.Index(s, prefix)
if start == -1 {
return s, nil
}
keyStart := start + len(prefix)
keyLen := strings.Index(s[keyStart:], postfix)
if keyLen == -1 {
return "", fmt.Errorf("malformed expression")
}
end := keyStart + keyLen + len(postfix) - 1
key := s[keyStart : keyStart+keyLen]
// fmt.Printf("s:%q pp:%q start:%d end:%d keyStart:%d keyLen:%d key:%q\n", s, prefix + "..." + postfix, start, end, keyStart, keyLen, key)
for _, k := range keys {
if key == k {
return "", fmt.Errorf("circular reference")
}
}
val, ok := values[key]
if !ok {
val = os.Getenv(key)
}
new_val, err := expand(val, append(keys, key), prefix, postfix, values)
if err != nil {
return "", err
}
s = s[:start] + new_val + s[end+1:]
}
end := keyStart + keyLen + len(postfix) - 1
key := s[keyStart : keyStart+keyLen]
// fmt.Printf("s:%q pp:%q start:%d end:%d keyStart:%d keyLen:%d key:%q\n", s, prefix + "..." + postfix, start, end, keyStart, keyLen, key)
if _, ok := keys[key]; ok {
return "", fmt.Errorf("circular reference")
}
val, ok := values[key]
if !ok {
val = os.Getenv(key)
}
// remember that we've seen the key
keys[key] = true
return expand(s[:start]+val+s[end+1:], keys, prefix, postfix, values)
return s, nil
}
// encode encodes a UTF-8 string to ISO-8859-1 and escapes some characters.

28
vendor/github.com/magiconair/properties/properties_test.go сгенерированный поставляемый
Просмотреть файл

@@ -90,6 +90,10 @@ var complexTests = [][]string{
{"key=value\nkey2=${key}bb", "key", "value", "key2", "valuebb"},
{"key=value\nkey2=aa${key}bb", "key", "value", "key2", "aavaluebb"},
{"key=value\nkey2=${key}\nkey3=${key2}", "key", "value", "key2", "value", "key3", "value"},
{"key=value\nkey2=${key}${key}", "key", "value", "key2", "valuevalue"},
{"key=value\nkey2=${key}${key}${key}${key}", "key", "value", "key2", "valuevaluevaluevalue"},
{"key=value\nkey2=${key}${key3}\nkey3=${key}", "key", "value", "key2", "valuevalue", "key3", "value"},
{"key=value\nkey2=${key3}${key}${key4}\nkey3=${key}\nkey4=${key}", "key", "value", "key2", "valuevaluevalue", "key3", "value", "key4", "value"},
{"key=${USER}", "key", os.Getenv("USER")},
{"key=${USER}\nUSER=value", "key", "value", "USER", "value"},
}
@@ -446,6 +450,30 @@ func TestErrors(t *testing.T) {
}
}
func TestVeryDeep(t *testing.T) {
input := "key0=value\n"
prefix := "${"
postfix := "}"
i := 0
for i = 0; i < maxExpansionDepth-1; i++ {
input += fmt.Sprintf("key%d=%skey%d%s\n", i+1, prefix, i, postfix)
}
p, err := Load([]byte(input), ISO_8859_1)
assert.Equal(t, err, nil)
p.Prefix = prefix
p.Postfix = postfix
assert.Equal(t, p.MustGet(fmt.Sprintf("key%d", i)), "value")
// Nudge input over the edge
input += fmt.Sprintf("key%d=%skey%d%s\n", i+1, prefix, i, postfix)
_, err = Load([]byte(input), ISO_8859_1)
assert.Equal(t, err != nil, true, "want error")
assert.Equal(t, strings.Contains(err.Error(), "expansion too deep"), true)
}
func TestDisableExpansion(t *testing.T) {
input := "key=value\nkey2=${key}"
p := mustParse(t, input)

4
vendor/github.com/minio/minio-go/api-compose-object.go сгенерированный поставляемый
Просмотреть файл

@@ -476,8 +476,8 @@ func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error {
// Single source object case (i.e. when only one source is
// involved, it is being copied wholly and at most 5GiB in
// size).
if totalParts == 1 && srcs[0].start == -1 && totalSize <= maxPartSize {
// size, emptyfiles are also supported).
if (totalParts == 1 && srcs[0].start == -1 && totalSize <= maxPartSize) || (totalSize == 0) {
h := srcs[0].Headers
// Add destination encryption headers
for k, v := range dst.encryption.getSSEHeaders(false) {

61
vendor/github.com/minio/minio-go/api.go сгенерированный поставляемый
Просмотреть файл

@@ -81,12 +81,25 @@ type Client struct {
// Random seed.
random *rand.Rand
// lookup indicates type of url lookup supported by server. If not specified,
// default to Auto.
lookup BucketLookupType
}
// Options for New method
type Options struct {
Creds *credentials.Credentials
Secure bool
Region string
BucketLookup BucketLookupType
// Add future fields here
}
// Global constants.
const (
libraryName = "minio-go"
libraryVersion = "4.0.6"
libraryVersion = "4.0.7"
)
// User Agent should always following the below style.
@@ -98,11 +111,21 @@ const (
libraryUserAgent = libraryUserAgentPrefix + libraryName + "/" + libraryVersion
)
// BucketLookupType is type of url lookup supported by server.
type BucketLookupType int
// Different types of url lookup supported by the server.Initialized to BucketLookupAuto
const (
BucketLookupAuto BucketLookupType = iota
BucketLookupDNS
BucketLookupPath
)
// NewV2 - instantiate minio client with Amazon S3 signature version
// '2' compatibility.
func NewV2(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Client, error) {
creds := credentials.NewStaticV2(accessKeyID, secretAccessKey, "")
clnt, err := privateNew(endpoint, creds, secure, "")
clnt, err := privateNew(endpoint, creds, secure, "", BucketLookupAuto)
if err != nil {
return nil, err
}
@@ -114,7 +137,7 @@ func NewV2(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*
// '4' compatibility.
func NewV4(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Client, error) {
creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "")
clnt, err := privateNew(endpoint, creds, secure, "")
clnt, err := privateNew(endpoint, creds, secure, "", BucketLookupAuto)
if err != nil {
return nil, err
}
@@ -125,7 +148,7 @@ func NewV4(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*
// New - instantiate minio client, adds automatic verification of signature.
func New(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Client, error) {
creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "")
clnt, err := privateNew(endpoint, creds, secure, "")
clnt, err := privateNew(endpoint, creds, secure, "", BucketLookupAuto)
if err != nil {
return nil, err
}
@@ -144,7 +167,7 @@ func New(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Client, e
// for retrieving credentials from various credentials provider such as
// IAM, File, Env etc.
func NewWithCredentials(endpoint string, creds *credentials.Credentials, secure bool, region string) (*Client, error) {
return privateNew(endpoint, creds, secure, region)
return privateNew(endpoint, creds, secure, region, BucketLookupAuto)
}
// NewWithRegion - instantiate minio client, with region configured. Unlike New(),
@@ -152,7 +175,12 @@ func NewWithCredentials(endpoint string, creds *credentials.Credentials, secure
// Use this function when if your application deals with single region.
func NewWithRegion(endpoint, accessKeyID, secretAccessKey string, secure bool, region string) (*Client, error) {
creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "")
return privateNew(endpoint, creds, secure, region)
return privateNew(endpoint, creds, secure, region, BucketLookupAuto)
}
// NewWithOptions - instantiate minio client with options
func NewWithOptions(endpoint string, opts *Options) (*Client, error) {
return privateNew(endpoint, opts.Creds, opts.Secure, opts.Region, opts.BucketLookup)
}
// lockedRandSource provides protected rand source, implements rand.Source interface.
@@ -239,7 +267,7 @@ func (c *Client) redirectHeaders(req *http.Request, via []*http.Request) error {
return nil
}
func privateNew(endpoint string, creds *credentials.Credentials, secure bool, region string) (*Client, error) {
func privateNew(endpoint string, creds *credentials.Credentials, secure bool, region string, lookup BucketLookupType) (*Client, error) {
// construct endpoint.
endpointURL, err := getEndpointURL(endpoint, secure)
if err != nil {
@@ -276,6 +304,9 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re
// Introduce a new locked random seed.
clnt.random = rand.New(&lockedRandSource{src: rand.NewSource(time.Now().UTC().UnixNano())})
// Sets bucket lookup style, whether server accepts DNS or Path lookup. Default is Auto - determined
// by the SDK. When Auto is specified, DNS lookup is used for Amazon/Google cloud endpoints and Path for all other endpoints.
clnt.lookup = lookup
// Return.
return clnt, nil
}
@@ -824,8 +855,7 @@ func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, que
// endpoint URL.
if bucketName != "" {
// Save if target url will have buckets which suppport virtual host.
isVirtualHostStyle := s3utils.IsVirtualHostSupported(*c.endpointURL, bucketName)
isVirtualHostStyle := c.isVirtualHostStyleRequest(*c.endpointURL, bucketName)
// If endpoint supports virtual host style use that always.
// Currently only S3 and Google Cloud Storage would support
// virtual host style.
@@ -850,3 +880,16 @@ func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, que
return url.Parse(urlStr)
}
// returns true if virtual hosted style requests are to be used.
func (c *Client) isVirtualHostStyleRequest(url url.URL, bucketName string) bool {
if c.lookup == BucketLookupDNS {
return true
}
if c.lookup == BucketLookupPath {
return false
}
// default to virtual only for Amazon/Google storage. In all other cases use
// path style requests
return s3utils.IsVirtualHostSupported(url, bucketName)
}

19
vendor/github.com/minio/minio-go/docs/API.md сгенерированный поставляемый
Просмотреть файл

@@ -86,16 +86,27 @@ __Parameters__
### NewWithRegion(endpoint, accessKeyID, secretAccessKey string, ssl bool, region string) (*Client, error)
Initializes minio client, with region configured. Unlike New(), NewWithRegion avoids bucket-location lookup operations and it is slightly faster. Use this function when your application deals with a single region.
### NewWithOptions(endpoint string, options *Options) (*Client, error)
Initializes minio client with options configured.
__Parameters__
|Param |Type |Description |
|:---|:---| :---|
|`endpoint` | _string_ |S3 compatible object storage endpoint |
|`accessKeyID` |_string_ |Access key for the object storage |
|`secretAccessKey` | _string_ |Secret key for the object storage |
|`ssl` | _bool_ | If 'true' API requests will be secure (HTTPS), and insecure (HTTP) otherwise |
|`region`| _string_ | Region for the object storage |
|`opts` |_minio.Options_ | Options for constructing a new client|
__minio.Options__
|Field | Type | Description |
|:--- |:--- | :--- |
| `opts.Creds` | _*credentials.Credentials_ | Access Credentials|
| `opts.Secure` | _bool_ | If 'true' API requests will be secure (HTTPS), and insecure (HTTP) otherwise |
| `opts.Region` | _string_ | region |
| `opts.BucketLookup` | _BucketLookupType_ | Bucket lookup type can be one of the following values |
| | | _minio.BucketLookupDNS_ |
| | | _minio.BucketLookupPath_ |
| | | _minio.BucketLookupAuto_ |
## 2. Bucket operations
<a name="MakeBucket"></a>

2
vendor/github.com/mitchellh/mapstructure/README.md сгенерированный поставляемый
Просмотреть файл

@@ -1,4 +1,4 @@
# mapstructure [![Godoc](https://godoc.org/github.com/mitchell/mapstructure?status.svg)](https://godoc.org/github.com/mitchell/mapstructure)
# mapstructure [![Godoc](https://godoc.org/github.com/mitchellh/mapstructure?status.svg)](https://godoc.org/github.com/mitchellh/mapstructure)
mapstructure is a Go library for decoding generic map values to structures
and vice versa, while providing helpful error handling.

19
vendor/github.com/mitchellh/mapstructure/decode_hooks.go сгенерированный поставляемый
Просмотреть файл

@@ -115,6 +115,25 @@ func StringToTimeDurationHookFunc() DecodeHookFunc {
}
}
// StringToTimeHookFunc returns a DecodeHookFunc that converts
// strings to time.Time.
func StringToTimeHookFunc(layout string) DecodeHookFunc {
return func(
f reflect.Type,
t reflect.Type,
data interface{}) (interface{}, error) {
if f.Kind() != reflect.String {
return data, nil
}
if t != reflect.TypeOf(time.Time{}) {
return data, nil
}
// Convert it by parsing
return time.Parse(layout, data.(string))
}
}
// WeaklyTypedHook is a DecodeHookFunc which adds support for weak typing to
// the decoder.
//

30
vendor/github.com/mitchellh/mapstructure/decode_hooks_test.go сгенерированный поставляемый
Просмотреть файл

@@ -153,6 +153,36 @@ func TestStringToTimeDurationHookFunc(t *testing.T) {
}
}
func TestStringToTimeHookFunc(t *testing.T) {
strType := reflect.TypeOf("")
timeType := reflect.TypeOf(time.Time{})
cases := []struct {
f, t reflect.Type
layout string
data interface{}
result interface{}
err bool
}{
{strType, timeType, time.RFC3339, "2006-01-02T15:04:05Z",
time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC), false},
{strType, timeType, time.RFC3339, "5", time.Time{}, true},
{strType, strType, time.RFC3339, "5", "5", false},
}
for i, tc := range cases {
f := StringToTimeHookFunc(tc.layout)
actual, err := DecodeHookExec(f, tc.f, tc.t, tc.data)
if tc.err != (err != nil) {
t.Fatalf("case %d: expected err %#v", i, tc.err)
}
if !reflect.DeepEqual(actual, tc.result) {
t.Fatalf(
"case %d: expected %#v, got %#v",
i, tc.result, actual)
}
}
}
func TestWeaklyTypedHook(t *testing.T) {
var f DecodeHookFunc = WeaklyTypedHook

249
vendor/github.com/mitchellh/mapstructure/mapstructure.go сгенерированный поставляемый
Просмотреть файл

@@ -114,12 +114,12 @@ type Metadata struct {
Unused []string
}
// Decode takes a map and uses reflection to convert it into the
// given Go native structure. val must be a pointer to a struct.
func Decode(m interface{}, rawVal interface{}) error {
// Decode takes an input structure and uses reflection to translate it to
// the output structure. output must be a pointer to a map or struct.
func Decode(input interface{}, output interface{}) error {
config := &DecoderConfig{
Metadata: nil,
Result: rawVal,
Result: output,
}
decoder, err := NewDecoder(config)
@@ -127,7 +127,7 @@ func Decode(m interface{}, rawVal interface{}) error {
return err
}
return decoder.Decode(m)
return decoder.Decode(input)
}
// WeakDecode is the same as Decode but is shorthand to enable
@@ -147,6 +147,40 @@ func WeakDecode(input, output interface{}) error {
return decoder.Decode(input)
}
// DecodeMetadata is the same as Decode, but is shorthand to
// enable metadata collection. See DecoderConfig for more info.
func DecodeMetadata(input interface{}, output interface{}, metadata *Metadata) error {
config := &DecoderConfig{
Metadata: metadata,
Result: output,
}
decoder, err := NewDecoder(config)
if err != nil {
return err
}
return decoder.Decode(input)
}
// WeakDecodeMetadata is the same as Decode, but is shorthand to
// enable both WeaklyTypedInput and metadata collection. See
// DecoderConfig for more info.
func WeakDecodeMetadata(input interface{}, output interface{}, metadata *Metadata) error {
config := &DecoderConfig{
Metadata: metadata,
Result: output,
WeaklyTypedInput: true,
}
decoder, err := NewDecoder(config)
if err != nil {
return err
}
return decoder.Decode(input)
}
// NewDecoder returns a new decoder for the given configuration. Once
// a decoder has been returned, the same configuration must not be used
// again.
@@ -184,70 +218,70 @@ func NewDecoder(config *DecoderConfig) (*Decoder, error) {
// Decode decodes the given raw interface to the target pointer specified
// by the configuration.
func (d *Decoder) Decode(raw interface{}) error {
return d.decode("", raw, reflect.ValueOf(d.config.Result).Elem())
func (d *Decoder) Decode(input interface{}) error {
return d.decode("", input, reflect.ValueOf(d.config.Result).Elem())
}
// Decodes an unknown data type into a specific reflection value.
func (d *Decoder) decode(name string, data interface{}, val reflect.Value) error {
if data == nil {
// If the data is nil, then we don't set anything.
func (d *Decoder) decode(name string, input interface{}, outVal reflect.Value) error {
if input == nil {
// If the input is nil, then we don't set anything.
return nil
}
dataVal := reflect.ValueOf(data)
if !dataVal.IsValid() {
// If the data value is invalid, then we just set the value
inputVal := reflect.ValueOf(input)
if !inputVal.IsValid() {
// If the input value is invalid, then we just set the value
// to be the zero value.
val.Set(reflect.Zero(val.Type()))
outVal.Set(reflect.Zero(outVal.Type()))
return nil
}
if d.config.DecodeHook != nil {
// We have a DecodeHook, so let's pre-process the data.
// We have a DecodeHook, so let's pre-process the input.
var err error
data, err = DecodeHookExec(
input, err = DecodeHookExec(
d.config.DecodeHook,
dataVal.Type(), val.Type(), data)
inputVal.Type(), outVal.Type(), input)
if err != nil {
return fmt.Errorf("error decoding '%s': %s", name, err)
}
}
var err error
dataKind := getKind(val)
switch dataKind {
inputKind := getKind(outVal)
switch inputKind {
case reflect.Bool:
err = d.decodeBool(name, data, val)
err = d.decodeBool(name, input, outVal)
case reflect.Interface:
err = d.decodeBasic(name, data, val)
err = d.decodeBasic(name, input, outVal)
case reflect.String:
err = d.decodeString(name, data, val)
err = d.decodeString(name, input, outVal)
case reflect.Int:
err = d.decodeInt(name, data, val)
err = d.decodeInt(name, input, outVal)
case reflect.Uint:
err = d.decodeUint(name, data, val)
err = d.decodeUint(name, input, outVal)
case reflect.Float32:
err = d.decodeFloat(name, data, val)
err = d.decodeFloat(name, input, outVal)
case reflect.Struct:
err = d.decodeStruct(name, data, val)
err = d.decodeStruct(name, input, outVal)
case reflect.Map:
err = d.decodeMap(name, data, val)
err = d.decodeMap(name, input, outVal)
case reflect.Ptr:
err = d.decodePtr(name, data, val)
err = d.decodePtr(name, input, outVal)
case reflect.Slice:
err = d.decodeSlice(name, data, val)
err = d.decodeSlice(name, input, outVal)
case reflect.Array:
err = d.decodeArray(name, data, val)
err = d.decodeArray(name, input, outVal)
case reflect.Func:
err = d.decodeFunc(name, data, val)
err = d.decodeFunc(name, input, outVal)
default:
// If we reached this point then we weren't able to decode it
return fmt.Errorf("%s: unsupported type: %s", name, dataKind)
return fmt.Errorf("%s: unsupported type: %s", name, inputKind)
}
// If we reached here, then we successfully decoded SOMETHING, so
// mark the key as used if we're tracking metadata.
// mark the key as used if we're tracking metainput.
if d.config.Metadata != nil && name != "" {
d.config.Metadata.Keys = append(d.config.Metadata.Keys, name)
}
@@ -258,6 +292,9 @@ func (d *Decoder) decode(name string, data interface{}, val reflect.Value) error
// This decodes a basic type (bool, int, string, etc.) and sets the
// value to "data" of that type.
func (d *Decoder) decodeBasic(name string, data interface{}, val reflect.Value) error {
if val.IsValid() && val.Elem().IsValid() {
return d.decode(name, data, val.Elem())
}
dataVal := reflect.ValueOf(data)
if !dataVal.IsValid() {
dataVal = reflect.Zero(val.Type())
@@ -499,34 +536,50 @@ func (d *Decoder) decodeMap(name string, data interface{}, val reflect.Value) er
valMap = reflect.MakeMap(mapType)
}
// Check input type
// Check input type and based on the input type jump to the proper func
dataVal := reflect.Indirect(reflect.ValueOf(data))
if dataVal.Kind() != reflect.Map {
// In weak mode, we accept a slice of maps as an input...
switch dataVal.Kind() {
case reflect.Map:
return d.decodeMapFromMap(name, dataVal, val, valMap)
case reflect.Struct:
return d.decodeMapFromStruct(name, dataVal, val, valMap)
case reflect.Array, reflect.Slice:
if d.config.WeaklyTypedInput {
switch dataVal.Kind() {
case reflect.Array, reflect.Slice:
// Special case for BC reasons (covered by tests)
if dataVal.Len() == 0 {
val.Set(valMap)
return nil
}
for i := 0; i < dataVal.Len(); i++ {
err := d.decode(
fmt.Sprintf("%s[%d]", name, i),
dataVal.Index(i).Interface(), val)
if err != nil {
return err
}
}
return nil
}
return d.decodeMapFromSlice(name, dataVal, val, valMap)
}
fallthrough
default:
return fmt.Errorf("'%s' expected a map, got '%s'", name, dataVal.Kind())
}
}
func (d *Decoder) decodeMapFromSlice(name string, dataVal reflect.Value, val reflect.Value, valMap reflect.Value) error {
// Special case for BC reasons (covered by tests)
if dataVal.Len() == 0 {
val.Set(valMap)
return nil
}
for i := 0; i < dataVal.Len(); i++ {
err := d.decode(
fmt.Sprintf("%s[%d]", name, i),
dataVal.Index(i).Interface(), val)
if err != nil {
return err
}
}
return nil
}
func (d *Decoder) decodeMapFromMap(name string, dataVal reflect.Value, val reflect.Value, valMap reflect.Value) error {
valType := val.Type()
valKeyType := valType.Key()
valElemType := valType.Elem()
// Accumulate errors
errors := make([]string, 0)
@@ -563,22 +616,88 @@ func (d *Decoder) decodeMap(name string, data interface{}, val reflect.Value) er
return nil
}
func (d *Decoder) decodeMapFromStruct(name string, dataVal reflect.Value, val reflect.Value, valMap reflect.Value) error {
typ := dataVal.Type()
for i := 0; i < typ.NumField(); i++ {
// Get the StructField first since this is a cheap operation. If the
// field is unexported, then ignore it.
f := typ.Field(i)
if f.PkgPath != "" {
continue
}
// Next get the actual value of this field and verify it is assignable
// to the map value.
v := dataVal.Field(i)
if !v.Type().AssignableTo(valMap.Type().Elem()) {
return fmt.Errorf("cannot assign type '%s' to map value field of type '%s'", v.Type(), valMap.Type().Elem())
}
// Determine the name of the key in the map
keyName := f.Name
tagValue := f.Tag.Get(d.config.TagName)
tagValue = strings.SplitN(tagValue, ",", 2)[0]
if tagValue != "" {
if tagValue == "-" {
continue
}
keyName = tagValue
}
switch v.Kind() {
// this is an embedded struct, so handle it differently
case reflect.Struct:
x := reflect.New(v.Type())
x.Elem().Set(v)
vType := valMap.Type()
vKeyType := vType.Key()
vElemType := vType.Elem()
mType := reflect.MapOf(vKeyType, vElemType)
vMap := reflect.MakeMap(mType)
err := d.decode(keyName, x.Interface(), vMap)
if err != nil {
return err
}
valMap.SetMapIndex(reflect.ValueOf(keyName), vMap)
default:
valMap.SetMapIndex(reflect.ValueOf(keyName), v)
}
}
if val.CanAddr() {
val.Set(valMap)
}
return nil
}
func (d *Decoder) decodePtr(name string, data interface{}, val reflect.Value) error {
// Create an element of the concrete (non pointer) type and decode
// into that. Then set the value of the pointer to this type.
valType := val.Type()
valElemType := valType.Elem()
realVal := val
if realVal.IsNil() || d.config.ZeroFields {
realVal = reflect.New(valElemType)
}
if val.CanSet() {
realVal := val
if realVal.IsNil() || d.config.ZeroFields {
realVal = reflect.New(valElemType)
}
if err := d.decode(name, data, reflect.Indirect(realVal)); err != nil {
return err
}
if err := d.decode(name, data, reflect.Indirect(realVal)); err != nil {
return err
}
val.Set(realVal)
val.Set(realVal)
} else {
if err := d.decode(name, data, reflect.Indirect(val)); err != nil {
return err
}
}
return nil
}
@@ -614,7 +733,8 @@ func (d *Decoder) decodeSlice(name string, data interface{}, val reflect.Value)
val.Set(reflect.MakeSlice(sliceType, 0, 0))
return nil
}
case dataValKind == reflect.String && valElemType.Kind() == reflect.Uint8:
return d.decodeSlice(name, []byte(dataVal.String()), val)
// All other types we try to convert to the slice type
// and "lift" it into it. i.e. a string becomes a string slice.
default:
@@ -622,7 +742,6 @@ func (d *Decoder) decodeSlice(name string, data interface{}, val reflect.Value)
return d.decodeSlice(name, []interface{}{data}, val)
}
}
return fmt.Errorf(
"'%s': source data must be an array or slice, got %s", name, dataValKind)

340
vendor/github.com/mitchellh/mapstructure/mapstructure_test.go сгенерированный поставляемый
Просмотреть файл

@@ -128,6 +128,7 @@ type TypeConversionResult struct {
FloatToBool bool
FloatToString string
SliceUint8ToString string
StringToSliceUint8 []byte
ArrayUint8ToString string
StringToInt int
StringToUint uint
@@ -248,6 +249,32 @@ func TestBasic_Merge(t *testing.T) {
}
}
// Test for issue #46.
func TestBasic_Struct(t *testing.T) {
t.Parallel()
input := map[string]interface{}{
"vdata": map[string]interface{}{
"vstring": "foo",
},
}
var result, inner Basic
result.Vdata = &inner
err := Decode(input, &result)
if err != nil {
t.Fatalf("got an err: %s", err)
}
expected := Basic{
Vdata: &Basic{
Vstring: "foo",
},
}
if !reflect.DeepEqual(result, expected) {
t.Fatalf("bad: %#v", result)
}
}
func TestDecode_BasicSquash(t *testing.T) {
t.Parallel()
@@ -626,6 +653,7 @@ func TestDecode_TypeConversion(t *testing.T) {
"FloatToBool": 42.42,
"FloatToString": 42.42,
"SliceUint8ToString": []uint8("foo"),
"StringToSliceUint8": "foo",
"ArrayUint8ToString": [3]uint8{'f', 'o', 'o'},
"StringToInt": "42",
"StringToUint": "42",
@@ -671,6 +699,7 @@ func TestDecode_TypeConversion(t *testing.T) {
FloatToBool: true,
FloatToString: "42.42",
SliceUint8ToString: "foo",
StringToSliceUint8: []byte("foo"),
ArrayUint8ToString: "foo",
StringToInt: 42,
StringToUint: 42,
@@ -926,6 +955,56 @@ func TestNestedTypePointer(t *testing.T) {
}
}
// Test for issue #46.
func TestNestedTypeInterface(t *testing.T) {
t.Parallel()
input := map[string]interface{}{
"vfoo": "foo",
"vbar": &map[string]interface{}{
"vstring": "foo",
"vint": 42,
"vbool": true,
"vdata": map[string]interface{}{
"vstring": "bar",
},
},
}
var result NestedPointer
result.Vbar = new(Basic)
result.Vbar.Vdata = new(Basic)
err := Decode(input, &result)
if err != nil {
t.Fatalf("got an err: %s", err.Error())
}
if result.Vfoo != "foo" {
t.Errorf("vfoo value should be 'foo': %#v", result.Vfoo)
}
if result.Vbar.Vstring != "foo" {
t.Errorf("vstring value should be 'foo': %#v", result.Vbar.Vstring)
}
if result.Vbar.Vint != 42 {
t.Errorf("vint value should be 42: %#v", result.Vbar.Vint)
}
if result.Vbar.Vbool != true {
t.Errorf("vbool value should be true: %#v", result.Vbar.Vbool)
}
if result.Vbar.Vextra != "" {
t.Errorf("vextra value should be empty: %#v", result.Vbar.Vextra)
}
if result.Vbar.Vdata.(*Basic).Vstring != "bar" {
t.Errorf("vstring value should be 'bar': %#v", result.Vbar.Vdata.(*Basic).Vstring)
}
}
func TestSlice(t *testing.T) {
t.Parallel()
@@ -1112,6 +1191,197 @@ func TestArrayToMap(t *testing.T) {
}
}
func TestMapOutputForStructuredInputs(t *testing.T) {
t.Parallel()
tests := []struct {
name string
in interface{}
target interface{}
out interface{}
wantErr bool
}{
{
"basic struct input",
&Basic{
Vstring: "vstring",
Vint: 2,
Vuint: 3,
Vbool: true,
Vfloat: 4.56,
Vextra: "vextra",
vsilent: true,
Vdata: []byte("data"),
},
&map[string]interface{}{},
&map[string]interface{}{
"Vstring": "vstring",
"Vint": 2,
"Vuint": uint(3),
"Vbool": true,
"Vfloat": 4.56,
"Vextra": "vextra",
"Vdata": []byte("data"),
"VjsonInt": 0,
"VjsonFloat": 0.0,
"VjsonNumber": json.Number(""),
},
false,
},
{
"embedded struct input",
&Embedded{
Vunique: "vunique",
Basic: Basic{
Vstring: "vstring",
Vint: 2,
Vuint: 3,
Vbool: true,
Vfloat: 4.56,
Vextra: "vextra",
vsilent: true,
Vdata: []byte("data"),
},
},
&map[string]interface{}{},
&map[string]interface{}{
"Vunique": "vunique",
"Basic": map[string]interface{}{
"Vstring": "vstring",
"Vint": 2,
"Vuint": uint(3),
"Vbool": true,
"Vfloat": 4.56,
"Vextra": "vextra",
"Vdata": []byte("data"),
"VjsonInt": 0,
"VjsonFloat": 0.0,
"VjsonNumber": json.Number(""),
},
},
false,
},
{
"slice input - should error",
[]string{"foo", "bar"},
&map[string]interface{}{},
&map[string]interface{}{},
true,
},
{
"struct with slice property",
&Slice{
Vfoo: "vfoo",
Vbar: []string{"foo", "bar"},
},
&map[string]interface{}{},
&map[string]interface{}{
"Vfoo": "vfoo",
"Vbar": []string{"foo", "bar"},
},
false,
},
{
"struct with slice of struct property",
&SliceOfStruct{
Value: []Basic{
Basic{
Vstring: "vstring",
Vint: 2,
Vuint: 3,
Vbool: true,
Vfloat: 4.56,
Vextra: "vextra",
vsilent: true,
Vdata: []byte("data"),
},
},
},
&map[string]interface{}{},
&map[string]interface{}{
"Value": []Basic{
Basic{
Vstring: "vstring",
Vint: 2,
Vuint: 3,
Vbool: true,
Vfloat: 4.56,
Vextra: "vextra",
vsilent: true,
Vdata: []byte("data"),
},
},
},
false,
},
{
"struct with map property",
&Map{
Vfoo: "vfoo",
Vother: map[string]string{"vother": "vother"},
},
&map[string]interface{}{},
&map[string]interface{}{
"Vfoo": "vfoo",
"Vother": map[string]string{
"vother": "vother",
}},
false,
},
{
"tagged struct",
&Tagged{
Extra: "extra",
Value: "value",
},
&map[string]string{},
&map[string]string{
"bar": "extra",
"foo": "value",
},
false,
},
{
"omit tag struct",
&struct {
Value string `mapstructure:"value"`
Omit string `mapstructure:"-"`
}{
Value: "value",
Omit: "omit",
},
&map[string]string{},
&map[string]string{
"value": "value",
},
false,
},
{
"decode to wrong map type",
&struct {
Value string
}{
Value: "string",
},
&map[string]int{},
&map[string]int{},
true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := Decode(tt.in, tt.target); (err != nil) != tt.wantErr {
t.Fatalf("%q: TestMapOutputForStructuredInputs() unexpected error: %s", tt.name, err)
}
if !reflect.DeepEqual(tt.out, tt.target) {
t.Fatalf("%q: TestMapOutputForStructuredInputs() expected: %#v, got: %#v", tt.name, tt.out, tt.target)
}
})
}
}
func TestInvalidType(t *testing.T) {
t.Parallel()
@@ -1171,6 +1441,39 @@ func TestInvalidType(t *testing.T) {
}
}
func TestDecodeMetadata(t *testing.T) {
t.Parallel()
input := map[string]interface{}{
"vfoo": "foo",
"vbar": map[string]interface{}{
"vstring": "foo",
"Vuint": 42,
"foo": "bar",
},
"bar": "nil",
}
var md Metadata
var result Nested
err := DecodeMetadata(input, &result, &md)
if err != nil {
t.Fatalf("err: %s", err.Error())
}
expectedKeys := []string{"Vbar", "Vbar.Vstring", "Vbar.Vuint", "Vfoo"}
sort.Strings(md.Keys)
if !reflect.DeepEqual(md.Keys, expectedKeys) {
t.Fatalf("bad keys: %#v", md.Keys)
}
expectedUnused := []string{"Vbar.foo", "bar"}
if !reflect.DeepEqual(md.Unused, expectedUnused) {
t.Fatalf("bad unused: %#v", md.Unused)
}
}
func TestMetadata(t *testing.T) {
t.Parallel()
@@ -1311,6 +1614,43 @@ func TestWeakDecode(t *testing.T) {
}
}
func TestWeakDecodeMetadata(t *testing.T) {
t.Parallel()
input := map[string]interface{}{
"foo": "4",
"bar": "value",
"unused": "value",
}
var md Metadata
var result struct {
Foo int
Bar string
}
if err := WeakDecodeMetadata(input, &result, &md); err != nil {
t.Fatalf("err: %s", err)
}
if result.Foo != 4 {
t.Fatalf("bad: %#v", result)
}
if result.Bar != "value" {
t.Fatalf("bad: %#v", result)
}
expectedKeys := []string{"Bar", "Foo"}
sort.Strings(md.Keys)
if !reflect.DeepEqual(md.Keys, expectedKeys) {
t.Fatalf("bad keys: %#v", md.Keys)
}
expectedUnused := []string{"unused"}
if !reflect.DeepEqual(md.Unused, expectedUnused) {
t.Fatalf("bad unused: %#v", md.Unused)
}
}
func testSliceInput(t *testing.T, input map[string]interface{}, expected *Slice) {
var result Slice
err := Decode(input, &result)

2
vendor/github.com/olivere/elastic/.travis.yml сгенерированный поставляемый
Просмотреть файл

@@ -12,4 +12,4 @@ services:
- docker
before_install:
- sudo sysctl -w vm.max_map_count=262144
- docker run -d --rm -p 9200:9200 -e "http.host=0.0.0.0" -e "transport.host=127.0.0.1" -e "bootstrap.memory_lock=true" -e "ES_JAVA_OPTS=-Xms1g -Xmx1g" docker.elastic.co/elasticsearch/elasticsearch:6.1.2 elasticsearch -Expack.security.enabled=false -Enetwork.host=_local_,_site_ -Enetwork.publish_host=_local_
- docker run -d --rm -p 9200:9200 -e "http.host=0.0.0.0" -e "transport.host=127.0.0.1" -e "bootstrap.memory_lock=true" -e "ES_JAVA_OPTS=-Xms1g -Xmx1g" docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.1 elasticsearch -Enetwork.host=_local_,_site_ -Enetwork.publish_host=_local_

11
vendor/github.com/olivere/elastic/CONTRIBUTORS сгенерированный поставляемый
Просмотреть файл

@@ -68,9 +68,11 @@ Joe Buck [@four2five](https://github.com/four2five)
John Barker [@j16r](https://github.com/j16r)
John Goodall [@jgoodall](https://github.com/jgoodall)
John Stanford [@jxstanford](https://github.com/jxstanford)
Jonas Groenaas Drange [@semafor](https://github.com/semafor)
Josh Chorlton [@jchorl](https://github.com/jchorl)
jun [@coseyo](https://github.com/coseyo)
Junpei Tsuji [@jun06t](https://github.com/jun06t)
kartlee [@kartlee](https://github.com/kartlee)
Keith Hatton [@khatton-ft](https://github.com/khatton-ft)
kel [@liketic](https://github.com/liketic)
Kenta SUZUKI [@suzuken](https://github.com/suzuken)
@@ -98,10 +100,13 @@ Orne Brocaar [@brocaar](https://github.com/brocaar)
Paul [@eyeamera](https://github.com/eyeamera)
Pete C [@peteclark-ft](https://github.com/peteclark-ft)
Radoslaw Wesolowski [r--w](https://github.com/r--w)
Roman Colohanin [@zuzmic](https://github.com/zuzmic)
Ryan Schmukler [@rschmukler](https://github.com/rschmukler)
Ryan Wynn [@rwynn](https://github.com/rwynn)
Sacheendra talluri [@sacheendra](https://github.com/sacheendra)
Sean DuBois [@Sean-Der](https://github.com/Sean-Der)
Shalin LK [@shalinlk](https://github.com/shalinlk)
singham [@zhaochenxiao90](https://github.com/zhaochenxiao90)
Stephen Kubovic [@stephenkubovic](https://github.com/stephenkubovic)
Stuart Warren [@Woz](https://github.com/stuart-warren)
Sulaiman [@salajlan](https://github.com/salajlan)
@@ -111,13 +116,13 @@ Take [ww24](https://github.com/ww24)
Tetsuya Morimoto [@t2y](https://github.com/t2y)
TimeEmit [@TimeEmit](https://github.com/timeemit)
TusharM [@tusharm](https://github.com/tusharm)
zhangxin [@visaxin](https://github.com/visaxin)
wangtuo [@wangtuo](https://github.com/wangtuo)
Wédney Yuri [@wedneyyuri](https://github.com/wedneyyuri)
wolfkdy [@wolfkdy](https://github.com/wolfkdy)
Wyndham Blanton [@wyndhblb](https://github.com/wyndhblb)
Yarden Bar [@ayashjorden](https://github.com/ayashjorden)
zakthomas [@zakthomas](https://github.com/zakthomas)
singham [@zhaochenxiao90](https://github.com/zhaochenxiao90)
Yuya Kusakabe [@higebu](https://github.com/higebu)
Zach [@snowzach](https://github.com/snowzach)
zhangxin [@visaxin](https://github.com/visaxin)
@林 [@zplzpl](https://github.com/zplzpl)
Roman Colohanin [@zuzmic](https://github.com/zuzmic)

16
vendor/github.com/olivere/elastic/README.md сгенерированный поставляемый
Просмотреть файл

@@ -199,6 +199,7 @@ See the [wiki](https://github.com/olivere/elastic/wiki) for more details.
- [x] Significant Terms
- [x] Significant Text
- [x] Terms
- [x] Composite
- Pipeline Aggregations
- [x] Avg Bucket
- [x] Derivative
@@ -212,6 +213,7 @@ See the [wiki](https://github.com/olivere/elastic/wiki) for more details.
- [x] Cumulative Sum
- [x] Bucket Script
- [x] Bucket Selector
- [ ] Bucket Sort
- [x] Serial Differencing
- [x] Matrix Aggregations
- [x] Matrix Stats
@@ -234,17 +236,17 @@ See the [wiki](https://github.com/olivere/elastic/wiki) for more details.
- [x] Update Indices Settings
- [x] Get Settings
- [x] Analyze
- [x] Explain Analyze
- [x] Index Templates
- [ ] Shadow Replica Indices
- [x] Indices Stats
- [x] Indices Segments
- [ ] Indices Recovery
- [ ] Indices Shard Stores
- [ ] Clear Cache
- [x] Flush
- [x] Synced Flush
- [x] Refresh
- [x] Force Merge
- [ ] Upgrade
### cat APIs
@@ -267,6 +269,7 @@ The cat APIs are not implemented as of now. We think they are better suited for
- [ ] cat shards
- [ ] cat segments
- [ ] cat snapshots
- [ ] cat templates
### Cluster APIs
@@ -278,6 +281,8 @@ The cat APIs are not implemented as of now. We think they are better suited for
- [ ] Cluster Update Settings
- [x] Nodes Stats
- [x] Nodes Info
- [ ] Nodes Feature Usage
- [ ] Remote Cluster Info
- [x] Task Management API
- [ ] Nodes hot_threads
- [ ] Cluster Allocation Explain API
@@ -297,6 +302,7 @@ The cat APIs are not implemented as of now. We think they are better suited for
- Term level queries
- [x] Term Query
- [x] Terms Query
- [x] Terms Set Query
- [x] Range Query
- [x] Exists Query
- [x] Prefix Query
@@ -311,7 +317,6 @@ The cat APIs are not implemented as of now. We think they are better suited for
- [x] Dis Max Query
- [x] Function Score Query
- [x] Boosting Query
- [x] Indices Query
- Joining queries
- [x] Nested Query
- [x] Has Child Query
@@ -321,12 +326,9 @@ The cat APIs are not implemented as of now. We think they are better suited for
- [ ] GeoShape Query
- [x] Geo Bounding Box Query
- [x] Geo Distance Query
- [ ] Geo Distance Range Query
- [x] Geo Polygon Query
- [ ] Geohash Cell Query
- Specialized queries
- [x] More Like This Query
- [x] Template Query
- [x] Script Query
- [x] Percolate Query
- Span queries
@@ -346,7 +348,7 @@ The cat APIs are not implemented as of now. We think they are better suited for
- Snapshot and Restore
- [x] Repositories
- [ ] Snapshot
- [x] Snapshot
- [ ] Restore
- [ ] Snapshot status
- [ ] Monitoring snapshot/restore status

61
vendor/github.com/olivere/elastic/bulk_processor.go сгенерированный поставляемый
Просмотреть файл

@@ -6,6 +6,7 @@ package elastic
import (
"context"
"net"
"sync"
"sync/atomic"
"time"
@@ -121,7 +122,7 @@ func (s *BulkProcessorService) Stats(wantStats bool) *BulkProcessorService {
return s
}
// Set the backoff strategy to use for errors
// Backoff sets the backoff strategy to use for errors.
func (s *BulkProcessorService) Backoff(backoff Backoff) *BulkProcessorService {
s.backoff = backoff
return s
@@ -248,6 +249,8 @@ type BulkProcessor struct {
statsMu sync.Mutex // guards the following block
stats *BulkProcessorStats
stopReconnC chan struct{} // channel to signal stop reconnection attempts
}
func newBulkProcessor(
@@ -293,6 +296,7 @@ func (p *BulkProcessor) Start(ctx context.Context) error {
p.requestsC = make(chan BulkableRequest)
p.executionId = 0
p.stats = newBulkProcessorStats(p.numWorkers)
p.stopReconnC = make(chan struct{})
// Create and start up workers.
p.workers = make([]*bulkWorker, p.numWorkers)
@@ -331,6 +335,12 @@ func (p *BulkProcessor) Close() error {
return nil
}
// Tell connection checkers to stop
if p.stopReconnC != nil {
close(p.stopReconnC)
p.stopReconnC = nil
}
// Stop flusher (if enabled)
if p.flusherStopC != nil {
p.flusherStopC <- struct{}{}
@@ -436,29 +446,43 @@ func (w *bulkWorker) work(ctx context.Context) {
var stop bool
for !stop {
var err error
select {
case req, open := <-w.p.requestsC:
if open {
// Received a new request
w.service.Add(req)
if w.commitRequired() {
w.commit(ctx) // TODO swallow errors here?
err = w.commit(ctx)
}
} else {
// Channel closed: Stop.
stop = true
if w.service.NumberOfActions() > 0 {
w.commit(ctx) // TODO swallow errors here?
err = w.commit(ctx)
}
}
case <-w.flushC:
// Commit outstanding requests
if w.service.NumberOfActions() > 0 {
w.commit(ctx) // TODO swallow errors here?
err = w.commit(ctx)
}
w.flushAckC <- struct{}{}
}
if !stop && err != nil {
waitForActive := func() {
// Add back pressure to prevent Add calls from filling up the request queue
ready := make(chan struct{})
go w.waitForActiveConnection(ready)
<-ready
}
if _, ok := err.(net.Error); ok {
waitForActive()
} else if IsConnErr(err) {
waitForActive()
}
}
}
}
@@ -511,6 +535,35 @@ func (w *bulkWorker) commit(ctx context.Context) error {
return err
}
func (w *bulkWorker) waitForActiveConnection(ready chan<- struct{}) {
defer close(ready)
t := time.NewTicker(5 * time.Second)
defer t.Stop()
client := w.p.c
stopReconnC := w.p.stopReconnC
w.p.c.errorf("elastic: bulk processor %q is waiting for an active connection", w.p.name)
// loop until a health check finds at least 1 active connection or the reconnection channel is closed
for {
select {
case _, ok := <-stopReconnC:
if !ok {
w.p.c.errorf("elastic: bulk processor %q active connection check interrupted", w.p.name)
return
}
case <-t.C:
client.healthcheck(time.Duration(3)*time.Second, true)
if client.mustActiveConn() == nil {
// found an active connection
// exit and signal done to the WaitGroup
return
}
}
}
}
func (w *bulkWorker) updateStats(res *BulkResponse) {
// Update stats
if res != nil {

8
vendor/github.com/olivere/elastic/client.go сгенерированный поставляемый
Просмотреть файл

@@ -26,7 +26,7 @@ import (
const (
// Version is the current version of Elastic.
Version = "6.1.4"
Version = "6.1.7"
// DefaultURL is the default endpoint of Elasticsearch on the local machine.
// It is used e.g. when initializing a new Client without a specific URL.
@@ -1778,9 +1778,3 @@ func (c *Client) WaitForGreenStatus(timeout string) error {
func (c *Client) WaitForYellowStatus(timeout string) error {
return c.WaitForStatus("yellow", timeout)
}
// IsConnError unwraps the given error value and checks if it is equal to
// elastic.ErrNoClient.
func IsConnErr(err error) bool {
return errors.Cause(err) == ErrNoClient
}

8
vendor/github.com/olivere/elastic/errors.go сгенерированный поставляемый
Просмотреть файл

@@ -9,6 +9,8 @@ import (
"fmt"
"io/ioutil"
"net/http"
"github.com/pkg/errors"
)
// checkResponse will return an error if the request/response indicates
@@ -89,6 +91,12 @@ func (e *Error) Error() string {
}
}
// IsConnErr returns true if the error indicates that Elastic could not
// find an Elasticsearch host to connect to.
func IsConnErr(err error) bool {
return err == ErrNoClient || errors.Cause(err) == ErrNoClient
}
// IsNotFound returns true if the given error indicates that Elasticsearch
// returned HTTP status 404. The err parameter can be of type *elastic.Error,
// elastic.Error, *http.Response or int (indicating the HTTP status code).

39
vendor/github.com/olivere/elastic/msearch.go сгенерированный поставляемый
Просмотреть файл

@@ -14,19 +14,17 @@ import (
// MultiSearch executes one or more searches in one roundtrip.
type MultiSearchService struct {
client *Client
requests []*SearchRequest
indices []string
pretty bool
routing string
preference string
client *Client
requests []*SearchRequest
indices []string
pretty bool
maxConcurrentRequests *int
preFilterShardSize *int
}
func NewMultiSearchService(client *Client) *MultiSearchService {
builder := &MultiSearchService{
client: client,
requests: make([]*SearchRequest, 0),
indices: make([]string, 0),
client: client,
}
return builder
}
@@ -46,6 +44,16 @@ func (s *MultiSearchService) Pretty(pretty bool) *MultiSearchService {
return s
}
func (s *MultiSearchService) MaxConcurrentSearches(max int) *MultiSearchService {
s.maxConcurrentRequests = &max
return s
}
func (s *MultiSearchService) PreFilterShardSize(size int) *MultiSearchService {
s.preFilterShardSize = &size
return s
}
func (s *MultiSearchService) Do(ctx context.Context) (*MultiSearchResult, error) {
// Build url
path := "/_msearch"
@@ -55,6 +63,12 @@ func (s *MultiSearchService) Do(ctx context.Context) (*MultiSearchResult, error)
if s.pretty {
params.Set("pretty", fmt.Sprintf("%v", s.pretty))
}
if v := s.maxConcurrentRequests; v != nil {
params.Set("max_concurrent_searches", fmt.Sprintf("%v", *v))
}
if v := s.preFilterShardSize; v != nil {
params.Set("pre_filter_shard_size", fmt.Sprintf("%v", *v))
}
// Set body
var lines []string
@@ -68,14 +82,14 @@ func (s *MultiSearchService) Do(ctx context.Context) (*MultiSearchResult, error)
if err != nil {
return nil, err
}
body, err := json.Marshal(sr.Body())
body, err := sr.Body()
if err != nil {
return nil, err
}
lines = append(lines, string(header))
lines = append(lines, string(body))
lines = append(lines, body)
}
body := strings.Join(lines, "\n") + "\n" // Don't forget trailing \n
body := strings.Join(lines, "\n") + "\n" // add trailing \n
// Get response
res, err := s.client.PerformRequest(ctx, PerformRequestOptions{
@@ -96,6 +110,7 @@ func (s *MultiSearchService) Do(ctx context.Context) (*MultiSearchResult, error)
return ret, nil
}
// MultiSearchResult is the outcome of running a multi-search operation.
type MultiSearchResult struct {
Responses []*SearchResult `json:"responses,omitempty"`
}

105
vendor/github.com/olivere/elastic/msearch_test.go сгенерированный поставляемый
Просмотреть файл

@@ -13,6 +13,7 @@ import (
func TestMultiSearch(t *testing.T) {
client := setupTestClientAndCreateIndex(t)
// client := setupTestClientAndCreateIndexAndLog(t)
tweet1 := tweet{
User: "olivere",
@@ -60,6 +61,110 @@ func TestMultiSearch(t *testing.T) {
sreq2 := NewSearchRequest().Index(testIndexName).Type("doc").
Source(NewSearchSource().Query(q2))
searchResult, err := client.MultiSearch().
Add(sreq1, sreq2).
Pretty(true).
Do(context.TODO())
if err != nil {
t.Fatal(err)
}
if searchResult.Responses == nil {
t.Fatal("expected responses != nil; got nil")
}
if len(searchResult.Responses) != 2 {
t.Fatalf("expected 2 responses; got %d", len(searchResult.Responses))
}
sres := searchResult.Responses[0]
if sres.Hits == nil {
t.Errorf("expected Hits != nil; got nil")
}
if sres.Hits.TotalHits != 3 {
t.Errorf("expected Hits.TotalHits = %d; got %d", 3, sres.Hits.TotalHits)
}
if len(sres.Hits.Hits) != 3 {
t.Errorf("expected len(Hits.Hits) = %d; got %d", 3, len(sres.Hits.Hits))
}
for _, hit := range sres.Hits.Hits {
if hit.Index != testIndexName {
t.Errorf("expected Hits.Hit.Index = %q; got %q", testIndexName, hit.Index)
}
item := make(map[string]interface{})
err := json.Unmarshal(*hit.Source, &item)
if err != nil {
t.Fatal(err)
}
}
sres = searchResult.Responses[1]
if sres.Hits == nil {
t.Errorf("expected Hits != nil; got nil")
}
if sres.Hits.TotalHits != 2 {
t.Errorf("expected Hits.TotalHits = %d; got %d", 2, sres.Hits.TotalHits)
}
if len(sres.Hits.Hits) != 2 {
t.Errorf("expected len(Hits.Hits) = %d; got %d", 2, len(sres.Hits.Hits))
}
for _, hit := range sres.Hits.Hits {
if hit.Index != testIndexName {
t.Errorf("expected Hits.Hit.Index = %q; got %q", testIndexName, hit.Index)
}
item := make(map[string]interface{})
err := json.Unmarshal(*hit.Source, &item)
if err != nil {
t.Fatal(err)
}
}
}
func TestMultiSearchWithStrings(t *testing.T) {
client := setupTestClientAndCreateIndex(t)
// client := setupTestClientAndCreateIndexAndLog(t)
tweet1 := tweet{
User: "olivere",
Message: "Welcome to Golang and Elasticsearch.",
Tags: []string{"golang", "elasticsearch"},
}
tweet2 := tweet{
User: "olivere",
Message: "Another unrelated topic.",
Tags: []string{"golang"},
}
tweet3 := tweet{
User: "sandrae",
Message: "Cycling is fun.",
Tags: []string{"sports", "cycling"},
}
// Add all documents
_, err := client.Index().Index(testIndexName).Type("doc").Id("1").BodyJson(&tweet1).Do(context.TODO())
if err != nil {
t.Fatal(err)
}
_, err = client.Index().Index(testIndexName).Type("doc").Id("2").BodyJson(&tweet2).Do(context.TODO())
if err != nil {
t.Fatal(err)
}
_, err = client.Index().Index(testIndexName).Type("doc").Id("3").BodyJson(&tweet3).Do(context.TODO())
if err != nil {
t.Fatal(err)
}
_, err = client.Flush().Index(testIndexName).Do(context.TODO())
if err != nil {
t.Fatal(err)
}
// Spawn two search queries with one roundtrip
sreq1 := NewSearchRequest().Index(testIndexName, testIndexName2).
Source(`{"query":{"match_all":{}}}`)
sreq2 := NewSearchRequest().Index(testIndexName).Type("doc").
Source(`{"query":{"term":{"tags":"golang"}}}`)
searchResult, err := client.MultiSearch().
Add(sreq1, sreq2).
Do(context.TODO())

149
vendor/github.com/olivere/elastic/recipes/bulk_processor/main.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,149 @@
// Copyright 2012-present Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.
// BulkProcessor runs a bulk processing job that fills an index
// given certain criteria like flush interval etc.
//
// Example
//
// bulk_processor -url=http://127.0.0.1:9200/bulk-processor-test?sniff=false -n=100000 -flush-interval=1s
//
package main
import (
"context"
"flag"
"fmt"
"log"
"math/rand"
"os"
"os/signal"
"sync/atomic"
"syscall"
"time"
"github.com/google/uuid"
"github.com/olivere/elastic"
"github.com/olivere/elastic/config"
)
func main() {
var (
url = flag.String("url", "http://localhost:9200/bulk-processor-test", "Elasticsearch URL")
numWorkers = flag.Int("num-workers", 4, "Number of workers")
n = flag.Int64("n", -1, "Number of documents to process (-1 for unlimited)")
flushInterval = flag.Duration("flush-interval", 1*time.Second, "Flush interval")
bulkActions = flag.Int("bulk-actions", 0, "Number of bulk actions before committing")
bulkSize = flag.Int("bulk-size", 0, "Size of bulk requests before committing")
)
flag.Parse()
log.SetFlags(0)
rand.Seed(time.Now().UnixNano())
// Parse configuration from URL
cfg, err := config.Parse(*url)
if err != nil {
log.Fatal(err)
}
// Create an Elasticsearch client from the parsed config
client, err := elastic.NewClientFromConfig(cfg)
if err != nil {
log.Fatal(err)
}
// Drop old index
exists, err := client.IndexExists(cfg.Index).Do(context.Background())
if err != nil {
log.Fatal(err)
}
if exists {
_, err = client.DeleteIndex(cfg.Index).Do(context.Background())
if err != nil {
log.Fatal(err)
}
}
// Create processor
bulkp := elastic.NewBulkProcessorService(client).
Name("bulk-test-processor").
Stats(true).
Backoff(elastic.StopBackoff{}).
FlushInterval(*flushInterval).
Workers(*numWorkers)
if *bulkActions > 0 {
bulkp = bulkp.BulkActions(*bulkActions)
}
if *bulkSize > 0 {
bulkp = bulkp.BulkSize(*bulkSize)
}
p, err := bulkp.Do(context.Background())
if err != nil {
log.Fatal(err)
}
var created int64
errc := make(chan error, 1)
go func() {
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
<-c
errc <- nil
}()
go func() {
defer func() {
if err := p.Close(); err != nil {
errc <- err
}
}()
type Doc struct {
Timestamp time.Time `json:"@timestamp"`
}
for {
current := atomic.AddInt64(&created, 1)
if *n > 0 && current >= *n {
errc <- nil
return
}
r := elastic.NewBulkIndexRequest().
Index(cfg.Index).
Type("doc").
Id(uuid.New().String()).
Doc(Doc{Timestamp: time.Now()})
p.Add(r)
time.Sleep(time.Duration(rand.Intn(1000)) * time.Microsecond)
}
}()
go func() {
t := time.NewTicker(1 * time.Second)
defer t.Stop()
for range t.C {
stats := p.Stats()
written := atomic.LoadInt64(&created)
var queued int64
for _, w := range stats.Workers {
queued += w.Queued
}
fmt.Printf("Queued=%5d Written=%8d Succeeded=%8d Failed=%8d Comitted=%6d Flushed=%6d\n",
queued,
written,
stats.Succeeded,
stats.Failed,
stats.Committed,
stats.Flushed,
)
}
}()
if err := <-errc; err != nil {
log.Fatal(err)
}
}

10
vendor/github.com/olivere/elastic/reindex.go сгенерированный поставляемый
Просмотреть файл

@@ -20,6 +20,7 @@ type ReindexService struct {
waitForActiveShards string
waitForCompletion *bool
requestsPerSecond *int
slices *int
body interface{}
source *ReindexSource
destination *ReindexDestination
@@ -51,6 +52,12 @@ func (s *ReindexService) RequestsPerSecond(requestsPerSecond int) *ReindexServic
return s
}
// Slices specifies the number of slices this task should be divided into. Defaults to 1.
func (s *ReindexService) Slices(slices int) *ReindexService {
s.slices = &slices
return s
}
// Refresh indicates whether Elasticsearch should refresh the effected indexes
// immediately.
func (s *ReindexService) Refresh(refresh string) *ReindexService {
@@ -179,6 +186,9 @@ func (s *ReindexService) buildURL() (string, url.Values, error) {
if s.requestsPerSecond != nil {
params.Set("requests_per_second", fmt.Sprintf("%v", *s.requestsPerSecond))
}
if s.slices != nil {
params.Set("slices", fmt.Sprintf("%v", *s.slices))
}
if s.waitForActiveShards != "" {
params.Set("wait_for_active_shards", s.waitForActiveShards)
}

4
vendor/github.com/olivere/elastic/run-es.sh сгенерированный поставляемый
Просмотреть файл

@@ -1,3 +1,3 @@
#!/bin/sh
VERSION=${VERSION:=6.1.2}
docker run --rm -p 9200:9200 -e "http.host=0.0.0.0" -e "transport.host=127.0.0.1" -e "bootstrap.memory_lock=true" -e "ES_JAVA_OPTS=-Xms1g -Xmx1g" docker.elastic.co/elasticsearch/elasticsearch:$VERSION elasticsearch -Expack.security.enabled=false -Enetwork.host=_local_,_site_ -Enetwork.publish_host=_local_
VERSION=${VERSION:=6.2.1}
docker run --rm -p 9200:9200 -e "http.host=0.0.0.0" -e "transport.host=127.0.0.1" -e "bootstrap.memory_lock=true" -e "ES_JAVA_OPTS=-Xms1g -Xmx1g" docker.elastic.co/elasticsearch/elasticsearch-oss:$VERSION elasticsearch -Enetwork.host=_local_,_site_ -Enetwork.publish_host=_local_

3
vendor/github.com/olivere/elastic/search.go сгенерированный поставляемый
Просмотреть файл

@@ -111,8 +111,7 @@ func (s *SearchService) TimeoutInMillis(timeoutInMillis int) *SearchService {
}
// SearchType sets the search operation type. Valid values are:
// "query_then_fetch", "query_and_fetch", "dfs_query_then_fetch",
// "dfs_query_and_fetch", "count", "scan".
// "dfs_query_then_fetch" and "query_then_fetch".
// See https://www.elastic.co/guide/en/elasticsearch/reference/6.0/search-request-search-type.html
// for details.
func (s *SearchService) SearchType(searchType string) *SearchService {

70
vendor/github.com/olivere/elastic/search_aggs.go сгенерированный поставляемый
Просмотреть файл

@@ -653,6 +653,23 @@ func (a Aggregations) SerialDiff(name string) (*AggregationPipelineSimpleValue,
return nil, false
}
// Composite returns composite bucket aggregation results.
//
// See https://www.elastic.co/guide/en/elasticsearch/reference/6.1/search-aggregations-bucket-composite-aggregation.html
// for details.
func (a Aggregations) Composite(name string) (*AggregationBucketCompositeItems, bool) {
if raw, found := a[name]; found {
agg := new(AggregationBucketCompositeItems)
if raw == nil {
return agg, true
}
if err := json.Unmarshal(*raw, agg); err == nil {
return agg, true
}
}
return nil, false
}
// -- Single value metric --
// AggregationValueMetric is a single-value metric, returned e.g. by a
@@ -1448,3 +1465,56 @@ func (a *AggregationPipelinePercentilesMetric) UnmarshalJSON(data []byte) error
a.Aggregations = aggs
return nil
}
// -- Composite key items --
// AggregationBucketCompositeItems implements the response structure
// for a bucket aggregation of type composite.
type AggregationBucketCompositeItems struct {
Aggregations
Buckets []*AggregationBucketCompositeItem //`json:"buckets"`
Meta map[string]interface{} // `json:"meta,omitempty"`
}
// UnmarshalJSON decodes JSON data and initializes an AggregationBucketCompositeItems structure.
func (a *AggregationBucketCompositeItems) UnmarshalJSON(data []byte) error {
var aggs map[string]*json.RawMessage
if err := json.Unmarshal(data, &aggs); err != nil {
return err
}
if v, ok := aggs["buckets"]; ok && v != nil {
json.Unmarshal(*v, &a.Buckets)
}
if v, ok := aggs["meta"]; ok && v != nil {
json.Unmarshal(*v, &a.Meta)
}
a.Aggregations = aggs
return nil
}
// AggregationBucketCompositeItem is a single bucket of an AggregationBucketCompositeItems structure.
type AggregationBucketCompositeItem struct {
Aggregations
Key map[string]interface{} //`json:"key"`
DocCount int64 //`json:"doc_count"`
}
// UnmarshalJSON decodes JSON data and initializes an AggregationBucketCompositeItem structure.
func (a *AggregationBucketCompositeItem) UnmarshalJSON(data []byte) error {
var aggs map[string]*json.RawMessage
dec := json.NewDecoder(bytes.NewReader(data))
dec.UseNumber()
if err := dec.Decode(&aggs); err != nil {
return err
}
if v, ok := aggs["key"]; ok && v != nil {
json.Unmarshal(*v, &a.Key)
}
if v, ok := aggs["doc_count"]; ok && v != nil {
json.Unmarshal(*v, &a.DocCount)
}
a.Aggregations = aggs
return nil
}

498
vendor/github.com/olivere/elastic/search_aggs_bucket_composite.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,498 @@
// Copyright 2012-present Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.
package elastic
// CompositeAggregation is a multi-bucket values source based aggregation
// that can be used to calculate unique composite values from source documents.
//
// See https://www.elastic.co/guide/en/elasticsearch/reference/6.1/search-aggregations-bucket-composite-aggregation.html
// for details.
type CompositeAggregation struct {
after map[string]interface{}
size *int
sources []CompositeAggregationValuesSource
subAggregations map[string]Aggregation
meta map[string]interface{}
}
// NewCompositeAggregation creates a new CompositeAggregation.
func NewCompositeAggregation() *CompositeAggregation {
return &CompositeAggregation{
sources: make([]CompositeAggregationValuesSource, 0),
subAggregations: make(map[string]Aggregation),
}
}
// Size represents the number of composite buckets to return.
// Defaults to 10 as of Elasticsearch 6.1.
func (a *CompositeAggregation) Size(size int) *CompositeAggregation {
a.size = &size
return a
}
// AggregateAfter sets the values that indicate which composite bucket this
// request should "aggregate after".
func (a *CompositeAggregation) AggregateAfter(after map[string]interface{}) *CompositeAggregation {
a.after = after
return a
}
// Sources specifies the list of CompositeAggregationValuesSource instances to
// use in the aggregation.
func (a *CompositeAggregation) Sources(sources ...CompositeAggregationValuesSource) *CompositeAggregation {
a.sources = append(a.sources, sources...)
return a
}
// SubAggregations of this aggregation.
func (a *CompositeAggregation) SubAggregation(name string, subAggregation Aggregation) *CompositeAggregation {
a.subAggregations[name] = subAggregation
return a
}
// Meta sets the meta data to be included in the aggregation response.
func (a *CompositeAggregation) Meta(metaData map[string]interface{}) *CompositeAggregation {
a.meta = metaData
return a
}
// Source returns the serializable JSON for this aggregation.
func (a *CompositeAggregation) Source() (interface{}, error) {
// Example:
// {
// "aggs" : {
// "my_composite_agg" : {
// "composite" : {
// "sources": [
// {"my_term": { "terms": { "field": "product" }}},
// {"my_histo": { "histogram": { "field": "price", "interval": 5 }}},
// {"my_date": { "date_histogram": { "field": "timestamp", "interval": "1d" }}},
// ],
// "size" : 10,
// "after" : ["a", 2, "c"]
// }
// }
// }
// }
//
// This method returns only the { "histogram" : { ... } } part.
source := make(map[string]interface{})
opts := make(map[string]interface{})
source["composite"] = opts
sources := make([]interface{}, len(a.sources))
for i, s := range a.sources {
src, err := s.Source()
if err != nil {
return nil, err
}
sources[i] = src
}
opts["sources"] = sources
if a.size != nil {
opts["size"] = *a.size
}
if a.after != nil {
opts["after"] = a.after
}
// AggregationBuilder (SubAggregations)
if len(a.subAggregations) > 0 {
aggsMap := make(map[string]interface{})
source["aggregations"] = aggsMap
for name, aggregate := range a.subAggregations {
src, err := aggregate.Source()
if err != nil {
return nil, err
}
aggsMap[name] = src
}
}
// Add Meta data if available
if len(a.meta) > 0 {
source["meta"] = a.meta
}
return source, nil
}
// -- Generic interface for CompositeAggregationValues --
// CompositeAggregationValuesSource specifies the interface that
// all implementations for CompositeAggregation's Sources method
// need to implement.
//
// The different implementations are described in
// https://www.elastic.co/guide/en/elasticsearch/reference/6.1/search-aggregations-bucket-composite-aggregation.html#_values_source_2.
type CompositeAggregationValuesSource interface {
Source() (interface{}, error)
}
// -- CompositeAggregationTermsValuesSource --
// CompositeAggregationTermsValuesSource is a source for the CompositeAggregation that handles terms
// it works very similar to a terms aggregation with slightly different syntax
//
// See https://www.elastic.co/guide/en/elasticsearch/reference/6.1/search-aggregations-bucket-composite-aggregation.html#_terms
// for details.
type CompositeAggregationTermsValuesSource struct {
name string
field string
script *Script
valueType string
missing interface{}
order string
}
// NewCompositeAggregationTermsValuesSource creates and initializes
// a new CompositeAggregationTermsValuesSource.
func NewCompositeAggregationTermsValuesSource(name string) *CompositeAggregationTermsValuesSource {
return &CompositeAggregationTermsValuesSource{
name: name,
}
}
// Field to use for this source.
func (a *CompositeAggregationTermsValuesSource) Field(field string) *CompositeAggregationTermsValuesSource {
a.field = field
return a
}
// Script to use for this source.
func (a *CompositeAggregationTermsValuesSource) Script(script *Script) *CompositeAggregationTermsValuesSource {
a.script = script
return a
}
// ValueType specifies the type of values produced by this source,
// e.g. "string" or "date".
func (a *CompositeAggregationTermsValuesSource) ValueType(valueType string) *CompositeAggregationTermsValuesSource {
a.valueType = valueType
return a
}
// Order specifies the order in the values produced by this source.
// It can be either "asc" or "desc".
func (a *CompositeAggregationTermsValuesSource) Order(order string) *CompositeAggregationTermsValuesSource {
a.order = order
return a
}
// Asc ensures the order of the values produced is ascending.
func (a *CompositeAggregationTermsValuesSource) Asc() *CompositeAggregationTermsValuesSource {
a.order = "asc"
return a
}
// Desc ensures the order of the values produced is descending.
func (a *CompositeAggregationTermsValuesSource) Desc() *CompositeAggregationTermsValuesSource {
a.order = "desc"
return a
}
// Missing specifies the value to use when the source finds a missing
// value in a document.
func (a *CompositeAggregationTermsValuesSource) Missing(missing interface{}) *CompositeAggregationTermsValuesSource {
a.missing = missing
return a
}
// Source returns the serializable JSON for this values source.
func (a *CompositeAggregationTermsValuesSource) Source() (interface{}, error) {
source := make(map[string]interface{})
name := make(map[string]interface{})
source[a.name] = name
values := make(map[string]interface{})
name["terms"] = values
// field
if a.field != "" {
values["field"] = a.field
}
// script
if a.script != nil {
src, err := a.script.Source()
if err != nil {
return nil, err
}
values["script"] = src
}
// missing
if a.missing != nil {
values["missing"] = a.missing
}
// value_type
if a.valueType != "" {
values["value_type"] = a.valueType
}
// order
if a.order != "" {
values["order"] = a.order
}
return source, nil
}
// -- CompositeAggregationHistogramValuesSource --
// CompositeAggregationHistogramValuesSource is a source for the CompositeAggregation that handles histograms
// it works very similar to a terms histogram with slightly different syntax
//
// See https://www.elastic.co/guide/en/elasticsearch/reference/6.1/search-aggregations-bucket-composite-aggregation.html#_histogram
// for details.
type CompositeAggregationHistogramValuesSource struct {
name string
field string
script *Script
valueType string
missing interface{}
order string
interval float64
}
// NewCompositeAggregationHistogramValuesSource creates and initializes
// a new CompositeAggregationHistogramValuesSource.
func NewCompositeAggregationHistogramValuesSource(name string, interval float64) *CompositeAggregationHistogramValuesSource {
return &CompositeAggregationHistogramValuesSource{
name: name,
interval: interval,
}
}
// Field to use for this source.
func (a *CompositeAggregationHistogramValuesSource) Field(field string) *CompositeAggregationHistogramValuesSource {
a.field = field
return a
}
// Script to use for this source.
func (a *CompositeAggregationHistogramValuesSource) Script(script *Script) *CompositeAggregationHistogramValuesSource {
a.script = script
return a
}
// ValueType specifies the type of values produced by this source,
// e.g. "string" or "date".
func (a *CompositeAggregationHistogramValuesSource) ValueType(valueType string) *CompositeAggregationHistogramValuesSource {
a.valueType = valueType
return a
}
// Missing specifies the value to use when the source finds a missing
// value in a document.
func (a *CompositeAggregationHistogramValuesSource) Missing(missing interface{}) *CompositeAggregationHistogramValuesSource {
a.missing = missing
return a
}
// Order specifies the order in the values produced by this source.
// It can be either "asc" or "desc".
func (a *CompositeAggregationHistogramValuesSource) Order(order string) *CompositeAggregationHistogramValuesSource {
a.order = order
return a
}
// Asc ensures the order of the values produced is ascending.
func (a *CompositeAggregationHistogramValuesSource) Asc() *CompositeAggregationHistogramValuesSource {
a.order = "asc"
return a
}
// Desc ensures the order of the values produced is descending.
func (a *CompositeAggregationHistogramValuesSource) Desc() *CompositeAggregationHistogramValuesSource {
a.order = "desc"
return a
}
// Interval specifies the interval to use.
func (a *CompositeAggregationHistogramValuesSource) Interval(interval float64) *CompositeAggregationHistogramValuesSource {
a.interval = interval
return a
}
// Source returns the serializable JSON for this values source.
func (a *CompositeAggregationHistogramValuesSource) Source() (interface{}, error) {
source := make(map[string]interface{})
name := make(map[string]interface{})
source[a.name] = name
values := make(map[string]interface{})
name["histogram"] = values
// field
if a.field != "" {
values["field"] = a.field
}
// script
if a.script != nil {
src, err := a.script.Source()
if err != nil {
return nil, err
}
values["script"] = src
}
// missing
if a.missing != nil {
values["missing"] = a.missing
}
// value_type
if a.valueType != "" {
values["value_type"] = a.valueType
}
// order
if a.order != "" {
values["order"] = a.order
}
// Histogram-related properties
values["interval"] = a.interval
return source, nil
}
// -- CompositeAggregationDateHistogramValuesSource --
// CompositeAggregationDateHistogramValuesSource is a source for the CompositeAggregation that handles date histograms
// it works very similar to a date histogram aggregation with slightly different syntax
//
// See https://www.elastic.co/guide/en/elasticsearch/reference/6.1/search-aggregations-bucket-composite-aggregation.html#_date_histogram
// for details.
type CompositeAggregationDateHistogramValuesSource struct {
name string
field string
script *Script
valueType string
missing interface{}
order string
interval interface{}
timeZone string
}
// NewCompositeAggregationDateHistogramValuesSource creates and initializes
// a new CompositeAggregationDateHistogramValuesSource.
func NewCompositeAggregationDateHistogramValuesSource(name string, interval interface{}) *CompositeAggregationDateHistogramValuesSource {
return &CompositeAggregationDateHistogramValuesSource{
name: name,
interval: interval,
}
}
// Field to use for this source.
func (a *CompositeAggregationDateHistogramValuesSource) Field(field string) *CompositeAggregationDateHistogramValuesSource {
a.field = field
return a
}
// Script to use for this source.
func (a *CompositeAggregationDateHistogramValuesSource) Script(script *Script) *CompositeAggregationDateHistogramValuesSource {
a.script = script
return a
}
// ValueType specifies the type of values produced by this source,
// e.g. "string" or "date".
func (a *CompositeAggregationDateHistogramValuesSource) ValueType(valueType string) *CompositeAggregationDateHistogramValuesSource {
a.valueType = valueType
return a
}
// Missing specifies the value to use when the source finds a missing
// value in a document.
func (a *CompositeAggregationDateHistogramValuesSource) Missing(missing interface{}) *CompositeAggregationDateHistogramValuesSource {
a.missing = missing
return a
}
// Order specifies the order in the values produced by this source.
// It can be either "asc" or "desc".
func (a *CompositeAggregationDateHistogramValuesSource) Order(order string) *CompositeAggregationDateHistogramValuesSource {
a.order = order
return a
}
// Asc ensures the order of the values produced is ascending.
func (a *CompositeAggregationDateHistogramValuesSource) Asc() *CompositeAggregationDateHistogramValuesSource {
a.order = "asc"
return a
}
// Desc ensures the order of the values produced is descending.
func (a *CompositeAggregationDateHistogramValuesSource) Desc() *CompositeAggregationDateHistogramValuesSource {
a.order = "desc"
return a
}
// Interval to use for the date histogram, e.g. "1d" or a numeric value like "60".
func (a *CompositeAggregationDateHistogramValuesSource) Interval(interval interface{}) *CompositeAggregationDateHistogramValuesSource {
a.interval = interval
return a
}
// TimeZone to use for the dates.
func (a *CompositeAggregationDateHistogramValuesSource) TimeZone(timeZone string) *CompositeAggregationDateHistogramValuesSource {
a.timeZone = timeZone
return a
}
// Source returns the serializable JSON for this values source.
func (a *CompositeAggregationDateHistogramValuesSource) Source() (interface{}, error) {
source := make(map[string]interface{})
name := make(map[string]interface{})
source[a.name] = name
values := make(map[string]interface{})
name["date_histogram"] = values
// field
if a.field != "" {
values["field"] = a.field
}
// script
if a.script != nil {
src, err := a.script.Source()
if err != nil {
return nil, err
}
values["script"] = src
}
// missing
if a.missing != nil {
values["missing"] = a.missing
}
// value_type
if a.valueType != "" {
values["value_type"] = a.valueType
}
// order
if a.order != "" {
values["order"] = a.order
}
// DateHistogram-related properties
values["interval"] = a.interval
// timeZone
if a.timeZone != "" {
values["time_zone"] = a.timeZone
}
return source, nil
}

92
vendor/github.com/olivere/elastic/search_aggs_bucket_composite_test.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,92 @@
// Copyright 2012-present Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.
package elastic
import (
"encoding/json"
"testing"
)
func TestCompositeAggregation(t *testing.T) {
agg := NewCompositeAggregation().
Sources(
NewCompositeAggregationTermsValuesSource("my_terms").Field("a_term").Missing("N/A").Order("asc"),
NewCompositeAggregationHistogramValuesSource("my_histogram", 5).Field("price").Asc(),
NewCompositeAggregationDateHistogramValuesSource("my_date_histogram", "1d").Field("purchase_date").Desc(),
).
Size(10).
AggregateAfter(map[string]interface{}{
"my_terms": "1",
"my_histogram": 2,
"my_date_histogram": "3",
})
src, err := agg.Source()
if err != nil {
t.Fatal(err)
}
data, err := json.Marshal(src)
if err != nil {
t.Fatalf("marshaling to JSON failed: %v", err)
}
got := string(data)
expected := `{"composite":{"after":{"my_date_histogram":"3","my_histogram":2,"my_terms":"1"},"size":10,"sources":[{"my_terms":{"terms":{"field":"a_term","missing":"N/A","order":"asc"}}},{"my_histogram":{"histogram":{"field":"price","interval":5,"order":"asc"}}},{"my_date_histogram":{"date_histogram":{"field":"purchase_date","interval":"1d","order":"desc"}}}]}}`
if got != expected {
t.Errorf("expected\n%s\n,got:\n%s", expected, got)
}
}
func TestCompositeAggregationTermsValuesSource(t *testing.T) {
in := NewCompositeAggregationTermsValuesSource("products").
Script(NewScript("doc['product'].value").Lang("painless"))
src, err := in.Source()
if err != nil {
t.Fatal(err)
}
data, err := json.Marshal(src)
if err != nil {
t.Fatalf("marshaling to JSON failed: %v", err)
}
got := string(data)
expected := `{"products":{"terms":{"script":{"lang":"painless","source":"doc['product'].value"}}}}`
if got != expected {
t.Errorf("expected\n%s\n,got:\n%s", expected, got)
}
}
func TestCompositeAggregationHistogramValuesSource(t *testing.T) {
in := NewCompositeAggregationHistogramValuesSource("histo", 5).
Field("price")
src, err := in.Source()
if err != nil {
t.Fatal(err)
}
data, err := json.Marshal(src)
if err != nil {
t.Fatalf("marshaling to JSON failed: %v", err)
}
got := string(data)
expected := `{"histo":{"histogram":{"field":"price","interval":5}}}`
if got != expected {
t.Errorf("expected\n%s\n,got:\n%s", expected, got)
}
}
func TestCompositeAggregationDateHistogramValuesSource(t *testing.T) {
in := NewCompositeAggregationDateHistogramValuesSource("date", "1d").
Field("timestamp")
src, err := in.Source()
if err != nil {
t.Fatal(err)
}
data, err := json.Marshal(src)
if err != nil {
t.Fatalf("marshaling to JSON failed: %v", err)
}
got := string(data)
expected := `{"date":{"date_histogram":{"field":"timestamp","interval":"1d"}}}`
if got != expected {
t.Errorf("expected\n%s\n,got:\n%s", expected, got)
}
}

9
vendor/github.com/olivere/elastic/search_aggs_bucket_date_range.go сгенерированный поставляемый
Просмотреть файл

@@ -23,6 +23,7 @@ type DateRangeAggregation struct {
meta map[string]interface{}
keyed *bool
unmapped *bool
timeZone string
format string
entries []DateRangeAggregationEntry
}
@@ -71,6 +72,11 @@ func (a *DateRangeAggregation) Unmapped(unmapped bool) *DateRangeAggregation {
return a
}
func (a *DateRangeAggregation) TimeZone(timeZone string) *DateRangeAggregation {
a.timeZone = timeZone
return a
}
func (a *DateRangeAggregation) Format(format string) *DateRangeAggregation {
a.format = format
return a
@@ -178,6 +184,9 @@ func (a *DateRangeAggregation) Source() (interface{}, error) {
if a.unmapped != nil {
opts["unmapped"] = *a.unmapped
}
if a.timeZone != "" {
opts["time_zone"] = a.timeZone
}
if a.format != "" {
opts["format"] = a.format
}

4
vendor/github.com/olivere/elastic/search_aggs_bucket_date_range_test.go сгенерированный поставляемый
Просмотреть файл

@@ -10,7 +10,7 @@ import (
)
func TestDateRangeAggregation(t *testing.T) {
agg := NewDateRangeAggregation().Field("created_at")
agg := NewDateRangeAggregation().Field("created_at").TimeZone("UTC")
agg = agg.AddRange(nil, "2012-12-31")
agg = agg.AddRange("2013-01-01", "2013-12-31")
agg = agg.AddRange("2014-01-01", nil)
@@ -23,7 +23,7 @@ func TestDateRangeAggregation(t *testing.T) {
t.Fatalf("marshaling to JSON failed: %v", err)
}
got := string(data)
expected := `{"date_range":{"field":"created_at","ranges":[{"to":"2012-12-31"},{"from":"2013-01-01","to":"2013-12-31"},{"from":"2014-01-01"}]}}`
expected := `{"date_range":{"field":"created_at","ranges":[{"to":"2012-12-31"},{"from":"2013-01-01","to":"2013-12-31"},{"from":"2014-01-01"}],"time_zone":"UTC"}}`
if got != expected {
t.Errorf("expected\n%s\n,got:\n%s", expected, got)
}

441
vendor/github.com/olivere/elastic/search_aggs_test.go сгенерированный поставляемый
Просмотреть файл

@@ -13,13 +13,15 @@ import (
)
func TestAggs(t *testing.T) {
// client := setupTestClientAndCreateIndex(t, SetTraceLog(log.New(os.Stdout, "", log.LstdFlags)))
//client := setupTestClientAndCreateIndex(t, SetTraceLog(log.New(os.Stdout, "", log.LstdFlags)))
client := setupTestClientAndCreateIndex(t)
esversion, err := client.ElasticsearchVersion(DefaultURL)
if err != nil {
t.Fatal(err)
}
/*
esversion, err := client.ElasticsearchVersion(DefaultURL)
if err != nil {
t.Fatal(err)
}
*/
tweet1 := tweet{
User: "olivere",
@@ -48,7 +50,7 @@ func TestAggs(t *testing.T) {
}
// Add all documents
_, err = client.Index().Index(testIndexName).Type("doc").Id("1").BodyJson(&tweet1).Do(context.TODO())
_, err := client.Index().Index(testIndexName).Type("doc").Id("1").BodyJson(&tweet1).Do(context.TODO())
if err != nil {
t.Fatal(err)
}
@@ -102,6 +104,11 @@ func TestAggs(t *testing.T) {
topTagsAgg := NewTermsAggregation().Field("tags").Size(3).SubAggregation("top_tag_hits", topTagsHitsAgg)
geoBoundsAgg := NewGeoBoundsAggregation().Field("location")
geoHashAgg := NewGeoHashGridAggregation().Field("location").Precision(5)
composite := NewCompositeAggregation().Sources(
NewCompositeAggregationTermsValuesSource("composite_users").Field("user"),
NewCompositeAggregationHistogramValuesSource("composite_retweets", 1).Field("retweets"),
NewCompositeAggregationDateHistogramValuesSource("composite_created", "1m").Field("created"),
)
// Run query
builder := client.Search().Index(testIndexName).Query(all).Pretty(true)
@@ -109,9 +116,7 @@ func TestAggs(t *testing.T) {
builder = builder.Aggregation("users", usersAgg)
builder = builder.Aggregation("retweets", retweetsAgg)
builder = builder.Aggregation("avgRetweets", avgRetweetsAgg)
if esversion >= "2.0" {
builder = builder.Aggregation("avgRetweetsWithMeta", avgRetweetsWithMetaAgg)
}
builder = builder.Aggregation("avgRetweetsWithMeta", avgRetweetsWithMetaAgg)
builder = builder.Aggregation("minRetweets", minRetweetsAgg)
builder = builder.Aggregation("maxRetweets", maxRetweetsAgg)
builder = builder.Aggregation("sumRetweets", sumRetweetsAgg)
@@ -134,44 +139,41 @@ func TestAggs(t *testing.T) {
builder = builder.Aggregation("top-tags", topTagsAgg)
builder = builder.Aggregation("viewport", geoBoundsAgg)
builder = builder.Aggregation("geohashed", geoHashAgg)
if esversion >= "1.4" {
// Unnamed filters
countByUserAgg := NewFiltersAggregation().
Filters(NewTermQuery("user", "olivere"), NewTermQuery("user", "sandrae"))
builder = builder.Aggregation("countByUser", countByUserAgg)
// Named filters
countByUserAgg2 := NewFiltersAggregation().
FilterWithName("olivere", NewTermQuery("user", "olivere")).
FilterWithName("sandrae", NewTermQuery("user", "sandrae"))
builder = builder.Aggregation("countByUser2", countByUserAgg2)
}
if esversion >= "2.0" {
// AvgBucket
dateHisto := NewDateHistogramAggregation().Field("created").Interval("year")
dateHisto = dateHisto.SubAggregation("sumOfRetweets", NewSumAggregation().Field("retweets"))
builder = builder.Aggregation("avgBucketDateHisto", dateHisto)
builder = builder.Aggregation("avgSumOfRetweets", NewAvgBucketAggregation().BucketsPath("avgBucketDateHisto>sumOfRetweets"))
// MinBucket
dateHisto = NewDateHistogramAggregation().Field("created").Interval("year")
dateHisto = dateHisto.SubAggregation("sumOfRetweets", NewSumAggregation().Field("retweets"))
builder = builder.Aggregation("minBucketDateHisto", dateHisto)
builder = builder.Aggregation("minBucketSumOfRetweets", NewMinBucketAggregation().BucketsPath("minBucketDateHisto>sumOfRetweets"))
// MaxBucket
dateHisto = NewDateHistogramAggregation().Field("created").Interval("year")
dateHisto = dateHisto.SubAggregation("sumOfRetweets", NewSumAggregation().Field("retweets"))
builder = builder.Aggregation("maxBucketDateHisto", dateHisto)
builder = builder.Aggregation("maxBucketSumOfRetweets", NewMaxBucketAggregation().BucketsPath("maxBucketDateHisto>sumOfRetweets"))
// SumBucket
dateHisto = NewDateHistogramAggregation().Field("created").Interval("year")
dateHisto = dateHisto.SubAggregation("sumOfRetweets", NewSumAggregation().Field("retweets"))
builder = builder.Aggregation("sumBucketDateHisto", dateHisto)
builder = builder.Aggregation("sumBucketSumOfRetweets", NewSumBucketAggregation().BucketsPath("sumBucketDateHisto>sumOfRetweets"))
// MovAvg
dateHisto = NewDateHistogramAggregation().Field("created").Interval("year")
dateHisto = dateHisto.SubAggregation("sumOfRetweets", NewSumAggregation().Field("retweets"))
dateHisto = dateHisto.SubAggregation("movingAvg", NewMovAvgAggregation().BucketsPath("sumOfRetweets"))
builder = builder.Aggregation("movingAvgDateHisto", dateHisto)
}
// Unnamed filters
countByUserAgg := NewFiltersAggregation().
Filters(NewTermQuery("user", "olivere"), NewTermQuery("user", "sandrae"))
builder = builder.Aggregation("countByUser", countByUserAgg)
// Named filters
countByUserAgg2 := NewFiltersAggregation().
FilterWithName("olivere", NewTermQuery("user", "olivere")).
FilterWithName("sandrae", NewTermQuery("user", "sandrae"))
builder = builder.Aggregation("countByUser2", countByUserAgg2)
// AvgBucket
dateHisto := NewDateHistogramAggregation().Field("created").Interval("year")
dateHisto = dateHisto.SubAggregation("sumOfRetweets", NewSumAggregation().Field("retweets"))
builder = builder.Aggregation("avgBucketDateHisto", dateHisto)
builder = builder.Aggregation("avgSumOfRetweets", NewAvgBucketAggregation().BucketsPath("avgBucketDateHisto>sumOfRetweets"))
// MinBucket
dateHisto = NewDateHistogramAggregation().Field("created").Interval("year")
dateHisto = dateHisto.SubAggregation("sumOfRetweets", NewSumAggregation().Field("retweets"))
builder = builder.Aggregation("minBucketDateHisto", dateHisto)
builder = builder.Aggregation("minBucketSumOfRetweets", NewMinBucketAggregation().BucketsPath("minBucketDateHisto>sumOfRetweets"))
// MaxBucket
dateHisto = NewDateHistogramAggregation().Field("created").Interval("year")
dateHisto = dateHisto.SubAggregation("sumOfRetweets", NewSumAggregation().Field("retweets"))
builder = builder.Aggregation("maxBucketDateHisto", dateHisto)
builder = builder.Aggregation("maxBucketSumOfRetweets", NewMaxBucketAggregation().BucketsPath("maxBucketDateHisto>sumOfRetweets"))
// SumBucket
dateHisto = NewDateHistogramAggregation().Field("created").Interval("year")
dateHisto = dateHisto.SubAggregation("sumOfRetweets", NewSumAggregation().Field("retweets"))
builder = builder.Aggregation("sumBucketDateHisto", dateHisto)
builder = builder.Aggregation("sumBucketSumOfRetweets", NewSumBucketAggregation().BucketsPath("sumBucketDateHisto>sumOfRetweets"))
// MovAvg
dateHisto = NewDateHistogramAggregation().Field("created").Interval("year")
dateHisto = dateHisto.SubAggregation("sumOfRetweets", NewSumAggregation().Field("retweets"))
dateHisto = dateHisto.SubAggregation("movingAvg", NewMovAvgAggregation().BucketsPath("sumOfRetweets"))
builder = builder.Aggregation("movingAvgDateHisto", dateHisto)
builder = builder.Aggregation("composite", composite)
searchResult, err := builder.Do(context.TODO())
if err != nil {
t.Fatal(err)
@@ -308,26 +310,24 @@ func TestAggs(t *testing.T) {
}
// avgRetweetsWithMeta
if esversion >= "2.0" {
avgMetaAggRes, found := agg.Avg("avgRetweetsWithMeta")
if !found {
t.Errorf("expected %v; got: %v", true, found)
}
if avgMetaAggRes == nil {
t.Fatalf("expected != nil; got: nil")
}
if avgMetaAggRes.Meta == nil {
t.Fatalf("expected != nil; got: %v", avgMetaAggRes.Meta)
}
metaDataValue, found := avgMetaAggRes.Meta["meta"]
if !found {
t.Fatalf("expected to return meta data key %q; got: %v", "meta", found)
}
if flag, ok := metaDataValue.(bool); !ok {
t.Fatalf("expected to return meta data key type %T; got: %T", true, metaDataValue)
} else if flag != true {
t.Fatalf("expected to return meta data key value %v; got: %v", true, flag)
}
avgMetaAggRes, found := agg.Avg("avgRetweetsWithMeta")
if !found {
t.Errorf("expected %v; got: %v", true, found)
}
if avgMetaAggRes == nil {
t.Fatalf("expected != nil; got: nil")
}
if avgMetaAggRes.Meta == nil {
t.Fatalf("expected != nil; got: %v", avgMetaAggRes.Meta)
}
metaDataValue, found := avgMetaAggRes.Meta["meta"]
if !found {
t.Fatalf("expected to return meta data key %q; got: %v", "meta", found)
}
if flag, ok := metaDataValue.(bool); !ok {
t.Fatalf("expected to return meta data key type %T; got: %T", true, metaDataValue)
} else if flag != true {
t.Fatalf("expected to return meta data key value %v; got: %v", true, flag)
}
// minRetweets
@@ -817,13 +817,11 @@ func TestAggs(t *testing.T) {
if topTags == nil {
t.Fatalf("expected != nil; got: nil")
}
if esversion >= "1.4.0" {
if topTags.DocCountErrorUpperBound != 0 {
t.Errorf("expected %v; got: %v", 0, topTags.DocCountErrorUpperBound)
}
if topTags.SumOfOtherDocCount != 1 {
t.Errorf("expected %v; got: %v", 1, topTags.SumOfOtherDocCount)
}
if topTags.DocCountErrorUpperBound != 0 {
t.Errorf("expected %v; got: %v", 0, topTags.DocCountErrorUpperBound)
}
if topTags.SumOfOtherDocCount != 1 {
t.Errorf("expected %v; got: %v", 1, topTags.SumOfOtherDocCount)
}
if len(topTags.Buckets) != 3 {
t.Fatalf("expected %d; got: %d", 3, len(topTags.Buckets))
@@ -924,62 +922,71 @@ func TestAggs(t *testing.T) {
t.Fatalf("expected != nil; got: nil")
}
if esversion >= "1.4" {
// Filters agg "countByUser" (unnamed)
countByUserAggRes, found := agg.Filters("countByUser")
if !found {
t.Errorf("expected %v; got: %v", true, found)
}
if countByUserAggRes == nil {
t.Fatalf("expected != nil; got: nil")
}
if len(countByUserAggRes.Buckets) != 2 {
t.Fatalf("expected %d; got: %d", 2, len(countByUserAggRes.Buckets))
}
if len(countByUserAggRes.NamedBuckets) != 0 {
t.Fatalf("expected %d; got: %d", 0, len(countByUserAggRes.NamedBuckets))
}
if countByUserAggRes.Buckets[0].DocCount != 2 {
t.Errorf("expected %d; got: %d", 2, countByUserAggRes.Buckets[0].DocCount)
}
if countByUserAggRes.Buckets[1].DocCount != 1 {
t.Errorf("expected %d; got: %d", 1, countByUserAggRes.Buckets[1].DocCount)
}
// Filters agg "countByUser" (unnamed)
countByUserAggRes, found := agg.Filters("countByUser")
if !found {
t.Errorf("expected %v; got: %v", true, found)
}
if countByUserAggRes == nil {
t.Fatalf("expected != nil; got: nil")
}
if len(countByUserAggRes.Buckets) != 2 {
t.Fatalf("expected %d; got: %d", 2, len(countByUserAggRes.Buckets))
}
if len(countByUserAggRes.NamedBuckets) != 0 {
t.Fatalf("expected %d; got: %d", 0, len(countByUserAggRes.NamedBuckets))
}
if countByUserAggRes.Buckets[0].DocCount != 2 {
t.Errorf("expected %d; got: %d", 2, countByUserAggRes.Buckets[0].DocCount)
}
if countByUserAggRes.Buckets[1].DocCount != 1 {
t.Errorf("expected %d; got: %d", 1, countByUserAggRes.Buckets[1].DocCount)
}
// Filters agg "countByUser2" (named)
countByUser2AggRes, found := agg.Filters("countByUser2")
if !found {
t.Errorf("expected %v; got: %v", true, found)
}
if countByUser2AggRes == nil {
t.Fatalf("expected != nil; got: nil")
}
if len(countByUser2AggRes.Buckets) != 0 {
t.Fatalf("expected %d; got: %d", 0, len(countByUser2AggRes.Buckets))
}
if len(countByUser2AggRes.NamedBuckets) != 2 {
t.Fatalf("expected %d; got: %d", 2, len(countByUser2AggRes.NamedBuckets))
}
b, found := countByUser2AggRes.NamedBuckets["olivere"]
if !found {
t.Fatalf("expected bucket %q; got: %v", "olivere", found)
}
if b == nil {
t.Fatalf("expected bucket %q; got: %v", "olivere", b)
}
if b.DocCount != 2 {
t.Errorf("expected %d; got: %d", 2, b.DocCount)
}
b, found = countByUser2AggRes.NamedBuckets["sandrae"]
if !found {
t.Fatalf("expected bucket %q; got: %v", "sandrae", found)
}
if b == nil {
t.Fatalf("expected bucket %q; got: %v", "sandrae", b)
}
if b.DocCount != 1 {
t.Errorf("expected %d; got: %d", 1, b.DocCount)
}
// Filters agg "countByUser2" (named)
countByUser2AggRes, found := agg.Filters("countByUser2")
if !found {
t.Errorf("expected %v; got: %v", true, found)
}
if countByUser2AggRes == nil {
t.Fatalf("expected != nil; got: nil")
}
if len(countByUser2AggRes.Buckets) != 0 {
t.Fatalf("expected %d; got: %d", 0, len(countByUser2AggRes.Buckets))
}
if len(countByUser2AggRes.NamedBuckets) != 2 {
t.Fatalf("expected %d; got: %d", 2, len(countByUser2AggRes.NamedBuckets))
}
b, found := countByUser2AggRes.NamedBuckets["olivere"]
if !found {
t.Fatalf("expected bucket %q; got: %v", "olivere", found)
}
if b == nil {
t.Fatalf("expected bucket %q; got: %v", "olivere", b)
}
if b.DocCount != 2 {
t.Errorf("expected %d; got: %d", 2, b.DocCount)
}
b, found = countByUser2AggRes.NamedBuckets["sandrae"]
if !found {
t.Fatalf("expected bucket %q; got: %v", "sandrae", found)
}
if b == nil {
t.Fatalf("expected bucket %q; got: %v", "sandrae", b)
}
if b.DocCount != 1 {
t.Errorf("expected %d; got: %d", 1, b.DocCount)
}
compositeAggRes, found := agg.Composite("composite")
if !found {
t.Errorf("expected %v; got: %v", true, found)
}
if compositeAggRes == nil {
t.Fatalf("expected != nil; got: nil")
}
if want, have := 3, len(compositeAggRes.Buckets); want != have {
t.Fatalf("expected %d; got: %d", want, have)
}
}
@@ -3231,3 +3238,179 @@ func TestAggsPipelineSerialDiff(t *testing.T) {
t.Fatalf("expected aggregation value = %v; got: %v", float64(20), *agg.Value)
}
}
func TestAggsComposite(t *testing.T) {
s := `{
"the_composite" : {
"buckets" : [
{
"key" : {
"composite_users" : "olivere",
"composite_retweets" : 0.0,
"composite_created" : 1349856720000
},
"doc_count" : 1
},
{
"key" : {
"composite_users" : "olivere",
"composite_retweets" : 108.0,
"composite_created" : 1355333880000
},
"doc_count" : 1
},
{
"key" : {
"composite_users" : "sandrae",
"composite_retweets" : 12.0,
"composite_created" : 1321009080000
},
"doc_count" : 1
}
]
}
}`
aggs := new(Aggregations)
err := json.Unmarshal([]byte(s), &aggs)
if err != nil {
t.Fatalf("expected no error decoding; got: %v", err)
}
agg, found := aggs.Composite("the_composite")
if !found {
t.Fatalf("expected aggregation to be found; got: %v", found)
}
if agg == nil {
t.Fatalf("expected aggregation != nil; got: %v", agg)
}
if want, have := 3, len(agg.Buckets); want != have {
t.Fatalf("expected aggregation buckets length = %v; got: %v", want, have)
}
// 1st bucket
bucket := agg.Buckets[0]
if want, have := int64(1), bucket.DocCount; want != have {
t.Fatalf("expected aggregation bucket doc count = %v; got: %v", want, have)
}
if want, have := 3, len(bucket.Key); want != have {
t.Fatalf("expected aggregation bucket key length = %v; got: %v", want, have)
}
v, found := bucket.Key["composite_users"]
if !found {
t.Fatalf("expected to find bucket key %q", "composite_users")
}
s, ok := v.(string)
if !ok {
t.Fatalf("expected to have bucket key of type string; got: %T", v)
}
if want, have := "olivere", s; want != have {
t.Fatalf("expected to find bucket key value %q; got: %q", want, have)
}
v, found = bucket.Key["composite_retweets"]
if !found {
t.Fatalf("expected to find bucket key %q", "composite_retweets")
}
f, ok := v.(float64)
if !ok {
t.Fatalf("expected to have bucket key of type string; got: %T", v)
}
if want, have := 0.0, f; want != have {
t.Fatalf("expected to find bucket key value %v; got: %v", want, have)
}
v, found = bucket.Key["composite_created"]
if !found {
t.Fatalf("expected to find bucket key %q", "composite_created")
}
f, ok = v.(float64)
if !ok {
t.Fatalf("expected to have bucket key of type string; got: %T", v)
}
if want, have := 1349856720000.0, f; want != have {
t.Fatalf("expected to find bucket key value %v; got: %v", want, have)
}
// 2nd bucket
bucket = agg.Buckets[1]
if want, have := int64(1), bucket.DocCount; want != have {
t.Fatalf("expected aggregation bucket doc count = %v; got: %v", want, have)
}
if want, have := 3, len(bucket.Key); want != have {
t.Fatalf("expected aggregation bucket key length = %v; got: %v", want, have)
}
v, found = bucket.Key["composite_users"]
if !found {
t.Fatalf("expected to find bucket key %q", "composite_users")
}
s, ok = v.(string)
if !ok {
t.Fatalf("expected to have bucket key of type string; got: %T", v)
}
if want, have := "olivere", s; want != have {
t.Fatalf("expected to find bucket key value %q; got: %q", want, have)
}
v, found = bucket.Key["composite_retweets"]
if !found {
t.Fatalf("expected to find bucket key %q", "composite_retweets")
}
f, ok = v.(float64)
if !ok {
t.Fatalf("expected to have bucket key of type string; got: %T", v)
}
if want, have := 108.0, f; want != have {
t.Fatalf("expected to find bucket key value %v; got: %v", want, have)
}
v, found = bucket.Key["composite_created"]
if !found {
t.Fatalf("expected to find bucket key %q", "composite_created")
}
f, ok = v.(float64)
if !ok {
t.Fatalf("expected to have bucket key of type string; got: %T", v)
}
if want, have := 1355333880000.0, f; want != have {
t.Fatalf("expected to find bucket key value %v; got: %v", want, have)
}
// 3rd bucket
bucket = agg.Buckets[2]
if want, have := int64(1), bucket.DocCount; want != have {
t.Fatalf("expected aggregation bucket doc count = %v; got: %v", want, have)
}
if want, have := 3, len(bucket.Key); want != have {
t.Fatalf("expected aggregation bucket key length = %v; got: %v", want, have)
}
v, found = bucket.Key["composite_users"]
if !found {
t.Fatalf("expected to find bucket key %q", "composite_users")
}
s, ok = v.(string)
if !ok {
t.Fatalf("expected to have bucket key of type string; got: %T", v)
}
if want, have := "sandrae", s; want != have {
t.Fatalf("expected to find bucket key value %q; got: %q", want, have)
}
v, found = bucket.Key["composite_retweets"]
if !found {
t.Fatalf("expected to find bucket key %q", "composite_retweets")
}
f, ok = v.(float64)
if !ok {
t.Fatalf("expected to have bucket key of type string; got: %T", v)
}
if want, have := 12.0, f; want != have {
t.Fatalf("expected to find bucket key value %v; got: %v", want, have)
}
v, found = bucket.Key["composite_created"]
if !found {
t.Fatalf("expected to find bucket key %q", "composite_created")
}
f, ok = v.(float64)
if !ok {
t.Fatalf("expected to have bucket key of type string; got: %T", v)
}
if want, have := 1321009080000.0, f; want != have {
t.Fatalf("expected to find bucket key value %v; got: %v", want, have)
}
}

96
vendor/github.com/olivere/elastic/search_queries_terms_set.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,96 @@
// Copyright 2012-present Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.
package elastic
// TermsSetQuery returns any documents that match with at least
// one or more of the provided terms. The terms are not analyzed
// and thus must match exactly. The number of terms that must
// match varies per document and is either controlled by a
// minimum should match field or computed per document in a
// minimum should match script.
//
// For more details, see
// https://www.elastic.co/guide/en/elasticsearch/reference/6.1/query-dsl-terms-set-query.html
type TermsSetQuery struct {
name string
values []interface{}
minimumShouldMatchField string
minimumShouldMatchScript *Script
queryName string
boost *float64
}
// NewTermsSetQuery creates and initializes a new TermsSetQuery.
func NewTermsSetQuery(name string, values ...interface{}) *TermsSetQuery {
q := &TermsSetQuery{
name: name,
}
if len(values) > 0 {
q.values = append(q.values, values...)
}
return q
}
// MinimumShouldMatchField specifies the field to match.
func (q *TermsSetQuery) MinimumShouldMatchField(minimumShouldMatchField string) *TermsSetQuery {
q.minimumShouldMatchField = minimumShouldMatchField
return q
}
// MinimumShouldMatchScript specifies the script to match.
func (q *TermsSetQuery) MinimumShouldMatchScript(minimumShouldMatchScript *Script) *TermsSetQuery {
q.minimumShouldMatchScript = minimumShouldMatchScript
return q
}
// Boost sets the boost for this query.
func (q *TermsSetQuery) Boost(boost float64) *TermsSetQuery {
q.boost = &boost
return q
}
// QueryName sets the query name for the filter that can be used
// when searching for matched_filters per hit
func (q *TermsSetQuery) QueryName(queryName string) *TermsSetQuery {
q.queryName = queryName
return q
}
// Source creates the query source for the term query.
func (q *TermsSetQuery) Source() (interface{}, error) {
// {"terms_set":{"codes":{"terms":["abc","def"],"minimum_should_match_field":"required_matches"}}}
source := make(map[string]interface{})
inner := make(map[string]interface{})
params := make(map[string]interface{})
inner[q.name] = params
source["terms_set"] = inner
// terms
params["terms"] = q.values
// minimum_should_match_field
if match := q.minimumShouldMatchField; match != "" {
params["minimum_should_match_field"] = match
}
// minimum_should_match_script
if match := q.minimumShouldMatchScript; match != nil {
src, err := match.Source()
if err != nil {
return nil, err
}
params["minimum_should_match_script"] = src
}
// Common parameters for all queries
if q.boost != nil {
params["boost"] = *q.boost
}
if q.queryName != "" {
params["_name"] = q.queryName
}
return source, nil
}

75
vendor/github.com/olivere/elastic/search_queries_terms_set_test.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,75 @@
// Copyright 2012-present Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.
package elastic
import (
"context"
"encoding/json"
"testing"
)
func TestTermsSetQueryWithField(t *testing.T) {
q := NewTermsSetQuery("codes", "abc", "def", "ghi").MinimumShouldMatchField("required_matches")
src, err := q.Source()
if err != nil {
t.Fatal(err)
}
data, err := json.Marshal(src)
if err != nil {
t.Fatalf("marshaling to JSON failed: %v", err)
}
got := string(data)
expected := `{"terms_set":{"codes":{"minimum_should_match_field":"required_matches","terms":["abc","def","ghi"]}}}`
if got != expected {
t.Errorf("expected\n%s\n,got:\n%s", expected, got)
}
}
func TestTermsSetQueryWithScript(t *testing.T) {
q := NewTermsSetQuery("codes", "abc", "def", "ghi").
MinimumShouldMatchScript(
NewScript(`Math.min(params.num_terms, doc['required_matches'].value)`),
)
src, err := q.Source()
if err != nil {
t.Fatal(err)
}
data, err := json.Marshal(src)
if err != nil {
t.Fatalf("marshaling to JSON failed: %v", err)
}
got := string(data)
expected := `{"terms_set":{"codes":{"minimum_should_match_script":{"source":"Math.min(params.num_terms, doc['required_matches'].value)"},"terms":["abc","def","ghi"]}}}`
if got != expected {
t.Errorf("expected\n%s\n,got:\n%s", expected, got)
}
}
func TestSearchTermsSetQuery(t *testing.T) {
//client := setupTestClientAndCreateIndexAndAddDocs(t, SetTraceLog(log.New(os.Stdout, "", log.LstdFlags)))
client := setupTestClientAndCreateIndexAndAddDocs(t)
// Match all should return all documents
searchResult, err := client.Search().
Index(testIndexName).
Query(
NewTermsSetQuery("user", "olivere", "sandrae").
MinimumShouldMatchField("retweets"),
).
Pretty(true).
Do(context.TODO())
if err != nil {
t.Fatal(err)
}
if searchResult.Hits == nil {
t.Errorf("expected SearchResult.Hits != nil; got nil")
}
if got, want := searchResult.Hits.TotalHits, int64(3); got != want {
t.Errorf("expected SearchResult.Hits.TotalHits = %d; got %d", want, got)
}
if got, want := len(searchResult.Hits.Hits), 3; got != want {
t.Errorf("expected len(SearchResult.Hits.Hits) = %d; got %d", want, got)
}
}

74
vendor/github.com/olivere/elastic/search_request.go сгенерированный поставляемый
Просмотреть файл

@@ -4,13 +4,16 @@
package elastic
import "strings"
import (
"encoding/json"
"strings"
)
// SearchRequest combines a search request and its
// query details (see SearchSource).
// It is used in combination with MultiSearch.
type SearchRequest struct {
searchType string // default in ES is "query_then_fetch"
searchType string
indices []string
types []string
routing *string
@@ -28,38 +31,23 @@ func NewSearchRequest() *SearchRequest {
return &SearchRequest{}
}
// SearchRequest must be one of "query_then_fetch", "query_and_fetch",
// "scan", "count", "dfs_query_then_fetch", or "dfs_query_and_fetch".
// Use one of the constants defined via SearchType.
// SearchRequest must be one of "dfs_query_then_fetch" or
// "query_then_fetch".
func (r *SearchRequest) SearchType(searchType string) *SearchRequest {
r.searchType = searchType
return r
}
// SearchTypeDfsQueryThenFetch sets search type to dfs_query_then_fetch.
func (r *SearchRequest) SearchTypeDfsQueryThenFetch() *SearchRequest {
return r.SearchType("dfs_query_then_fetch")
}
func (r *SearchRequest) SearchTypeDfsQueryAndFetch() *SearchRequest {
return r.SearchType("dfs_query_and_fetch")
}
// SearchTypeQueryThenFetch sets search type to query_then_fetch.
func (r *SearchRequest) SearchTypeQueryThenFetch() *SearchRequest {
return r.SearchType("query_then_fetch")
}
func (r *SearchRequest) SearchTypeQueryAndFetch() *SearchRequest {
return r.SearchType("query_and_fetch")
}
func (r *SearchRequest) SearchTypeScan() *SearchRequest {
return r.SearchType("scan")
}
func (r *SearchRequest) SearchTypeCount() *SearchRequest {
return r.SearchType("count")
}
func (r *SearchRequest) Index(indices ...string) *SearchRequest {
r.indices = append(r.indices, indices...)
return r
@@ -130,17 +118,7 @@ func (r *SearchRequest) SearchSource(searchSource *SearchSource) *SearchRequest
}
func (r *SearchRequest) Source(source interface{}) *SearchRequest {
switch v := source.(type) {
case *SearchSource:
src, err := v.Source()
if err != nil {
// Do not do anything in case of an error
return r
}
r.source = src
default:
r.source = source
}
r.source = source
return r
}
@@ -200,6 +178,34 @@ func (r *SearchRequest) header() interface{} {
// Body is used e.g. by MultiSearch to get information about the search body
// of one SearchRequest.
// See https://www.elastic.co/guide/en/elasticsearch/reference/6.0/search-multi-search.html
func (r *SearchRequest) Body() interface{} {
return r.source
func (r *SearchRequest) Body() (string, error) {
switch t := r.source.(type) {
default:
body, err := json.Marshal(r.source)
if err != nil {
return "", err
}
return string(body), nil
case *SearchSource:
src, err := t.Source()
if err != nil {
return "", err
}
body, err := json.Marshal(src)
if err != nil {
return "", err
}
return string(body), nil
case json.RawMessage:
return string(t), nil
case *json.RawMessage:
return string(*t), nil
case string:
return t, nil
case *string:
if t != nil {
return *t, nil
}
return "{}", nil
}
}

55
vendor/github.com/olivere/elastic/search_test.go сгенерированный поставляемый
Просмотреть файл

@@ -607,6 +607,61 @@ func TestSearchSource(t *testing.T) {
}
}
func TestSearchSourceWithString(t *testing.T) {
client := setupTestClientAndCreateIndex(t)
tweet1 := tweet{
User: "olivere", Retweets: 108,
Message: "Welcome to Golang and Elasticsearch.",
Created: time.Date(2012, 12, 12, 17, 38, 34, 0, time.UTC),
}
tweet2 := tweet{
User: "olivere", Retweets: 0,
Message: "Another unrelated topic.",
Created: time.Date(2012, 10, 10, 8, 12, 03, 0, time.UTC),
}
tweet3 := tweet{
User: "sandrae", Retweets: 12,
Message: "Cycling is fun.",
Created: time.Date(2011, 11, 11, 10, 58, 12, 0, time.UTC),
}
// Add all documents
_, err := client.Index().Index(testIndexName).Type("doc").Id("1").BodyJson(&tweet1).Do(context.TODO())
if err != nil {
t.Fatal(err)
}
_, err = client.Index().Index(testIndexName).Type("doc").Id("2").BodyJson(&tweet2).Do(context.TODO())
if err != nil {
t.Fatal(err)
}
_, err = client.Index().Index(testIndexName).Type("doc").Id("3").BodyJson(&tweet3).Do(context.TODO())
if err != nil {
t.Fatal(err)
}
_, err = client.Flush().Index(testIndexName).Do(context.TODO())
if err != nil {
t.Fatal(err)
}
searchResult, err := client.Search().
Index(testIndexName).
Source(`{"query":{"match_all":{}}}`). // sets the JSON request
Do(context.TODO())
if err != nil {
t.Fatal(err)
}
if searchResult.Hits == nil {
t.Errorf("expected SearchResult.Hits != nil; got nil")
}
if searchResult.Hits.TotalHits != 3 {
t.Errorf("expected SearchResult.Hits.TotalHits = %d; got %d", 3, searchResult.Hits.TotalHits)
}
}
func TestSearchRawString(t *testing.T) {
// client := setupTestClientAndCreateIndexAndLog(t, SetTraceLog(log.New(os.Stdout, "", 0)))
client := setupTestClientAndCreateIndex(t)

8
vendor/github.com/prometheus/client_golang/ISSUE_TEMPLATE.md сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,8 @@
<!--
If you are unhappy how your favorite Go dependency management tool deals
with this repository, please do not file an issue but read
https://github.com/prometheus/client_golang#important-note-about-releases-versioning-tagging-stability-and-your-favorite-go-dependency-management-tool
instead. Thank you very much.
-->

47
vendor/github.com/prometheus/client_golang/README.md сгенерированный поставляемый
Просмотреть файл

@@ -2,6 +2,7 @@
[![Build Status](https://travis-ci.org/prometheus/client_golang.svg?branch=master)](https://travis-ci.org/prometheus/client_golang)
[![Go Report Card](https://goreportcard.com/badge/github.com/prometheus/client_golang)](https://goreportcard.com/report/github.com/prometheus/client_golang)
[![go-doc](https://godoc.org/github.com/prometheus/client_golang?status.svg)](https://godoc.org/github.com/prometheus/client_golang)
This is the [Go](http://golang.org) client library for
[Prometheus](http://prometheus.io). It has two separate parts, one for
@@ -10,6 +11,50 @@ Prometheus HTTP API.
__This library requires Go1.7 or later.__
## Important note about releases, versioning, tagging, stability, and your favorite Go dependency management tool
While our goal is to follow [Semantic Versioning](https://semver.org/), this
repository is still pre-1.0.0. To quote the
[Semantic Versioning spec](https://semver.org/#spec-item-4): “Anything may
change at any time. The public API should not be considered stable.” We know
that this is at odds with the widespread use of this library. However, just
declaring something 1.0.0 doesn't make it 1.0.0. Instead, we are working
towards a 1.0.0 release that actually deserves its major version number.
Having said that, we aim for always keeping the tip of master in a workable
state and for only introducing ”mildly” breaking changes up to and including
[v0.9.0](https://github.com/prometheus/client_golang/milestone/1). After that,
a number of ”hard” breaking changes are planned, see the
[v0.10.0 milestone](https://github.com/prometheus/client_golang/milestone/2),
which should get the library much closer to 1.0.0 state.
Dependency management in Go projects is still in flux, and there are many tools
floating around. While [dep](https://golang.github.io/dep/) might develop into
the de-facto standard tool, it is still officially experimental. The roadmap
for this library has been laid out with a lot of sometimes painful experience
in mind. We really cannot adjust it every other month to the needs of the
currently most popular or most promising Go dependency management tool. The
recommended course of action with dependency management tools is the following:
- Do not expect strict post-1.0.0 semver semantics prior to the 1.0.0
release. If your dependency management tool expects strict post-1.0.0 semver
semantics, you have to wait. Sorry.
- If you want absolute certainty, please lock to a specific commit. You can
also lock to tags, but please don't ask for more tagging. This would suggest
some release or stability testing procedure that simply is not in place. As
said above, we are aiming for stability of the tip of master, but if we
tagged every single commit, locking to tags would be the same as locking to
commits.
- If you want to get the newer features and improvements and are willing to
take the minor risk of newly introduced bugs and “mild” breakage, just always
update to the tip of master (which is essentially the original idea of Go
dependency management). We recommend to not use features marked as
_deprecated_ in this case.
- Once [v0.9.0](https://github.com/prometheus/client_golang/milestone/1) is
out, you could lock to v0.9.x to get bugfixes (and perhaps minor new
features) while avoiding the “hard” breakage that will come with post-0.9
features.
## Instrumenting applications
[![code-coverage](http://gocover.io/_badge/github.com/prometheus/client_golang/prometheus)](http://gocover.io/github.com/prometheus/client_golang/prometheus) [![go-doc](https://godoc.org/github.com/prometheus/client_golang/prometheus?status.svg)](https://godoc.org/github.com/prometheus/client_golang/prometheus)
@@ -26,7 +71,7 @@ contains simple examples of instrumented code.
## Client for the Prometheus HTTP API
[![code-coverage](http://gocover.io/_badge/github.com/prometheus/client_golang/api/prometheus)](http://gocover.io/github.com/prometheus/client_golang/api/prometheus) [![go-doc](https://godoc.org/github.com/prometheus/client_golang/api/prometheus?status.svg)](https://godoc.org/github.com/prometheus/client_golang/api/prometheus)
[![code-coverage](http://gocover.io/_badge/github.com/prometheus/client_golang/api/prometheus/v1)](http://gocover.io/github.com/prometheus/client_golang/api/prometheus/v1) [![go-doc](https://godoc.org/github.com/prometheus/client_golang/api/prometheus?status.svg)](https://godoc.org/github.com/prometheus/client_golang/api)
The
[`api/prometheus` directory](https://github.com/prometheus/client_golang/tree/master/api/prometheus)

8
vendor/github.com/prometheus/client_golang/prometheus/doc.go сгенерированный поставляемый
Просмотреть файл

@@ -11,10 +11,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package prometheus provides metrics primitives to instrument code for
// monitoring. It also offers a registry for metrics. Sub-packages allow to
// expose the registered metrics via HTTP (package promhttp) or push them to a
// Pushgateway (package push).
// Package prometheus is the core instrumentation package. It provides metrics
// primitives to instrument code for monitoring. It also offers a registry for
// metrics. Sub-packages allow to expose the registered metrics via HTTP
// (package promhttp) or push them to a Pushgateway (package push).
//
// All exported functions and methods are safe to be used concurrently unless
// specified otherwise.

5
vendor/github.com/prometheus/client_golang/prometheus/http.go сгенерированный поставляемый
Просмотреть файл

@@ -61,9 +61,8 @@ func giveBuf(buf *bytes.Buffer) {
// name).
//
// Deprecated: Please note the issues described in the doc comment of
// InstrumentHandler. You might want to consider using promhttp.Handler instead
// (which is not instrumented, but can be instrumented with the tooling provided
// in package promhttp).
// InstrumentHandler. You might want to consider using
// promhttp.InstrumentedHandler instead.
func Handler() http.Handler {
return InstrumentHandler("prometheus", UninstrumentedHandler())
}

14
vendor/github.com/prometheus/client_golang/prometheus/process_collector.go сгенерированный поставляемый
Просмотреть файл

@@ -26,8 +26,11 @@ type processCollector struct {
}
// NewProcessCollector returns a collector which exports the current state of
// process metrics including cpu, memory and file descriptor usage as well as
// the process start time for the given process id under the given namespace.
// process metrics including CPU, memory and file descriptor usage as well as
// the process start time for the given process ID under the given namespace.
//
// Currently, the collector depends on a Linux-style proc filesystem and
// therefore only exports metrics for Linux.
func NewProcessCollector(pid int, namespace string) Collector {
return NewProcessCollectorPIDFn(
func() (int, error) { return pid, nil },
@@ -35,11 +38,8 @@ func NewProcessCollector(pid int, namespace string) Collector {
)
}
// NewProcessCollectorPIDFn returns a collector which exports the current state
// of process metrics including cpu, memory and file descriptor usage as well
// as the process start time under the given namespace. The given pidFn is
// called on each collect and is used to determine the process to export
// metrics for.
// NewProcessCollectorPIDFn works like NewProcessCollector but the process ID is
// determined on each collect anew by calling the given pidFn function.
func NewProcessCollectorPIDFn(
pidFn func() (int, error),
namespace string,

6
vendor/github.com/prometheus/client_golang/prometheus/promhttp/delegator.go сгенерированный поставляемый
Просмотреть файл

@@ -102,10 +102,10 @@ func init() {
return d
}
pickDelegator[closeNotifier] = func(d *responseWriterDelegator) delegator { // 1
return closeNotifierDelegator{d}
return &closeNotifierDelegator{d}
}
pickDelegator[flusher] = func(d *responseWriterDelegator) delegator { // 2
return flusherDelegator{d}
return &flusherDelegator{d}
}
pickDelegator[flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 3
return struct {
@@ -115,7 +115,7 @@ func init() {
}{d, &flusherDelegator{d}, &closeNotifierDelegator{d}}
}
pickDelegator[hijacker] = func(d *responseWriterDelegator) delegator { // 4
return hijackerDelegator{d}
return &hijackerDelegator{d}
}
pickDelegator[hijacker+closeNotifier] = func(d *responseWriterDelegator) delegator { // 5
return struct {

2
vendor/github.com/prometheus/client_golang/prometheus/promhttp/delegator_1_8.go сгенерированный поставляемый
Просмотреть файл

@@ -28,7 +28,7 @@ func (d *pusherDelegator) Push(target string, opts *http.PushOptions) error {
func init() {
pickDelegator[pusher] = func(d *responseWriterDelegator) delegator { // 16
return pusherDelegator{d}
return &pusherDelegator{d}
}
pickDelegator[pusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 17
return struct {

129
vendor/github.com/prometheus/client_golang/prometheus/promhttp/http.go сгенерированный поставляемый
Просмотреть файл

@@ -39,6 +39,7 @@ import (
"net/http"
"strings"
"sync"
"time"
"github.com/prometheus/common/expfmt"
@@ -67,21 +68,51 @@ func giveBuf(buf *bytes.Buffer) {
bufPool.Put(buf)
}
// Handler returns an HTTP handler for the prometheus.DefaultGatherer. The
// Handler uses the default HandlerOpts, i.e. report the first error as an HTTP
// error, no error logging, and compression if requested by the client.
// Handler returns an http.Handler for the prometheus.DefaultGatherer, using
// default HandlerOpts, i.e. it reports the first error as an HTTP error, it has
// no error logging, and it applies compression if requested by the client.
//
// If you want to create a Handler for the DefaultGatherer with different
// HandlerOpts, create it with HandlerFor with prometheus.DefaultGatherer and
// your desired HandlerOpts.
// The returned http.Handler is already instrumented using the
// InstrumentMetricHandler function and the prometheus.DefaultRegisterer. If you
// create multiple http.Handlers by separate calls of the Handler function, the
// metrics used for instrumentation will be shared between them, providing
// global scrape counts.
//
// This function is meant to cover the bulk of basic use cases. If you are doing
// anything that requires more customization (including using a non-default
// Gatherer, different instrumentation, and non-default HandlerOpts), use the
// HandlerFor function. See there for details.
func Handler() http.Handler {
return HandlerFor(prometheus.DefaultGatherer, HandlerOpts{})
return InstrumentMetricHandler(
prometheus.DefaultRegisterer, HandlerFor(prometheus.DefaultGatherer, HandlerOpts{}),
)
}
// HandlerFor returns an http.Handler for the provided Gatherer. The behavior
// of the Handler is defined by the provided HandlerOpts.
// HandlerFor returns an uninstrumented http.Handler for the provided
// Gatherer. The behavior of the Handler is defined by the provided
// HandlerOpts. Thus, HandlerFor is useful to create http.Handlers for custom
// Gatherers, with non-default HandlerOpts, and/or with custom (or no)
// instrumentation. Use the InstrumentMetricHandler function to apply the same
// kind of instrumentation as it is used by the Handler function.
func HandlerFor(reg prometheus.Gatherer, opts HandlerOpts) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
var inFlightSem chan struct{}
if opts.MaxRequestsInFlight > 0 {
inFlightSem = make(chan struct{}, opts.MaxRequestsInFlight)
}
h := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
if inFlightSem != nil {
select {
case inFlightSem <- struct{}{}: // All good, carry on.
defer func() { <-inFlightSem }()
default:
http.Error(w, fmt.Sprintf(
"Limit of concurrent requests reached (%d), try again later.", opts.MaxRequestsInFlight,
), http.StatusServiceUnavailable)
return
}
}
mfs, err := reg.Gather()
if err != nil {
if opts.ErrorLog != nil {
@@ -137,9 +168,70 @@ func HandlerFor(reg prometheus.Gatherer, opts HandlerOpts) http.Handler {
if encoding != "" {
header.Set(contentEncodingHeader, encoding)
}
w.Write(buf.Bytes())
if _, err := w.Write(buf.Bytes()); err != nil && opts.ErrorLog != nil {
opts.ErrorLog.Println("error while sending encoded metrics:", err)
}
// TODO(beorn7): Consider streaming serving of metrics.
})
if opts.Timeout <= 0 {
return h
}
return http.TimeoutHandler(h, opts.Timeout, fmt.Sprintf(
"Exceeded configured timeout of %v.\n",
opts.Timeout,
))
}
// InstrumentMetricHandler is usually used with an http.Handler returned by the
// HandlerFor function. It instruments the provided http.Handler with two
// metrics: A counter vector "promhttp_metric_handler_requests_total" to count
// scrapes partitioned by HTTP status code, and a gauge
// "promhttp_metric_handler_requests_in_flight" to track the number of
// simultaneous scrapes. This function idempotently registers collectors for
// both metrics with the provided Registerer. It panics if the registration
// fails. The provided metrics are useful to see how many scrapes hit the
// monitored target (which could be from different Prometheus servers or other
// scrapers), and how often they overlap (which would result in more than one
// scrape in flight at the same time). Note that the scrapes-in-flight gauge
// will contain the scrape by which it is exposed, while the scrape counter will
// only get incremented after the scrape is complete (as only then the status
// code is known). For tracking scrape durations, use the
// "scrape_duration_seconds" gauge created by the Prometheus server upon each
// scrape.
func InstrumentMetricHandler(reg prometheus.Registerer, handler http.Handler) http.Handler {
cnt := prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "promhttp_metric_handler_requests_total",
Help: "Total number of scrapes by HTTP status code.",
},
[]string{"code"},
)
// Initialize the most likely HTTP status codes.
cnt.WithLabelValues("200")
cnt.WithLabelValues("500")
cnt.WithLabelValues("503")
if err := reg.Register(cnt); err != nil {
if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
cnt = are.ExistingCollector.(*prometheus.CounterVec)
} else {
panic(err)
}
}
gge := prometheus.NewGauge(prometheus.GaugeOpts{
Name: "promhttp_metric_handler_requests_in_flight",
Help: "Current number of scrapes being served.",
})
if err := reg.Register(gge); err != nil {
if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
gge = are.ExistingCollector.(prometheus.Gauge)
} else {
panic(err)
}
}
return InstrumentHandlerCounter(cnt, InstrumentHandlerInFlight(gge, handler))
}
// HandlerErrorHandling defines how a Handler serving metrics will handle
@@ -183,6 +275,21 @@ type HandlerOpts struct {
// If DisableCompression is true, the handler will never compress the
// response, even if requested by the client.
DisableCompression bool
// The number of concurrent HTTP requests is limited to
// MaxRequestsInFlight. Additional requests are responded to with 503
// Service Unavailable and a suitable message in the body. If
// MaxRequestsInFlight is 0 or negative, no limit is applied.
MaxRequestsInFlight int
// If handling a request takes longer than Timeout, it is responded to
// with 503 ServiceUnavailable and a suitable Message. No timeout is
// applied if Timeout is 0 or negative. Note that with the current
// implementation, reaching the timeout simply ends the HTTP requests as
// described above (and even that only if sending of the body hasn't
// started yet), while the bulk work of gathering all the metrics keeps
// running in the background (with the eventual result to be thrown
// away). Until the implementation is improved, it is recommended to
// implement a separate timeout in potentially slow Collectors.
Timeout time.Duration
}
// decorateWriter wraps a writer to handle gzip compression if requested. It

119
vendor/github.com/prometheus/client_golang/prometheus/promhttp/http_test.go сгенерированный поставляемый
Просмотреть файл

@@ -19,7 +19,9 @@ import (
"log"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/prometheus/client_golang/prometheus"
)
@@ -37,6 +39,23 @@ func (e errorCollector) Collect(ch chan<- prometheus.Metric) {
)
}
type blockingCollector struct {
CollectStarted, Block chan struct{}
}
func (b blockingCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- prometheus.NewDesc("dummy_desc", "not helpful", nil, nil)
}
func (b blockingCollector) Collect(ch chan<- prometheus.Metric) {
select {
case b.CollectStarted <- struct{}{}:
default:
}
// Collects nothing, just waits for a channel receive.
<-b.Block
}
func TestHandlerErrorHandling(t *testing.T) {
// Create a registry that collects a MetricFamily with two elements,
@@ -129,3 +148,103 @@ the_count 0
}()
panicHandler.ServeHTTP(writer, request)
}
func TestInstrumentMetricHandler(t *testing.T) {
reg := prometheus.NewRegistry()
handler := InstrumentMetricHandler(reg, HandlerFor(reg, HandlerOpts{}))
// Do it again to test idempotency.
InstrumentMetricHandler(reg, HandlerFor(reg, HandlerOpts{}))
writer := httptest.NewRecorder()
request, _ := http.NewRequest("GET", "/", nil)
request.Header.Add("Accept", "test/plain")
handler.ServeHTTP(writer, request)
if got, want := writer.Code, http.StatusOK; got != want {
t.Errorf("got HTTP status code %d, want %d", got, want)
}
want := "promhttp_metric_handler_requests_in_flight 1\n"
if got := writer.Body.String(); !strings.Contains(got, want) {
t.Errorf("got body %q, does not contain %q", got, want)
}
want = "promhttp_metric_handler_requests_total{code=\"200\"} 0\n"
if got := writer.Body.String(); !strings.Contains(got, want) {
t.Errorf("got body %q, does not contain %q", got, want)
}
writer.Body.Reset()
handler.ServeHTTP(writer, request)
if got, want := writer.Code, http.StatusOK; got != want {
t.Errorf("got HTTP status code %d, want %d", got, want)
}
want = "promhttp_metric_handler_requests_in_flight 1\n"
if got := writer.Body.String(); !strings.Contains(got, want) {
t.Errorf("got body %q, does not contain %q", got, want)
}
want = "promhttp_metric_handler_requests_total{code=\"200\"} 1\n"
if got := writer.Body.String(); !strings.Contains(got, want) {
t.Errorf("got body %q, does not contain %q", got, want)
}
}
func TestHandlerMaxRequestsInFlight(t *testing.T) {
reg := prometheus.NewRegistry()
handler := HandlerFor(reg, HandlerOpts{MaxRequestsInFlight: 1})
w1 := httptest.NewRecorder()
w2 := httptest.NewRecorder()
w3 := httptest.NewRecorder()
request, _ := http.NewRequest("GET", "/", nil)
request.Header.Add("Accept", "test/plain")
c := blockingCollector{Block: make(chan struct{}), CollectStarted: make(chan struct{}, 1)}
reg.MustRegister(c)
rq1Done := make(chan struct{})
go func() {
handler.ServeHTTP(w1, request)
close(rq1Done)
}()
<-c.CollectStarted
handler.ServeHTTP(w2, request)
if got, want := w2.Code, http.StatusServiceUnavailable; got != want {
t.Errorf("got HTTP status code %d, want %d", got, want)
}
if got, want := w2.Body.String(), "Limit of concurrent requests reached (1), try again later.\n"; got != want {
t.Errorf("got body %q, want %q", got, want)
}
close(c.Block)
<-rq1Done
handler.ServeHTTP(w3, request)
if got, want := w3.Code, http.StatusOK; got != want {
t.Errorf("got HTTP status code %d, want %d", got, want)
}
}
func TestHandlerTimeout(t *testing.T) {
reg := prometheus.NewRegistry()
handler := HandlerFor(reg, HandlerOpts{Timeout: time.Millisecond})
w := httptest.NewRecorder()
request, _ := http.NewRequest("GET", "/", nil)
request.Header.Add("Accept", "test/plain")
c := blockingCollector{Block: make(chan struct{}), CollectStarted: make(chan struct{}, 1)}
reg.MustRegister(c)
handler.ServeHTTP(w, request)
if got, want := w.Code, http.StatusServiceUnavailable; got != want {
t.Errorf("got HTTP status code %d, want %d", got, want)
}
if got, want := w.Body.String(), "Exceeded configured timeout of 1ms.\n"; got != want {
t.Errorf("got body %q, want %q", got, want)
}
close(c.Block) // To not leak a goroutine.
}

26
vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server_test.go сгенерированный поставляемый
Просмотреть файл

@@ -281,6 +281,16 @@ func (t *testResponseWriter) ReadFrom(io.Reader) (int64, error) {
return 0, nil
}
// testFlusher is an http.ResponseWriter that also implements http.Flusher.
type testFlusher struct {
flushCalled bool
}
func (t *testFlusher) Header() http.Header { return nil }
func (t *testFlusher) Write([]byte) (int, error) { return 0, nil }
func (t *testFlusher) WriteHeader(int) {}
func (t *testFlusher) Flush() { t.flushCalled = true }
func TestInterfaceUpgrade(t *testing.T) {
w := &testResponseWriter{}
d := newDelegator(w, nil)
@@ -299,6 +309,22 @@ func TestInterfaceUpgrade(t *testing.T) {
if _, ok := d.(http.Hijacker); ok {
t.Error("delegator unexpectedly implements http.Hijacker")
}
f := &testFlusher{}
d = newDelegator(f, nil)
if _, ok := d.(http.CloseNotifier); ok {
t.Error("delegator unexpectedly implements http.CloseNotifier")
}
d.(http.Flusher).Flush()
if !w.flushCalled {
t.Error("Flush not called")
}
if _, ok := d.(io.ReaderFrom); ok {
t.Error("delegator unexpectedly implements io.ReaderFrom")
}
if _, ok := d.(http.Hijacker); ok {
t.Error("delegator unexpectedly implements http.Hijacker")
}
}
func ExampleInstrumentHandlerDuration() {

172
vendor/github.com/prometheus/client_golang/prometheus/push/deprecated.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,172 @@
// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package push
// This file contains only deprecated code. Remove after v0.9 is released.
import (
"bytes"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"os"
"strings"
"github.com/prometheus/common/expfmt"
"github.com/prometheus/common/model"
"github.com/prometheus/client_golang/prometheus"
)
// FromGatherer triggers a metric collection by the provided Gatherer (which is
// usually implemented by a prometheus.Registry) and pushes all gathered metrics
// to the Pushgateway specified by url, using the provided job name and the
// (optional) further grouping labels (the grouping map may be nil). See the
// Pushgateway documentation for detailed implications of the job and other
// grouping labels. Neither the job name nor any grouping label value may
// contain a "/". The metrics pushed must not contain a job label of their own
// nor any of the grouping labels.
//
// You can use just host:port or ip:port as url, in which case 'http://' is
// added automatically. You can also include the schema in the URL. However, do
// not include the '/metrics/jobs/...' part.
//
// Note that all previously pushed metrics with the same job and other grouping
// labels will be replaced with the metrics pushed by this call. (It uses HTTP
// method 'PUT' to push to the Pushgateway.)
//
// Deprecated: Please use a Pusher created with New instead.
func FromGatherer(job string, grouping map[string]string, url string, g prometheus.Gatherer) error {
return push(job, grouping, url, g, "PUT")
}
// AddFromGatherer works like FromGatherer, but only previously pushed metrics
// with the same name (and the same job and other grouping labels) will be
// replaced. (It uses HTTP method 'POST' to push to the Pushgateway.)
//
// Deprecated: Please use a Pusher created with New instead.
func AddFromGatherer(job string, grouping map[string]string, url string, g prometheus.Gatherer) error {
return push(job, grouping, url, g, "POST")
}
func push(job string, grouping map[string]string, pushURL string, g prometheus.Gatherer, method string) error {
if !strings.Contains(pushURL, "://") {
pushURL = "http://" + pushURL
}
if strings.HasSuffix(pushURL, "/") {
pushURL = pushURL[:len(pushURL)-1]
}
if strings.Contains(job, "/") {
return fmt.Errorf("job contains '/': %s", job)
}
urlComponents := []string{url.QueryEscape(job)}
for ln, lv := range grouping {
if !model.LabelName(ln).IsValid() {
return fmt.Errorf("grouping label has invalid name: %s", ln)
}
if strings.Contains(lv, "/") {
return fmt.Errorf("value of grouping label %s contains '/': %s", ln, lv)
}
urlComponents = append(urlComponents, ln, lv)
}
pushURL = fmt.Sprintf("%s/metrics/job/%s", pushURL, strings.Join(urlComponents, "/"))
mfs, err := g.Gather()
if err != nil {
return err
}
buf := &bytes.Buffer{}
enc := expfmt.NewEncoder(buf, expfmt.FmtProtoDelim)
// Check for pre-existing grouping labels:
for _, mf := range mfs {
for _, m := range mf.GetMetric() {
for _, l := range m.GetLabel() {
if l.GetName() == "job" {
return fmt.Errorf("pushed metric %s (%s) already contains a job label", mf.GetName(), m)
}
if _, ok := grouping[l.GetName()]; ok {
return fmt.Errorf(
"pushed metric %s (%s) already contains grouping label %s",
mf.GetName(), m, l.GetName(),
)
}
}
}
enc.Encode(mf)
}
req, err := http.NewRequest(method, pushURL, buf)
if err != nil {
return err
}
req.Header.Set(contentTypeHeader, string(expfmt.FmtProtoDelim))
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != 202 {
body, _ := ioutil.ReadAll(resp.Body) // Ignore any further error as this is for an error message only.
return fmt.Errorf("unexpected status code %d while pushing to %s: %s", resp.StatusCode, pushURL, body)
}
return nil
}
// Collectors works like FromGatherer, but it does not use a Gatherer. Instead,
// it collects from the provided collectors directly. It is a convenient way to
// push only a few metrics.
//
// Deprecated: Please use a Pusher created with New instead.
func Collectors(job string, grouping map[string]string, url string, collectors ...prometheus.Collector) error {
return pushCollectors(job, grouping, url, "PUT", collectors...)
}
// AddCollectors works like AddFromGatherer, but it does not use a Gatherer.
// Instead, it collects from the provided collectors directly. It is a
// convenient way to push only a few metrics.
//
// Deprecated: Please use a Pusher created with New instead.
func AddCollectors(job string, grouping map[string]string, url string, collectors ...prometheus.Collector) error {
return pushCollectors(job, grouping, url, "POST", collectors...)
}
func pushCollectors(job string, grouping map[string]string, url, method string, collectors ...prometheus.Collector) error {
r := prometheus.NewRegistry()
for _, collector := range collectors {
if err := r.Register(collector); err != nil {
return err
}
}
return push(job, grouping, url, r, method)
}
// HostnameGroupingKey returns a label map with the only entry
// {instance="<hostname>"}. This can be conveniently used as the grouping
// parameter if metrics should be pushed with the hostname as label. The
// returned map is created upon each call so that the caller is free to add more
// labels to the map.
//
// Deprecated: Usually, metrics pushed to the Pushgateway should not be
// host-centric. (You would use https://github.com/prometheus/node_exporter in
// that case.) If you have the need to add the hostname to the grouping key, you
// are probably doing something wrong. See
// https://prometheus.io/docs/practices/pushing/ for details.
func HostnameGroupingKey() map[string]string {
hostname, err := os.Hostname()
if err != nil {
return map[string]string{"instance": "unknown"}
}
return map[string]string{"instance": hostname}
}

32
vendor/github.com/prometheus/client_golang/prometheus/push/example_add_from_gatherer_test.go сгенерированный поставляемый
Просмотреть файл

@@ -11,12 +11,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright (c) 2013, The Prometheus Authors
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be found
// in the LICENSE file.
package push_test
import (
@@ -53,10 +47,14 @@ func performBackup() (int, error) {
return 42, nil
}
func ExampleAddFromGatherer() {
func ExamplePusher_Add() {
// We use a registry here to benefit from the consistency checks that
// happen during registration.
registry := prometheus.NewRegistry()
registry.MustRegister(completionTime, duration, records)
// Note that successTime is not registered at this time.
// Note that successTime is not registered.
pusher := push.New("http://pushgateway:9091", "db_backup").Gatherer(registry)
start := time.Now()
n, err := performBackup()
@@ -67,18 +65,16 @@ func ExampleAddFromGatherer() {
if err != nil {
fmt.Println("DB backup failed:", err)
} else {
// Only now register successTime.
registry.MustRegister(successTime)
// Add successTime to pusher only in case of success.
// We could as well register it with the registry.
// This example, however, demonstrates that you can
// mix Gatherers and Collectors when handling a Pusher.
pusher.Collector(successTime)
successTime.SetToCurrentTime()
}
// AddFromGatherer is used here rather than FromGatherer to not delete a
// previously pushed success timestamp in case of a failure of this
// backup.
if err := push.AddFromGatherer(
"db_backup", nil,
"http://pushgateway:9091",
registry,
); err != nil {
// Add is used here rather than Push to not delete a previously pushed
// success timestamp in case of a failure of this backup.
if err := pusher.Add(); err != nil {
fmt.Println("Could not push to Pushgateway:", err)
}
}

11
vendor/github.com/prometheus/client_golang/prometheus/push/examples_test.go сгенерированный поставляемый
Просмотреть файл

@@ -20,17 +20,16 @@ import (
"github.com/prometheus/client_golang/prometheus/push"
)
func ExampleCollectors() {
func ExamplePusher_Push() {
completionTime := prometheus.NewGauge(prometheus.GaugeOpts{
Name: "db_backup_last_completion_timestamp_seconds",
Help: "The timestamp of the last successful completion of a DB backup.",
})
completionTime.SetToCurrentTime()
if err := push.Collectors(
"db_backup", push.HostnameGroupingKey(),
"http://pushgateway:9091",
completionTime,
); err != nil {
if err := push.New("http://pushgateway:9091", "db_backup").
Collector(completionTime).
Grouping("db", "customers").
Push(); err != nil {
fmt.Println("Could not push completion time to Pushgateway:", err)
}
}

248
vendor/github.com/prometheus/client_golang/prometheus/push/push.go сгенерированный поставляемый
Просмотреть файл

@@ -11,20 +11,27 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright (c) 2013, The Prometheus Authors
// All rights reserved.
// Package push provides functions to push metrics to a Pushgateway. It uses a
// builder approach. Create a Pusher with New and then add the various options
// by using its methods, finally calling Add or Push, like this:
//
// Use of this source code is governed by a BSD-style license that can be found
// in the LICENSE file.
// Package push provides functions to push metrics to a Pushgateway. The metrics
// to push are either collected from a provided registry, or from explicitly
// listed collectors.
// // Easy case:
// push.New("http://example.org/metrics", "my_job").Gatherer(myRegistry).Push()
//
// See the documentation of the Pushgateway to understand the meaning of the
// grouping parameters and the differences between push.Registry and
// push.Collectors on the one hand and push.AddRegistry and push.AddCollectors
// on the other hand: https://github.com/prometheus/pushgateway
// // Complex case:
// push.New("http://example.org/metrics", "my_job").
// Collector(myCollector1).
// Collector(myCollector2).
// Grouping("zone", "xy").
// Client(&myHTTPClient).
// BasicAuth("top", "secret").
// Add()
//
// See the examples section for more detailed examples.
//
// See the documentation of the Pushgateway to understand the meaning of
// the grouping key and the differences between Push and Add:
// https://github.com/prometheus/pushgateway
package push
import (
@@ -33,7 +40,6 @@ import (
"io/ioutil"
"net/http"
"net/url"
"os"
"strings"
"github.com/prometheus/common/expfmt"
@@ -44,57 +50,149 @@ import (
const contentTypeHeader = "Content-Type"
// FromGatherer triggers a metric collection by the provided Gatherer (which is
// usually implemented by a prometheus.Registry) and pushes all gathered metrics
// to the Pushgateway specified by url, using the provided job name and the
// (optional) further grouping labels (the grouping map may be nil). See the
// Pushgateway documentation for detailed implications of the job and other
// grouping labels. Neither the job name nor any grouping label value may
// contain a "/". The metrics pushed must not contain a job label of their own
// nor any of the grouping labels.
//
// You can use just host:port or ip:port as url, in which case 'http://' is
// added automatically. You can also include the schema in the URL. However, do
// not include the '/metrics/jobs/...' part.
//
// Note that all previously pushed metrics with the same job and other grouping
// labels will be replaced with the metrics pushed by this call. (It uses HTTP
// method 'PUT' to push to the Pushgateway.)
func FromGatherer(job string, grouping map[string]string, url string, g prometheus.Gatherer) error {
return push(job, grouping, url, g, "PUT")
// Pusher manages a push to the Pushgateway. Use New to create one, configure it
// with its methods, and finally use the Add or Push method to push.
type Pusher struct {
error error
url, job string
grouping map[string]string
gatherers prometheus.Gatherers
registerer prometheus.Registerer
client *http.Client
useBasicAuth bool
username, password string
}
// AddFromGatherer works like FromGatherer, but only previously pushed metrics
// with the same name (and the same job and other grouping labels) will be
// replaced. (It uses HTTP method 'POST' to push to the Pushgateway.)
func AddFromGatherer(job string, grouping map[string]string, url string, g prometheus.Gatherer) error {
return push(job, grouping, url, g, "POST")
}
func push(job string, grouping map[string]string, pushURL string, g prometheus.Gatherer, method string) error {
if !strings.Contains(pushURL, "://") {
pushURL = "http://" + pushURL
// New creates a new Pusher to push to the provided URL withe the provided job
// name. You can use just host:port or ip:port as url, in which case “http://”
// is added automatically. Alternatively, include the schema in the
// URL. However, do not include the “/metrics/jobs/…” part.
//
// Note that until https://github.com/prometheus/pushgateway/issues/97 is
// resolved, a “/” character in the job name is prohibited.
func New(url, job string) *Pusher {
var (
reg = prometheus.NewRegistry()
err error
)
if !strings.Contains(url, "://") {
url = "http://" + url
}
if strings.HasSuffix(pushURL, "/") {
pushURL = pushURL[:len(pushURL)-1]
if strings.HasSuffix(url, "/") {
url = url[:len(url)-1]
}
if strings.Contains(job, "/") {
return fmt.Errorf("job contains '/': %s", job)
err = fmt.Errorf("job contains '/': %s", job)
}
urlComponents := []string{url.QueryEscape(job)}
for ln, lv := range grouping {
if !model.LabelName(ln).IsValid() {
return fmt.Errorf("grouping label has invalid name: %s", ln)
return &Pusher{
error: err,
url: url,
job: job,
grouping: map[string]string{},
gatherers: prometheus.Gatherers{reg},
registerer: reg,
client: &http.Client{},
}
}
// Push collects/gathers all metrics from all Collectors and Gatherers added to
// this Pusher. Then, it pushes them to the Pushgateway configured while
// creating this Pusher, using the configured job name and any added grouping
// labels as grouping key. All previously pushed metrics with the same job and
// other grouping labels will be replaced with the metrics pushed by this
// call. (It uses HTTP method “PUT” to push to the Pushgateway.)
//
// Push returns the first error encountered by any method call (including this
// one) in the lifetime of the Pusher.
func (p *Pusher) Push() error {
return p.push("PUT")
}
// Add works like push, but only previously pushed metrics with the same name
// (and the same job and other grouping labels) will be replaced. (It uses HTTP
// method “POST” to push to the Pushgateway.)
func (p *Pusher) Add() error {
return p.push("POST")
}
// Gatherer adds a Gatherer to the Pusher, from which metrics will be gathered
// to push them to the Pushgateway. The gathered metrics must not contain a job
// label of their own.
//
// For convenience, this method returns a pointer to the Pusher itself.
func (p *Pusher) Gatherer(g prometheus.Gatherer) *Pusher {
p.gatherers = append(p.gatherers, g)
return p
}
// Collector adds a Collector to the Pusher, from which metrics will be
// collected to push them to the Pushgateway. The collected metrics must not
// contain a job label of their own.
//
// For convenience, this method returns a pointer to the Pusher itself.
func (p *Pusher) Collector(c prometheus.Collector) *Pusher {
if p.error == nil {
p.error = p.registerer.Register(c)
}
return p
}
// Grouping adds a label pair to the grouping key of the Pusher, replacing any
// previously added label pair with the same label name. Note that setting any
// labels in the grouping key that are already contained in the metrics to push
// will lead to an error.
//
// For convenience, this method returns a pointer to the Pusher itself.
//
// Note that until https://github.com/prometheus/pushgateway/issues/97 is
// resolved, this method does not allow a “/” character in the label value.
func (p *Pusher) Grouping(name, value string) *Pusher {
if p.error == nil {
if !model.LabelName(name).IsValid() {
p.error = fmt.Errorf("grouping label has invalid name: %s", name)
return p
}
if strings.Contains(lv, "/") {
return fmt.Errorf("value of grouping label %s contains '/': %s", ln, lv)
if strings.Contains(value, "/") {
p.error = fmt.Errorf("value of grouping label %s contains '/': %s", name, value)
return p
}
p.grouping[name] = value
}
return p
}
// Client sets a custom HTTP client for the Pusher. For convenience, this method
// returns a pointer to the Pusher itself.
func (p *Pusher) Client(c *http.Client) *Pusher {
p.client = c
return p
}
// BasicAuth configures the Pusher to use HTTP Basic Authentication with the
// provided username and password. For convenience, this method returns a
// pointer to the Pusher itself.
func (p *Pusher) BasicAuth(username, password string) *Pusher {
p.useBasicAuth = true
p.username = username
p.password = password
return p
}
func (p *Pusher) push(method string) error {
if p.error != nil {
return p.error
}
urlComponents := []string{url.QueryEscape(p.job)}
for ln, lv := range p.grouping {
urlComponents = append(urlComponents, ln, lv)
}
pushURL = fmt.Sprintf("%s/metrics/job/%s", pushURL, strings.Join(urlComponents, "/"))
pushURL := fmt.Sprintf("%s/metrics/job/%s", p.url, strings.Join(urlComponents, "/"))
mfs, err := g.Gather()
mfs, err := p.gatherers.Gather()
if err != nil {
return err
}
@@ -107,7 +205,7 @@ func push(job string, grouping map[string]string, pushURL string, g prometheus.G
if l.GetName() == "job" {
return fmt.Errorf("pushed metric %s (%s) already contains a job label", mf.GetName(), m)
}
if _, ok := grouping[l.GetName()]; ok {
if _, ok := p.grouping[l.GetName()]; ok {
return fmt.Errorf(
"pushed metric %s (%s) already contains grouping label %s",
mf.GetName(), m, l.GetName(),
@@ -121,8 +219,11 @@ func push(job string, grouping map[string]string, pushURL string, g prometheus.G
if err != nil {
return err
}
if p.useBasicAuth {
req.SetBasicAuth(p.username, p.password)
}
req.Header.Set(contentTypeHeader, string(expfmt.FmtProtoDelim))
resp, err := http.DefaultClient.Do(req)
resp, err := p.client.Do(req)
if err != nil {
return err
}
@@ -133,40 +234,3 @@ func push(job string, grouping map[string]string, pushURL string, g prometheus.G
}
return nil
}
// Collectors works like FromGatherer, but it does not use a Gatherer. Instead,
// it collects from the provided collectors directly. It is a convenient way to
// push only a few metrics.
func Collectors(job string, grouping map[string]string, url string, collectors ...prometheus.Collector) error {
return pushCollectors(job, grouping, url, "PUT", collectors...)
}
// AddCollectors works like AddFromGatherer, but it does not use a Gatherer.
// Instead, it collects from the provided collectors directly. It is a
// convenient way to push only a few metrics.
func AddCollectors(job string, grouping map[string]string, url string, collectors ...prometheus.Collector) error {
return pushCollectors(job, grouping, url, "POST", collectors...)
}
func pushCollectors(job string, grouping map[string]string, url, method string, collectors ...prometheus.Collector) error {
r := prometheus.NewRegistry()
for _, collector := range collectors {
if err := r.Register(collector); err != nil {
return err
}
}
return push(job, grouping, url, r, method)
}
// HostnameGroupingKey returns a label map with the only entry
// {instance="<hostname>"}. This can be conveniently used as the grouping
// parameter if metrics should be pushed with the hostname as label. The
// returned map is created upon each call so that the caller is free to add more
// labels to the map.
func HostnameGroupingKey() map[string]string {
hostname, err := os.Hostname()
if err != nil {
return map[string]string{"instance": "unknown"}
}
return map[string]string{"instance": hostname}
}

98
vendor/github.com/prometheus/client_golang/prometheus/push/push_test.go сгенерированный поставляемый
Просмотреть файл

@@ -11,12 +11,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright (c) 2013, The Prometheus Authors
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be found
// in the LICENSE file.
package push
import (
@@ -24,7 +18,6 @@ import (
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"testing"
"github.com/prometheus/common/expfmt"
@@ -40,11 +33,6 @@ func TestPush(t *testing.T) {
lastPath string
)
host, err := os.Hostname()
if err != nil {
t.Error(err)
}
// Fake a Pushgateway that always responds with 202.
pgwOK := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -98,26 +86,15 @@ func TestPush(t *testing.T) {
}
wantBody := buf.Bytes()
// PushCollectors, all good.
if err := Collectors("testjob", HostnameGroupingKey(), pgwOK.URL, metric1, metric2); err != nil {
// Push some Collectors, all good.
if err := New(pgwOK.URL, "testjob").
Collector(metric1).
Collector(metric2).
Push(); err != nil {
t.Fatal(err)
}
if lastMethod != "PUT" {
t.Error("want method PUT for PushCollectors, got", lastMethod)
}
if bytes.Compare(lastBody, wantBody) != 0 {
t.Errorf("got body %v, want %v", lastBody, wantBody)
}
if lastPath != "/metrics/job/testjob/instance/"+host {
t.Error("unexpected path:", lastPath)
}
// PushAddCollectors, with nil grouping, all good.
if err := AddCollectors("testjob", nil, pgwOK.URL, metric1, metric2); err != nil {
t.Fatal(err)
}
if lastMethod != "POST" {
t.Error("want method POST for PushAddCollectors, got", lastMethod)
t.Error("want method PUT for Push, got", lastMethod)
}
if bytes.Compare(lastBody, wantBody) != 0 {
t.Errorf("got body %v, want %v", lastBody, wantBody)
@@ -126,8 +103,28 @@ func TestPush(t *testing.T) {
t.Error("unexpected path:", lastPath)
}
// PushCollectors with a broken PGW.
if err := Collectors("testjob", nil, pgwErr.URL, metric1, metric2); err == nil {
// Add some Collectors, with nil grouping, all good.
if err := New(pgwOK.URL, "testjob").
Collector(metric1).
Collector(metric2).
Add(); err != nil {
t.Fatal(err)
}
if lastMethod != "POST" {
t.Error("want method POST for Add, got", lastMethod)
}
if bytes.Compare(lastBody, wantBody) != 0 {
t.Errorf("got body %v, want %v", lastBody, wantBody)
}
if lastPath != "/metrics/job/testjob" {
t.Error("unexpected path:", lastPath)
}
// Push some Collectors with a broken PGW.
if err := New(pgwErr.URL, "testjob").
Collector(metric1).
Collector(metric2).
Push(); err == nil {
t.Error("push to broken Pushgateway succeeded")
} else {
if got, want := err.Error(), "unexpected status code 500 while pushing to "+pgwErr.URL+"/metrics/job/testjob: fake error\n"; got != want {
@@ -135,22 +132,39 @@ func TestPush(t *testing.T) {
}
}
// PushCollectors with invalid grouping or job.
if err := Collectors("testjob", map[string]string{"foo": "bums"}, pgwErr.URL, metric1, metric2); err == nil {
// Push some Collectors with invalid grouping or job.
if err := New(pgwOK.URL, "testjob").
Grouping("foo", "bums").
Collector(metric1).
Collector(metric2).
Push(); err == nil {
t.Error("push with grouping contained in metrics succeeded")
}
if err := Collectors("test/job", nil, pgwErr.URL, metric1, metric2); err == nil {
if err := New(pgwOK.URL, "test/job").
Collector(metric1).
Collector(metric2).
Push(); err == nil {
t.Error("push with invalid job value succeeded")
}
if err := Collectors("testjob", map[string]string{"foo/bar": "bums"}, pgwErr.URL, metric1, metric2); err == nil {
if err := New(pgwOK.URL, "testjob").
Grouping("foobar", "bu/ms").
Collector(metric1).
Collector(metric2).
Push(); err == nil {
t.Error("push with invalid grouping succeeded")
}
if err := Collectors("testjob", map[string]string{"foo-bar": "bums"}, pgwErr.URL, metric1, metric2); err == nil {
if err := New(pgwOK.URL, "testjob").
Grouping("foo-bar", "bums").
Collector(metric1).
Collector(metric2).
Push(); err == nil {
t.Error("push with invalid grouping succeeded")
}
// Push registry, all good.
if err := FromGatherer("testjob", HostnameGroupingKey(), pgwOK.URL, reg); err != nil {
if err := New(pgwOK.URL, "testjob").
Gatherer(reg).
Push(); err != nil {
t.Fatal(err)
}
if lastMethod != "PUT" {
@@ -160,12 +174,16 @@ func TestPush(t *testing.T) {
t.Errorf("got body %v, want %v", lastBody, wantBody)
}
// PushAdd registry, all good.
if err := AddFromGatherer("testjob", map[string]string{"a": "x", "b": "y"}, pgwOK.URL, reg); err != nil {
// Add registry, all good.
if err := New(pgwOK.URL, "testjob").
Grouping("a", "x").
Grouping("b", "y").
Gatherer(reg).
Add(); err != nil {
t.Fatal(err)
}
if lastMethod != "POST" {
t.Error("want method POSTT for PushAdd, got", lastMethod)
t.Error("want method POST for Add, got", lastMethod)
}
if bytes.Compare(lastBody, wantBody) != 0 {
t.Errorf("got body %v, want %v", lastBody, wantBody)

303
vendor/github.com/prometheus/client_golang/prometheus/registry.go сгенерированный поставляемый
Просмотреть файл

@@ -18,6 +18,7 @@ import (
"errors"
"fmt"
"os"
"runtime"
"sort"
"sync"
"unicode/utf8"
@@ -36,13 +37,13 @@ const (
// DefaultRegisterer and DefaultGatherer are the implementations of the
// Registerer and Gatherer interface a number of convenience functions in this
// package act on. Initially, both variables point to the same Registry, which
// has a process collector (see NewProcessCollector) and a Go collector (see
// NewGoCollector) already registered. This approach to keep default instances
// as global state mirrors the approach of other packages in the Go standard
// library. Note that there are caveats. Change the variables with caution and
// only if you understand the consequences. Users who want to avoid global state
// altogether should not use the convenience function and act on custom
// instances instead.
// has a process collector (currently on Linux only, see NewProcessCollector)
// and a Go collector (see NewGoCollector) already registered. This approach to
// keep default instances as global state mirrors the approach of other packages
// in the Go standard library. Note that there are caveats. Change the variables
// with caution and only if you understand the consequences. Users who want to
// avoid global state altogether should not use the convenience functions and
// act on custom instances instead.
var (
defaultRegistry = NewRegistry()
DefaultRegisterer Registerer = defaultRegistry
@@ -202,6 +203,13 @@ func (errs MultiError) Error() string {
return buf.String()
}
// Append appends the provided error if it is not nil.
func (errs *MultiError) Append(err error) {
if err != nil {
*errs = append(*errs, err)
}
}
// MaybeUnwrap returns nil if len(errs) is 0. It returns the first and only
// contained error as error if len(errs is 1). In all other cases, it returns
// the MultiError directly. This is helpful for returning a MultiError in a way
@@ -368,22 +376,12 @@ func (r *Registry) Gather() ([]*dto.MetricFamily, error) {
)
r.mtx.RLock()
goroutineBudget := len(r.collectorsByID)
metricFamiliesByName := make(map[string]*dto.MetricFamily, len(r.dimHashesByName))
// Scatter.
// (Collectors could be complex and slow, so we call them all at once.)
wg.Add(len(r.collectorsByID))
go func() {
wg.Wait()
close(metricChan)
}()
collectors := make(chan Collector, len(r.collectorsByID))
for _, collector := range r.collectorsByID {
go func(collector Collector) {
defer wg.Done()
collector.Collect(metricChan)
}(collector)
collectors <- collector
}
// In case pedantic checks are enabled, we have to copy the map before
// giving up the RLock.
if r.pedanticChecksEnabled {
@@ -392,129 +390,176 @@ func (r *Registry) Gather() ([]*dto.MetricFamily, error) {
registeredDescIDs[id] = struct{}{}
}
}
r.mtx.RUnlock()
wg.Add(goroutineBudget)
collectWorker := func() {
for {
select {
case collector := <-collectors:
collector.Collect(metricChan)
wg.Done()
default:
return
}
}
}
// Start the first worker now to make sure at least one is running.
go collectWorker()
goroutineBudget--
// Close the metricChan once all collectors are collected.
go func() {
wg.Wait()
close(metricChan)
}()
// Drain metricChan in case of premature return.
defer func() {
for range metricChan {
}
}()
// Gather.
for metric := range metricChan {
// This could be done concurrently, too, but it required locking
// of metricFamiliesByName (and of metricHashes if checks are
// enabled). Most likely not worth it.
desc := metric.Desc()
dtoMetric := &dto.Metric{}
if err := metric.Write(dtoMetric); err != nil {
errs = append(errs, fmt.Errorf(
"error collecting metric %v: %s", desc, err,
collectLoop:
for {
select {
case metric, ok := <-metricChan:
if !ok {
// metricChan is closed, we are done.
break collectLoop
}
errs.Append(processMetric(
metric, metricFamiliesByName,
metricHashes, dimHashes,
registeredDescIDs,
))
continue
default:
if goroutineBudget <= 0 || len(collectors) == 0 {
// All collectors are aleady being worked on or
// we have already as many goroutines started as
// there are collectors. Just process metrics
// from now on.
for metric := range metricChan {
errs.Append(processMetric(
metric, metricFamiliesByName,
metricHashes, dimHashes,
registeredDescIDs,
))
}
break collectLoop
}
// Start more workers.
go collectWorker()
goroutineBudget--
runtime.Gosched()
}
metricFamily, ok := metricFamiliesByName[desc.fqName]
if ok {
if metricFamily.GetHelp() != desc.help {
errs = append(errs, fmt.Errorf(
"collected metric %s %s has help %q but should have %q",
desc.fqName, dtoMetric, desc.help, metricFamily.GetHelp(),
))
continue
}
// TODO(beorn7): Simplify switch once Desc has type.
switch metricFamily.GetType() {
case dto.MetricType_COUNTER:
if dtoMetric.Counter == nil {
errs = append(errs, fmt.Errorf(
"collected metric %s %s should be a Counter",
desc.fqName, dtoMetric,
))
continue
}
case dto.MetricType_GAUGE:
if dtoMetric.Gauge == nil {
errs = append(errs, fmt.Errorf(
"collected metric %s %s should be a Gauge",
desc.fqName, dtoMetric,
))
continue
}
case dto.MetricType_SUMMARY:
if dtoMetric.Summary == nil {
errs = append(errs, fmt.Errorf(
"collected metric %s %s should be a Summary",
desc.fqName, dtoMetric,
))
continue
}
case dto.MetricType_UNTYPED:
if dtoMetric.Untyped == nil {
errs = append(errs, fmt.Errorf(
"collected metric %s %s should be Untyped",
desc.fqName, dtoMetric,
))
continue
}
case dto.MetricType_HISTOGRAM:
if dtoMetric.Histogram == nil {
errs = append(errs, fmt.Errorf(
"collected metric %s %s should be a Histogram",
desc.fqName, dtoMetric,
))
continue
}
default:
panic("encountered MetricFamily with invalid type")
}
} else {
metricFamily = &dto.MetricFamily{}
metricFamily.Name = proto.String(desc.fqName)
metricFamily.Help = proto.String(desc.help)
// TODO(beorn7): Simplify switch once Desc has type.
switch {
case dtoMetric.Gauge != nil:
metricFamily.Type = dto.MetricType_GAUGE.Enum()
case dtoMetric.Counter != nil:
metricFamily.Type = dto.MetricType_COUNTER.Enum()
case dtoMetric.Summary != nil:
metricFamily.Type = dto.MetricType_SUMMARY.Enum()
case dtoMetric.Untyped != nil:
metricFamily.Type = dto.MetricType_UNTYPED.Enum()
case dtoMetric.Histogram != nil:
metricFamily.Type = dto.MetricType_HISTOGRAM.Enum()
default:
errs = append(errs, fmt.Errorf(
"empty metric collected: %s", dtoMetric,
))
continue
}
metricFamiliesByName[desc.fqName] = metricFamily
}
if err := checkMetricConsistency(metricFamily, dtoMetric, metricHashes, dimHashes); err != nil {
errs = append(errs, err)
continue
}
if r.pedanticChecksEnabled {
// Is the desc registered at all?
if _, exist := registeredDescIDs[desc.id]; !exist {
errs = append(errs, fmt.Errorf(
"collected metric %s %s with unregistered descriptor %s",
metricFamily.GetName(), dtoMetric, desc,
))
continue
}
if err := checkDescConsistency(metricFamily, dtoMetric, desc); err != nil {
errs = append(errs, err)
continue
}
}
metricFamily.Metric = append(metricFamily.Metric, dtoMetric)
}
return normalizeMetricFamilies(metricFamiliesByName), errs.MaybeUnwrap()
}
// processMetric is an internal helper method only used by the Gather method.
func processMetric(
metric Metric,
metricFamiliesByName map[string]*dto.MetricFamily,
metricHashes map[uint64]struct{},
dimHashes map[string]uint64,
registeredDescIDs map[uint64]struct{},
) error {
desc := metric.Desc()
dtoMetric := &dto.Metric{}
if err := metric.Write(dtoMetric); err != nil {
return fmt.Errorf("error collecting metric %v: %s", desc, err)
}
metricFamily, ok := metricFamiliesByName[desc.fqName]
if ok {
if metricFamily.GetHelp() != desc.help {
return fmt.Errorf(
"collected metric %s %s has help %q but should have %q",
desc.fqName, dtoMetric, desc.help, metricFamily.GetHelp(),
)
}
// TODO(beorn7): Simplify switch once Desc has type.
switch metricFamily.GetType() {
case dto.MetricType_COUNTER:
if dtoMetric.Counter == nil {
return fmt.Errorf(
"collected metric %s %s should be a Counter",
desc.fqName, dtoMetric,
)
}
case dto.MetricType_GAUGE:
if dtoMetric.Gauge == nil {
return fmt.Errorf(
"collected metric %s %s should be a Gauge",
desc.fqName, dtoMetric,
)
}
case dto.MetricType_SUMMARY:
if dtoMetric.Summary == nil {
return fmt.Errorf(
"collected metric %s %s should be a Summary",
desc.fqName, dtoMetric,
)
}
case dto.MetricType_UNTYPED:
if dtoMetric.Untyped == nil {
return fmt.Errorf(
"collected metric %s %s should be Untyped",
desc.fqName, dtoMetric,
)
}
case dto.MetricType_HISTOGRAM:
if dtoMetric.Histogram == nil {
return fmt.Errorf(
"collected metric %s %s should be a Histogram",
desc.fqName, dtoMetric,
)
}
default:
panic("encountered MetricFamily with invalid type")
}
} else {
metricFamily = &dto.MetricFamily{}
metricFamily.Name = proto.String(desc.fqName)
metricFamily.Help = proto.String(desc.help)
// TODO(beorn7): Simplify switch once Desc has type.
switch {
case dtoMetric.Gauge != nil:
metricFamily.Type = dto.MetricType_GAUGE.Enum()
case dtoMetric.Counter != nil:
metricFamily.Type = dto.MetricType_COUNTER.Enum()
case dtoMetric.Summary != nil:
metricFamily.Type = dto.MetricType_SUMMARY.Enum()
case dtoMetric.Untyped != nil:
metricFamily.Type = dto.MetricType_UNTYPED.Enum()
case dtoMetric.Histogram != nil:
metricFamily.Type = dto.MetricType_HISTOGRAM.Enum()
default:
return fmt.Errorf("empty metric collected: %s", dtoMetric)
}
metricFamiliesByName[desc.fqName] = metricFamily
}
if err := checkMetricConsistency(metricFamily, dtoMetric, metricHashes, dimHashes); err != nil {
return err
}
if registeredDescIDs != nil {
// Is the desc registered at all?
if _, exist := registeredDescIDs[desc.id]; !exist {
return fmt.Errorf(
"collected metric %s %s with unregistered descriptor %s",
metricFamily.GetName(), dtoMetric, desc,
)
}
if err := checkDescConsistency(metricFamily, dtoMetric, desc); err != nil {
return err
}
}
metricFamily.Metric = append(metricFamily.Metric, dtoMetric)
return nil
}
// Gatherers is a slice of Gatherer instances that implements the Gatherer
// interface itself. Its Gather method calls Gather on all Gatherers in the
// slice in order and returns the merged results. Errors returned from the

4
vendor/github.com/prometheus/procfs/fs.go сгенерированный поставляемый
Просмотреть файл

@@ -46,8 +46,8 @@ func (fs FS) XFSStats() (*xfs.Stats, error) {
return xfs.ParseStats(f)
}
// NFSdClientRPCStats retrieves NFS daemon RPC statistics.
func (fs FS) NFSdClientRPCStats() (*nfs.ClientRPCStats, error) {
// NFSClientRPCStats retrieves NFS client RPC statistics.
func (fs FS) NFSClientRPCStats() (*nfs.ClientRPCStats, error) {
f, err := os.Open(fs.Path("net/rpc/nfs"))
if err != nil {
return nil, err

13
vendor/github.com/prometheus/procfs/nfs/parse.go сгенерированный поставляемый
Просмотреть файл

@@ -178,8 +178,17 @@ func parseV3Stats(v []uint64) (V3Stats, error) {
func parseClientV4Stats(v []uint64) (ClientV4Stats, error) {
values := int(v[0])
if len(v[1:]) != values || values < 59 {
return ClientV4Stats{}, fmt.Errorf("invalid V4Stats line %q", v)
if len(v[1:]) != values {
return ClientV4Stats{}, fmt.Errorf("invalid ClientV4Stats line %q", v)
}
// This function currently supports mapping 59 NFS v4 client stats. Older
// kernels may emit fewer stats, so we must detect this and pad out the
// values to match the expected slice size.
if values < 59 {
newValues := make([]uint64, 60)
copy(newValues, v)
v = newValues
}
return ClientV4Stats{

Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше