Этот коммит содержится в:
Christopher Speller
2018-09-28 12:40:17 -07:00
коммит произвёл GitHub
родитель 006623e0f7
Коммит a8c01377bc
272 изменённых файлов: 23302 добавлений и 4225 удалений

18
vendor/github.com/minio/minio-go/README.md сгенерированный поставляемый
Просмотреть файл

@@ -14,9 +14,9 @@ go get -u github.com/minio/minio-go
## Initialize Minio Client
Minio client requires the following four parameters specified to connect to an Amazon S3 compatible object storage.
| Parameter | Description|
| Parameter | Description|
| :--- | :--- |
| endpoint | URL to object storage service. |
| endpoint | URL to object storage service. |
| accessKeyID | Access key is the user ID that uniquely identifies your account. |
| secretAccessKey | Secret key is the password to your account. |
| secure | Set this value to 'true' to enable secure (HTTPS) access. |
@@ -106,7 +106,7 @@ func main() {
### Run FileUploader
```sh
go run file-uploader.go
2016/08/13 17:03:28 Successfully created mymusic
2016/08/13 17:03:28 Successfully created mymusic
2016/08/13 17:03:40 Successfully uploaded golden-oldies.zip of size 16253413
mc ls play/mymusic/
@@ -114,7 +114,7 @@ mc ls play/mymusic/
```
## API Reference
The full API Reference is available here.
The full API Reference is available here.
* [Complete API Reference](https://docs.minio.io/docs/golang-client-api-reference)
@@ -154,6 +154,8 @@ The full API Reference is available here.
* [`RemoveObject`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObject)
* [`RemoveObjects`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObjects)
* [`RemoveIncompleteUpload`](https://docs.minio.io/docs/golang-client-api-reference#RemoveIncompleteUpload)
* [`SelectObjectContent`](https://docs.minio.io/docs/golang-client-api-reference#SelectObjectContent)
### API Reference : Presigned Operations
* [`PresignedGetObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedGetObject)
@@ -182,7 +184,11 @@ The full API Reference is available here.
* [setbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketpolicy.go)
* [getbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketpolicy.go)
* [listbucketpolicies.go](https://github.com/minio/minio-go/blob/master/examples/s3/listbucketpolicies.go)
### Full Examples : Bucket lifecycle Operations
* [setbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketlifecycle.go)
* [getbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketlifecycle.go)
### Full Examples : Bucket notification Operations
* [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go)
* [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go)
@@ -219,7 +225,7 @@ The full API Reference is available here.
## Explore Further
* [Complete Documentation](https://docs.minio.io)
* [Minio Go Client SDK API Reference](https://docs.minio.io/docs/golang-client-api-reference)
* [Minio Go Client SDK API Reference](https://docs.minio.io/docs/golang-client-api-reference)
* [Go Music Player App Full Application Example](https://docs.minio.io/docs/go-music-player-app)
## Contribute

33
vendor/github.com/minio/minio-go/api-compose-object.go сгенерированный поставляемый
Просмотреть файл

@@ -101,7 +101,11 @@ func (d *DestinationInfo) getUserMetaHeadersMap(withCopyDirectiveHeader bool) ma
r["x-amz-metadata-directive"] = "REPLACE"
}
for k, v := range d.userMetadata {
r["x-amz-meta-"+k] = v
if isAmzHeader(k) || isStandardHeader(k) || isStorageClassHeader(k) {
r[k] = v
} else {
r["x-amz-meta-"+k] = v
}
}
return r
}
@@ -373,15 +377,6 @@ func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo
fmt.Sprintf("Client side encryption is used in source object %s/%s", src.bucket, src.object))
}
// Since we did a HEAD to get size, we use the ETag
// value to make sure the object has not changed by
// the time we perform the copy. This is done, only if
// the user has not set their own ETag match
// condition.
if src.Headers.Get("x-amz-copy-source-if-match") == "" {
src.SetMatchETagCond(etag)
}
// Check if a segment is specified, and if so, is the
// segment within object bounds?
if src.start != -1 {
@@ -429,7 +424,15 @@ func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo
// Now, handle multipart-copy cases.
// 1. Initiate a new multipart upload.
// 1. Ensure that the object has not been changed while
// we are copying data.
for _, src := range srcs {
if src.Headers.Get("x-amz-copy-source-if-match") == "" {
src.SetMatchETagCond(etag)
}
}
// 2. Initiate a new multipart upload.
// Set user-metadata on the destination object. If no
// user-metadata is specified, and there is only one source,
@@ -449,13 +452,13 @@ func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo
return err
}
// 2. Perform copy part uploads
// 3. Perform copy part uploads
objParts := []CompletePart{}
partIndex := 1
for i, src := range srcs {
h := src.Headers
if src.encryption != nil {
src.encryption.Marshal(h)
encrypt.SSECopy(src.encryption).Marshal(h)
}
// Add destination encryption headers
if dst.encryption != nil {
@@ -480,14 +483,14 @@ func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo
return err
}
if progress != nil {
io.CopyN(ioutil.Discard, progress, start+end-1)
io.CopyN(ioutil.Discard, progress, end-start+1)
}
objParts = append(objParts, complPart)
partIndex++
}
}
// 3. Make final complete-multipart request.
// 4. Make final complete-multipart request.
_, err = c.completeMultipartUpload(ctx, dst.bucket, dst.object, uploadID,
completeMultipartUpload{Parts: objParts})
if err != nil {

77
vendor/github.com/minio/minio-go/api-get-lifecycle.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,77 @@
/*
* Minio Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2015-2017 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package minio
import (
"context"
"io/ioutil"
"net/http"
"net/url"
"github.com/minio/minio-go/pkg/s3utils"
)
// GetBucketLifecycle - get bucket lifecycle.
func (c Client) GetBucketLifecycle(bucketName string) (string, error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return "", err
}
bucketLifecycle, err := c.getBucketLifecycle(bucketName)
if err != nil {
errResponse := ToErrorResponse(err)
if errResponse.Code == "NoSuchLifecycleConfiguration" {
return "", nil
}
return "", err
}
return bucketLifecycle, nil
}
// Request server for current bucket lifecycle.
func (c Client) getBucketLifecycle(bucketName string) (string, error) {
// Get resources properly escaped and lined up before
// using them in http request.
urlValues := make(url.Values)
urlValues.Set("lifecycle", "")
// Execute GET on bucket to get lifecycle.
resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
})
defer closeResponse(resp)
if err != nil {
return "", err
}
if resp != nil {
if resp.StatusCode != http.StatusOK {
return "", httpRespToErrorResponse(resp, bucketName, "")
}
}
bucketLifecycleBuf, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", err
}
lifecycle := string(bucketLifecycleBuf)
return lifecycle, err
}

136
vendor/github.com/minio/minio-go/api-get-object-acl.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,136 @@
/*
* Minio Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2018 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package minio
import (
"context"
"net/http"
"net/url"
)
type accessControlPolicy struct {
Owner struct {
ID string `xml:"ID"`
DisplayName string `xml:"DisplayName"`
} `xml:"Owner"`
AccessControlList struct {
Grant []struct {
Grantee struct {
ID string `xml:"ID"`
DisplayName string `xml:"DisplayName"`
URI string `xml:"URI"`
} `xml:"Grantee"`
Permission string `xml:"Permission"`
} `xml:"Grant"`
} `xml:"AccessControlList"`
}
//GetObjectACL get object ACLs
func (c Client) GetObjectACL(bucketName, objectName string) (*ObjectInfo, error) {
resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{
bucketName: bucketName,
objectName: objectName,
queryValues: url.Values{
"acl": []string{""},
},
})
if err != nil {
return nil, err
}
defer closeResponse(resp)
if resp.StatusCode != http.StatusOK {
return nil, httpRespToErrorResponse(resp, bucketName, objectName)
}
res := &accessControlPolicy{}
if err := xmlDecoder(resp.Body, res); err != nil {
return nil, err
}
objInfo, err := c.statObject(context.Background(), bucketName, objectName, StatObjectOptions{})
if err != nil {
return nil, err
}
cannedACL := getCannedACL(res)
if cannedACL != "" {
objInfo.Metadata.Add("X-Amz-Acl", cannedACL)
return &objInfo, nil
}
grantACL := getAmzGrantACL(res)
for k, v := range grantACL {
objInfo.Metadata[k] = v
}
return &objInfo, nil
}
func getCannedACL(aCPolicy *accessControlPolicy) string {
grants := aCPolicy.AccessControlList.Grant
switch {
case len(grants) == 1:
if grants[0].Grantee.URI == "" && grants[0].Permission == "FULL_CONTROL" {
return "private"
}
case len(grants) == 2:
for _, g := range grants {
if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" && g.Permission == "READ" {
return "authenticated-read"
}
if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "READ" {
return "public-read"
}
if g.Permission == "READ" && g.Grantee.ID == aCPolicy.Owner.ID {
return "bucket-owner-read"
}
}
case len(grants) == 3:
for _, g := range grants {
if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "WRITE" {
return "public-read-write"
}
}
}
return ""
}
func getAmzGrantACL(aCPolicy *accessControlPolicy) map[string][]string {
grants := aCPolicy.AccessControlList.Grant
res := map[string][]string{}
for _, g := range grants {
switch {
case g.Permission == "READ":
res["X-Amz-Grant-Read"] = append(res["X-Amz-Grant-Read"], "id="+g.Grantee.ID)
case g.Permission == "WRITE":
res["X-Amz-Grant-Write"] = append(res["X-Amz-Grant-Write"], "id="+g.Grantee.ID)
case g.Permission == "READ_ACP":
res["X-Amz-Grant-Read-Acp"] = append(res["X-Amz-Grant-Read-Acp"], "id="+g.Grantee.ID)
case g.Permission == "WRITE_ACP":
res["X-Amz-Grant-Write-Acp"] = append(res["X-Amz-Grant-Write-Acp"], "id="+g.Grantee.ID)
case g.Permission == "FULL_CONTROL":
res["X-Amz-Grant-Full-Control"] = append(res["X-Amz-Grant-Full-Control"], "id="+g.Grantee.ID)
}
}
return res
}

2
vendor/github.com/minio/minio-go/api-get-options.go сгенерированный поставляемый
Просмотреть файл

@@ -44,7 +44,7 @@ func (o GetObjectOptions) Header() http.Header {
for k, v := range o.headers {
headers.Set(k, v)
}
if o.ServerSideEncryption != nil && o.ServerSideEncryption.Type() != encrypt.S3 {
if o.ServerSideEncryption != nil && o.ServerSideEncryption.Type() == encrypt.SSEC {
o.ServerSideEncryption.Marshal(headers)
}
return headers

81
vendor/github.com/minio/minio-go/api-put-bucket.go сгенерированный поставляемый
Просмотреть файл

@@ -178,6 +178,87 @@ func (c Client) removeBucketPolicy(bucketName string) error {
return nil
}
// SetBucketLifecycle set the lifecycle on an existing bucket.
func (c Client) SetBucketLifecycle(bucketName, lifecycle string) error {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return err
}
// If lifecycle is empty then delete it.
if lifecycle == "" {
return c.removeBucketLifecycle(bucketName)
}
// Save the updated lifecycle.
return c.putBucketLifecycle(bucketName, lifecycle)
}
// Saves a new bucket lifecycle.
func (c Client) putBucketLifecycle(bucketName, lifecycle string) error {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return err
}
// Get resources properly escaped and lined up before
// using them in http request.
urlValues := make(url.Values)
urlValues.Set("lifecycle", "")
// Content-length is mandatory for put lifecycle request
lifecycleReader := strings.NewReader(lifecycle)
b, err := ioutil.ReadAll(lifecycleReader)
if err != nil {
return err
}
reqMetadata := requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
contentBody: lifecycleReader,
contentLength: int64(len(b)),
contentMD5Base64: sumMD5Base64(b),
}
// Execute PUT to upload a new bucket lifecycle.
resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata)
defer closeResponse(resp)
if err != nil {
return err
}
if resp != nil {
if resp.StatusCode != http.StatusOK {
return httpRespToErrorResponse(resp, bucketName, "")
}
}
return nil
}
// Remove lifecycle from a bucket.
func (c Client) removeBucketLifecycle(bucketName string) error {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return err
}
// Get resources properly escaped and lined up before
// using them in http request.
urlValues := make(url.Values)
urlValues.Set("lifecycle", "")
// Execute DELETE on objectName.
resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Hex: emptySHA256Hex,
})
defer closeResponse(resp)
if err != nil {
return err
}
return nil
}
// SetBucketNotification saves a new bucket notification.
func (c Client) SetBucketNotification(bucketName string, bucketNotification BucketNotification) error {
// Input validation.

6
vendor/github.com/minio/minio-go/api-put-object-multipart.go сгенерированный поставляемый
Просмотреть файл

@@ -259,7 +259,11 @@ func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID
// Set encryption headers, if any.
customHeader := make(http.Header)
if sse != nil && sse.Type() != encrypt.S3 && sse.Type() != encrypt.KMS {
// https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html
// Server-side encryption is supported by the S3 Multipart Upload actions.
// Unless you are using a customer-provided encryption key, you don't need
// to specify the encryption parameters in each UploadPart request.
if sse != nil && sse.Type() == encrypt.SSEC {
sse.Marshal(customHeader)
}

520
vendor/github.com/minio/minio-go/api-select.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,520 @@
/*
* Minio Go Library for Amazon S3 Compatible Cloud Storage
* (C) 2018 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package minio
import (
"bytes"
"context"
"encoding/binary"
"encoding/xml"
"errors"
"fmt"
"hash"
"hash/crc32"
"io"
"net/http"
"net/url"
"strings"
"github.com/minio/minio-go/pkg/encrypt"
"github.com/minio/minio-go/pkg/s3utils"
)
// CSVFileHeaderInfo - is the parameter for whether to utilize headers.
type CSVFileHeaderInfo string
// Constants for file header info.
const (
CSVFileHeaderInfoNone CSVFileHeaderInfo = "NONE"
CSVFileHeaderInfoIgnore = "IGNORE"
CSVFileHeaderInfoUse = "USE"
)
// SelectCompressionType - is the parameter for what type of compression is
// present
type SelectCompressionType string
// Constants for compression types under select API.
const (
SelectCompressionNONE SelectCompressionType = "NONE"
SelectCompressionGZIP = "GZIP"
SelectCompressionBZIP = "BZIP2"
)
// CSVQuoteFields - is the parameter for how CSV fields are quoted.
type CSVQuoteFields string
// Constants for csv quote styles.
const (
CSVQuoteFieldsAlways CSVQuoteFields = "Always"
CSVQuoteFieldsAsNeeded = "AsNeeded"
)
// QueryExpressionType - is of what syntax the expression is, this should only
// be SQL
type QueryExpressionType string
// Constants for expression type.
const (
QueryExpressionTypeSQL QueryExpressionType = "SQL"
)
// JSONType determines json input serialization type.
type JSONType string
// Constants for JSONTypes.
const (
JSONDocumentType JSONType = "DOCUMENT"
JSONLinesType = "LINES"
)
// ParquetInputOptions parquet input specific options
type ParquetInputOptions struct{}
// CSVInputOptions csv input specific options
type CSVInputOptions struct {
FileHeaderInfo CSVFileHeaderInfo
RecordDelimiter string
FieldDelimiter string
QuoteCharacter string
QuoteEscapeCharacter string
Comments string
}
// CSVOutputOptions csv output specific options
type CSVOutputOptions struct {
QuoteFields CSVQuoteFields
RecordDelimiter string
FieldDelimiter string
QuoteCharacter string
QuoteEscapeCharacter string
}
// JSONInputOptions json input specific options
type JSONInputOptions struct {
Type JSONType
}
// JSONOutputOptions - json output specific options
type JSONOutputOptions struct {
RecordDelimiter string
}
// SelectObjectInputSerialization - input serialization parameters
type SelectObjectInputSerialization struct {
CompressionType SelectCompressionType
Parquet *ParquetInputOptions `xml:"Parquet,omitempty"`
CSV *CSVInputOptions `xml:"CSV,omitempty"`
JSON *JSONInputOptions `xml:"JSON,omitempty"`
}
// SelectObjectOutputSerialization - output serialization parameters.
type SelectObjectOutputSerialization struct {
CSV *CSVOutputOptions `xml:"CSV,omitempty"`
JSON *JSONOutputOptions `xml:"JSON,omitempty"`
}
// SelectObjectOptions - represents the input select body
type SelectObjectOptions struct {
XMLName xml.Name `xml:"SelectObjectContentRequest" json:"-"`
ServerSideEncryption encrypt.ServerSide `xml:"-"`
Expression string
ExpressionType QueryExpressionType
InputSerialization SelectObjectInputSerialization
OutputSerialization SelectObjectOutputSerialization
RequestProgress struct {
Enabled bool
}
}
// Header returns the http.Header representation of the SelectObject options.
func (o SelectObjectOptions) Header() http.Header {
headers := make(http.Header)
if o.ServerSideEncryption != nil && o.ServerSideEncryption.Type() == encrypt.SSEC {
o.ServerSideEncryption.Marshal(headers)
}
return headers
}
// SelectObjectType - is the parameter which defines what type of object the
// operation is being performed on.
type SelectObjectType string
// Constants for input data types.
const (
SelectObjectTypeCSV SelectObjectType = "CSV"
SelectObjectTypeJSON = "JSON"
SelectObjectTypeParquet = "Parquet"
)
// preludeInfo is used for keeping track of necessary information from the
// prelude.
type preludeInfo struct {
totalLen uint32
headerLen uint32
}
// SelectResults is used for the streaming responses from the server.
type SelectResults struct {
pipeReader *io.PipeReader
resp *http.Response
stats *StatsMessage
progress *ProgressMessage
}
// ProgressMessage is a struct for progress xml message.
type ProgressMessage struct {
XMLName xml.Name `xml:"Progress" json:"-"`
StatsMessage
}
// StatsMessage is a struct for stat xml message.
type StatsMessage struct {
XMLName xml.Name `xml:"Stats" json:"-"`
BytesScanned int64
BytesProcessed int64
BytesReturned int64
}
// eventType represents the type of event.
type eventType string
// list of event-types returned by Select API.
const (
endEvent eventType = "End"
errorEvent = "Error"
recordsEvent = "Records"
progressEvent = "Progress"
statsEvent = "Stats"
)
// contentType represents content type of event.
type contentType string
const (
xmlContent contentType = "text/xml"
)
// SelectObjectContent is a implementation of http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html AWS S3 API.
func (c Client) SelectObjectContent(ctx context.Context, bucketName, objectName string, opts SelectObjectOptions) (*SelectResults, error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return nil, err
}
if err := s3utils.CheckValidObjectName(objectName); err != nil {
return nil, err
}
selectReqBytes, err := xml.Marshal(opts)
if err != nil {
return nil, err
}
urlValues := make(url.Values)
urlValues.Set("select", "")
urlValues.Set("select-type", "2")
// Execute POST on bucket/object.
resp, err := c.executeMethod(ctx, "POST", requestMetadata{
bucketName: bucketName,
objectName: objectName,
queryValues: urlValues,
customHeader: opts.Header(),
contentMD5Base64: sumMD5Base64(selectReqBytes),
contentSHA256Hex: sum256Hex(selectReqBytes),
contentBody: bytes.NewReader(selectReqBytes),
contentLength: int64(len(selectReqBytes)),
})
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, httpRespToErrorResponse(resp, bucketName, "")
}
pipeReader, pipeWriter := io.Pipe()
streamer := &SelectResults{
resp: resp,
stats: &StatsMessage{},
progress: &ProgressMessage{},
pipeReader: pipeReader,
}
streamer.start(pipeWriter)
return streamer, nil
}
// Close - closes the underlying response body and the stream reader.
func (s *SelectResults) Close() error {
defer closeResponse(s.resp)
return s.pipeReader.Close()
}
// Read - is a reader compatible implementation for SelectObjectContent records.
func (s *SelectResults) Read(b []byte) (n int, err error) {
return s.pipeReader.Read(b)
}
// Stats - information about a request's stats when processing is complete.
func (s *SelectResults) Stats() *StatsMessage {
return s.stats
}
// Progress - information about the progress of a request.
func (s *SelectResults) Progress() *ProgressMessage {
return s.progress
}
// start is the main function that decodes the large byte array into
// several events that are sent through the eventstream.
func (s *SelectResults) start(pipeWriter *io.PipeWriter) {
go func() {
for {
var prelude preludeInfo
var headers = make(http.Header)
var err error
// Create CRC code
crc := crc32.New(crc32.IEEETable)
crcReader := io.TeeReader(s.resp.Body, crc)
// Extract the prelude(12 bytes) into a struct to extract relevant information.
prelude, err = processPrelude(crcReader, crc)
if err != nil {
pipeWriter.CloseWithError(err)
closeResponse(s.resp)
return
}
// Extract the headers(variable bytes) into a struct to extract relevant information
if prelude.headerLen > 0 {
if err = extractHeader(io.LimitReader(crcReader, int64(prelude.headerLen)), headers); err != nil {
pipeWriter.CloseWithError(err)
closeResponse(s.resp)
return
}
}
// Get the actual payload length so that the appropriate amount of
// bytes can be read or parsed.
payloadLen := prelude.PayloadLen()
// Get content-type of the payload.
c := contentType(headers.Get("content-type"))
// Get event type of the payload.
e := eventType(headers.Get("event-type"))
// Handle all supported events.
switch e {
case endEvent:
pipeWriter.Close()
closeResponse(s.resp)
return
case errorEvent:
pipeWriter.CloseWithError(errors.New("Error Type of " + headers.Get("error-type") + " " + headers.Get("error-message")))
closeResponse(s.resp)
return
case recordsEvent:
if _, err = io.Copy(pipeWriter, io.LimitReader(crcReader, payloadLen)); err != nil {
pipeWriter.CloseWithError(err)
closeResponse(s.resp)
return
}
case progressEvent:
switch c {
case xmlContent:
if err = xmlDecoder(io.LimitReader(crcReader, payloadLen), s.progress); err != nil {
pipeWriter.CloseWithError(err)
closeResponse(s.resp)
return
}
default:
pipeWriter.CloseWithError(fmt.Errorf("Unexpected content-type %s sent for event-type %s", c, progressEvent))
closeResponse(s.resp)
return
}
case statsEvent:
switch c {
case xmlContent:
if err = xmlDecoder(io.LimitReader(crcReader, payloadLen), s.stats); err != nil {
pipeWriter.CloseWithError(err)
closeResponse(s.resp)
return
}
default:
pipeWriter.CloseWithError(fmt.Errorf("Unexpected content-type %s sent for event-type %s", c, statsEvent))
closeResponse(s.resp)
return
}
}
// Ensures that the full message's CRC is correct and
// that the message is not corrupted
if err := checkCRC(s.resp.Body, crc.Sum32()); err != nil {
pipeWriter.CloseWithError(err)
closeResponse(s.resp)
return
}
}
}()
}
// PayloadLen is a function that calculates the length of the payload.
func (p preludeInfo) PayloadLen() int64 {
return int64(p.totalLen - p.headerLen - 16)
}
// processPrelude is the function that reads the 12 bytes of the prelude and
// ensures the CRC is correct while also extracting relevant information into
// the struct,
func processPrelude(prelude io.Reader, crc hash.Hash32) (preludeInfo, error) {
var err error
var pInfo = preludeInfo{}
// reads total length of the message (first 4 bytes)
pInfo.totalLen, err = extractUint32(prelude)
if err != nil {
return pInfo, err
}
// reads total header length of the message (2nd 4 bytes)
pInfo.headerLen, err = extractUint32(prelude)
if err != nil {
return pInfo, err
}
// checks that the CRC is correct (3rd 4 bytes)
preCRC := crc.Sum32()
if err := checkCRC(prelude, preCRC); err != nil {
return pInfo, err
}
return pInfo, nil
}
// extracts the relevant information from the Headers.
func extractHeader(body io.Reader, myHeaders http.Header) error {
for {
// extracts the first part of the header,
headerTypeName, err := extractHeaderType(body)
if err != nil {
// Since end of file, we have read all of our headers
if err == io.EOF {
break
}
return err
}
// reads the 7 present in the header and ignores it.
extractUint8(body)
headerValueName, err := extractHeaderValue(body)
if err != nil {
return err
}
myHeaders.Set(headerTypeName, headerValueName)
}
return nil
}
// extractHeaderType extracts the first half of the header message, the header type.
func extractHeaderType(body io.Reader) (string, error) {
// extracts 2 bit integer
headerNameLen, err := extractUint8(body)
if err != nil {
return "", err
}
// extracts the string with the appropriate number of bytes
headerName, err := extractString(body, int(headerNameLen))
if err != nil {
return "", err
}
return strings.TrimPrefix(headerName, ":"), nil
}
// extractsHeaderValue extracts the second half of the header message, the
// header value
func extractHeaderValue(body io.Reader) (string, error) {
bodyLen, err := extractUint16(body)
if err != nil {
return "", err
}
bodyName, err := extractString(body, int(bodyLen))
if err != nil {
return "", err
}
return bodyName, nil
}
// extracts a string from byte array of a particular number of bytes.
func extractString(source io.Reader, lenBytes int) (string, error) {
myVal := make([]byte, lenBytes)
_, err := source.Read(myVal)
if err != nil {
return "", err
}
return string(myVal), nil
}
// extractUint32 extracts a 4 byte integer from the byte array.
func extractUint32(r io.Reader) (uint32, error) {
buf := make([]byte, 4)
_, err := io.ReadFull(r, buf)
if err != nil {
return 0, err
}
return binary.BigEndian.Uint32(buf), nil
}
// extractUint16 extracts a 2 byte integer from the byte array.
func extractUint16(r io.Reader) (uint16, error) {
buf := make([]byte, 2)
_, err := io.ReadFull(r, buf)
if err != nil {
return 0, err
}
return binary.BigEndian.Uint16(buf), nil
}
// extractUint8 extracts a 1 byte integer from the byte array.
func extractUint8(r io.Reader) (uint8, error) {
buf := make([]byte, 1)
_, err := io.ReadFull(r, buf)
if err != nil {
return 0, err
}
return buf[0], nil
}
// checkCRC ensures that the CRC matches with the one from the reader.
func checkCRC(r io.Reader, expect uint32) error {
msgCRC, err := extractUint32(r)
if err != nil {
return err
}
if msgCRC != expect {
return fmt.Errorf("Checksum Mismatch, MessageCRC of 0x%X does not equal expected CRC of 0x%X", msgCRC, expect)
}
return nil
}

25
vendor/github.com/minio/minio-go/api.go сгенерированный поставляемый
Просмотреть файл

@@ -99,7 +99,7 @@ type Options struct {
// Global constants.
const (
libraryName = "minio-go"
libraryVersion = "v6.0.5"
libraryVersion = "v6.0.7"
)
// User Agent should always following the below style.
@@ -455,22 +455,9 @@ func (c Client) dumpHTTP(req *http.Request, resp *http.Response) error {
return err
}
} else {
// WORKAROUND for https://github.com/golang/go/issues/13942.
// httputil.DumpResponse does not print response headers for
// all successful calls which have response ContentLength set
// to zero. Keep this workaround until the above bug is fixed.
if resp.ContentLength == 0 {
var buffer bytes.Buffer
if err = resp.Header.Write(&buffer); err != nil {
return err
}
respTrace = buffer.Bytes()
respTrace = append(respTrace, []byte("\r\n")...)
} else {
respTrace, err = httputil.DumpResponse(resp, false)
if err != nil {
return err
}
respTrace, err = httputil.DumpResponse(resp, false)
if err != nil {
return err
}
}
@@ -599,8 +586,8 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque
// Initiate the request.
res, err = c.do(req)
if err != nil {
// For supported network errors verify.
if isNetErrorRetryable(err) {
// For supported http requests errors verify.
if isHTTPReqErrorRetryable(err) {
continue // Retry.
}
// For other errors, return here no need to retry.

9
vendor/github.com/minio/minio-go/core.go сгенерированный поставляемый
Просмотреть файл

@@ -21,6 +21,8 @@ import (
"context"
"io"
"strings"
"github.com/minio/minio-go/pkg/encrypt"
)
// Core - Inherits Client and adds new methods to expose the low level S3 APIs.
@@ -68,7 +70,7 @@ func (c Core) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string
}
// PutObject - Upload object. Uploads using single PUT call.
func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, metadata map[string]string) (ObjectInfo, error) {
func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, metadata map[string]string, sse encrypt.ServerSide) (ObjectInfo, error) {
opts := PutObjectOptions{}
m := make(map[string]string)
for k, v := range metadata {
@@ -89,6 +91,7 @@ func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Ba
}
}
opts.UserMetadata = m
opts.ServerSideEncryption = sse
return c.putObjectDo(context.Background(), bucket, object, data, md5Base64, sha256Hex, size, opts)
}
@@ -104,8 +107,8 @@ func (c Core) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, de
}
// PutObjectPart - Upload an object part.
func (c Core) PutObjectPart(bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string) (ObjectPart, error) {
return c.uploadPart(context.Background(), bucket, object, uploadID, data, partID, md5Base64, sha256Hex, size, nil)
func (c Core) PutObjectPart(bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string, sse encrypt.ServerSide) (ObjectPart, error) {
return c.uploadPart(context.Background(), bucket, object, uploadID, data, partID, md5Base64, sha256Hex, size, sse)
}
// ListObjectParts - List uploaded parts of an incomplete upload.x

165
vendor/github.com/minio/minio-go/functional_tests.go сгенерированный поставляемый
Просмотреть файл

@@ -2271,7 +2271,8 @@ func testPresignedPostPolicy() {
defer reader.Close()
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "")
// Azure requires the key to not start with a number
metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "user")
metadataValue := randString(60, rand.NewSource(time.Now().UnixNano()), "")
buf, err := ioutil.ReadAll(reader)
@@ -7023,6 +7024,167 @@ func testFGetObjectWithContext() {
}
// Test get object ACLs with GetObjectACL
func testGetObjectACL() {
// initialize logging params
startTime := time.Now()
testName := getFuncName()
function := "GetObjectACL(bucketName, objectName)"
args := map[string]interface{}{
"bucketName": "",
"objectName": "",
}
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// skipping region functional tests for non s3 runs
if os.Getenv(serverEndpoint) != "s3.amazonaws.com" {
ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info()
return
}
// Instantiate new minio client object.
c, err := minio.NewV4(
os.Getenv(serverEndpoint),
os.Getenv(accessKey),
os.Getenv(secretKey),
mustParseBool(os.Getenv(enableHTTPS)),
)
if err != nil {
logError(testName, function, args, startTime, "", "Minio client v4 object creation failed", err)
return
}
// Enable tracing, write to stderr.
// c.TraceOn(os.Stderr)
// Set user agent.
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
// Generate a new random bucket name.
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
args["bucketName"] = bucketName
// Make a new bucket.
err = c.MakeBucket(bucketName, "us-east-1")
if err != nil {
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
return
}
bufSize := dataFileMap["datafile-1-MB"]
var reader = getDataReader("datafile-1-MB")
defer reader.Close()
// Save the data
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
args["objectName"] = objectName
// Add meta data to add a canned acl
metaData := map[string]string{
"X-Amz-Acl": "public-read-write",
}
_, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", UserMetadata: metaData})
if err != nil {
logError(testName, function, args, startTime, "", "PutObject failed", err)
return
}
// Read the data back
objectInfo, getObjectACLErr := c.GetObjectACL(bucketName, objectName)
if getObjectACLErr == nil {
logError(testName, function, args, startTime, "", "GetObjectACL fail", getObjectACLErr)
return
}
s, ok := objectInfo.Metadata["X-Amz-Acl"]
if !ok {
logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Acl\"", nil)
return
}
if len(s) != 1 {
logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" canned acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil)
return
}
if s[0] != "public-read-write" {
logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" expected \"public-read-write\" but got"+fmt.Sprintf("%q", s[0]), nil)
return
}
bufSize = dataFileMap["datafile-1-MB"]
var reader2 = getDataReader("datafile-1-MB")
defer reader2.Close()
// Save the data
objectName = randString(60, rand.NewSource(time.Now().UnixNano()), "")
args["objectName"] = objectName
// Add meta data to add a canned acl
metaData = map[string]string{
"X-Amz-Grant-Read": "id=fooread@minio.go",
"X-Amz-Grant-Write": "id=foowrite@minio.go",
}
_, err = c.PutObject(bucketName, objectName, reader2, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", UserMetadata: metaData})
if err != nil {
logError(testName, function, args, startTime, "", "PutObject failed", err)
return
}
// Read the data back
objectInfo, getObjectACLErr = c.GetObjectACL(bucketName, objectName)
if getObjectACLErr == nil {
logError(testName, function, args, startTime, "", "GetObjectACL fail", getObjectACLErr)
return
}
if len(objectInfo.Metadata) != 3 {
logError(testName, function, args, startTime, "", "GetObjectACL fail expected \"3\" ACLs but got "+fmt.Sprintf(`"%d"`, len(objectInfo.Metadata)), nil)
return
}
s, ok = objectInfo.Metadata["X-Amz-Grant-Read"]
if !ok {
logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Read\"", nil)
return
}
if len(s) != 1 {
logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil)
return
}
if s[0] != "fooread@minio.go" {
logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"fooread@minio.go\" got "+fmt.Sprintf("%q", s), nil)
return
}
s, ok = objectInfo.Metadata["X-Amz-Grant-Write"]
if !ok {
logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Write\"", nil)
return
}
if len(s) != 1 {
logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil)
return
}
if s[0] != "foowrite@minio.go" {
logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"foowrite@minio.go\" got "+fmt.Sprintf("%q", s), nil)
return
}
// Delete all objects and buckets
if err = cleanupBucket(bucketName, c); err != nil {
logError(testName, function, args, startTime, "", "Cleanup failed", err)
return
}
successLogger(testName, function, args, startTime).Info()
}
// Test validates putObject with context to see if request cancellation is honored for V2.
func testPutObjectWithContextV2() {
// initialize logging params
@@ -7481,6 +7643,7 @@ func main() {
testGetObjectWithContext()
testFPutObjectWithContext()
testFGetObjectWithContext()
testGetObjectACL()
testPutObjectWithContext()
testStorageClassMetadataPutObject()
testStorageClassInvalidMetadataPutObject()

18
vendor/github.com/minio/minio-go/pkg/credentials/file_minio_client.go сгенерированный поставляемый
Просмотреть файл

@@ -62,13 +62,17 @@ func NewFileMinioClient(filename string, alias string) *Credentials {
// users home directory.
func (p *FileMinioClient) Retrieve() (Value, error) {
if p.filename == "" {
homeDir, err := homedir.Dir()
if err != nil {
return Value{}, err
}
p.filename = filepath.Join(homeDir, ".mc", "config.json")
if runtime.GOOS == "windows" {
p.filename = filepath.Join(homeDir, "mc", "config.json")
if value, ok := os.LookupEnv("MINIO_SHARED_CREDENTIALS_FILE"); ok {
p.filename = value
} else {
homeDir, err := homedir.Dir()
if err != nil {
return Value{}, err
}
p.filename = filepath.Join(homeDir, ".mc", "config.json")
if runtime.GOOS == "windows" {
p.filename = filepath.Join(homeDir, "mc", "config.json")
}
}
}

46
vendor/github.com/minio/minio-go/retry.go сгенерированный поставляемый
Просмотреть файл

@@ -85,36 +85,32 @@ func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duratio
return attemptCh
}
// isNetErrorRetryable - is network error retryable.
func isNetErrorRetryable(err error) bool {
// isHTTPReqErrorRetryable - is http requests error retryable, such
// as i/o timeout, connection broken etc..
func isHTTPReqErrorRetryable(err error) bool {
if err == nil {
return false
}
switch err.(type) {
case net.Error:
switch err.(type) {
switch e := err.(type) {
case *url.Error:
switch e.Err.(type) {
case *net.DNSError, *net.OpError, net.UnknownNetworkError:
return true
case *url.Error:
// For a URL error, where it replies back "connection closed"
// retry again.
if strings.Contains(err.Error(), "Connection closed by foreign host") {
return true
}
default:
if strings.Contains(err.Error(), "net/http: TLS handshake timeout") {
// If error is - tlsHandshakeTimeoutError, retry.
return true
} else if strings.Contains(err.Error(), "i/o timeout") {
// If error is - tcp timeoutError, retry.
return true
} else if strings.Contains(err.Error(), "connection timed out") {
// If err is a net.Dial timeout, retry.
return true
} else if strings.Contains(err.Error(), "net/http: HTTP/1.x transport connection broken") {
// If error is transport connection broken, retry.
return true
}
}
if strings.Contains(err.Error(), "Connection closed by foreign host") {
return true
} else if strings.Contains(err.Error(), "net/http: TLS handshake timeout") {
// If error is - tlsHandshakeTimeoutError, retry.
return true
} else if strings.Contains(err.Error(), "i/o timeout") {
// If error is - tcp timeoutError, retry.
return true
} else if strings.Contains(err.Error(), "connection timed out") {
// If err is a net.Dial timeout, retry.
return true
} else if strings.Contains(err.Error(), "net/http: HTTP/1.x transport connection broken") {
// If error is transport connection broken, retry.
return true
}
}
return false

3
vendor/github.com/minio/minio-go/utils.go сгенерированный поставляемый
Просмотреть файл

@@ -223,6 +223,7 @@ var supportedHeaders = []string{
"content-disposition",
"content-language",
"x-amz-website-redirect-location",
"expires",
// Add more supported headers here.
}
@@ -267,5 +268,5 @@ func isSSEHeader(headerKey string) bool {
func isAmzHeader(headerKey string) bool {
key := strings.ToLower(headerKey)
return strings.HasPrefix(key, "x-amz-meta-") || key == "x-amz-acl"
return strings.HasPrefix(key, "x-amz-meta-") || strings.HasPrefix(key, "x-amz-grant-") || key == "x-amz-acl" || isSSEHeader(headerKey)
}