Updating server dependancies. (#9498)
Этот коммит содержится в:
коммит произвёл
GitHub
родитель
006623e0f7
Коммит
a8c01377bc
18
vendor/github.com/minio/minio-go/README.md
сгенерированный
поставляемый
18
vendor/github.com/minio/minio-go/README.md
сгенерированный
поставляемый
@@ -14,9 +14,9 @@ go get -u github.com/minio/minio-go
|
||||
## Initialize Minio Client
|
||||
Minio client requires the following four parameters specified to connect to an Amazon S3 compatible object storage.
|
||||
|
||||
| Parameter | Description|
|
||||
| Parameter | Description|
|
||||
| :--- | :--- |
|
||||
| endpoint | URL to object storage service. |
|
||||
| endpoint | URL to object storage service. |
|
||||
| accessKeyID | Access key is the user ID that uniquely identifies your account. |
|
||||
| secretAccessKey | Secret key is the password to your account. |
|
||||
| secure | Set this value to 'true' to enable secure (HTTPS) access. |
|
||||
@@ -106,7 +106,7 @@ func main() {
|
||||
### Run FileUploader
|
||||
```sh
|
||||
go run file-uploader.go
|
||||
2016/08/13 17:03:28 Successfully created mymusic
|
||||
2016/08/13 17:03:28 Successfully created mymusic
|
||||
2016/08/13 17:03:40 Successfully uploaded golden-oldies.zip of size 16253413
|
||||
|
||||
mc ls play/mymusic/
|
||||
@@ -114,7 +114,7 @@ mc ls play/mymusic/
|
||||
```
|
||||
|
||||
## API Reference
|
||||
The full API Reference is available here.
|
||||
The full API Reference is available here.
|
||||
|
||||
* [Complete API Reference](https://docs.minio.io/docs/golang-client-api-reference)
|
||||
|
||||
@@ -154,6 +154,8 @@ The full API Reference is available here.
|
||||
* [`RemoveObject`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObject)
|
||||
* [`RemoveObjects`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObjects)
|
||||
* [`RemoveIncompleteUpload`](https://docs.minio.io/docs/golang-client-api-reference#RemoveIncompleteUpload)
|
||||
* [`SelectObjectContent`](https://docs.minio.io/docs/golang-client-api-reference#SelectObjectContent)
|
||||
|
||||
|
||||
### API Reference : Presigned Operations
|
||||
* [`PresignedGetObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedGetObject)
|
||||
@@ -182,7 +184,11 @@ The full API Reference is available here.
|
||||
* [setbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketpolicy.go)
|
||||
* [getbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketpolicy.go)
|
||||
* [listbucketpolicies.go](https://github.com/minio/minio-go/blob/master/examples/s3/listbucketpolicies.go)
|
||||
|
||||
|
||||
### Full Examples : Bucket lifecycle Operations
|
||||
* [setbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketlifecycle.go)
|
||||
* [getbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketlifecycle.go)
|
||||
|
||||
### Full Examples : Bucket notification Operations
|
||||
* [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go)
|
||||
* [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go)
|
||||
@@ -219,7 +225,7 @@ The full API Reference is available here.
|
||||
|
||||
## Explore Further
|
||||
* [Complete Documentation](https://docs.minio.io)
|
||||
* [Minio Go Client SDK API Reference](https://docs.minio.io/docs/golang-client-api-reference)
|
||||
* [Minio Go Client SDK API Reference](https://docs.minio.io/docs/golang-client-api-reference)
|
||||
* [Go Music Player App Full Application Example](https://docs.minio.io/docs/go-music-player-app)
|
||||
|
||||
## Contribute
|
||||
|
||||
33
vendor/github.com/minio/minio-go/api-compose-object.go
сгенерированный
поставляемый
33
vendor/github.com/minio/minio-go/api-compose-object.go
сгенерированный
поставляемый
@@ -101,7 +101,11 @@ func (d *DestinationInfo) getUserMetaHeadersMap(withCopyDirectiveHeader bool) ma
|
||||
r["x-amz-metadata-directive"] = "REPLACE"
|
||||
}
|
||||
for k, v := range d.userMetadata {
|
||||
r["x-amz-meta-"+k] = v
|
||||
if isAmzHeader(k) || isStandardHeader(k) || isStorageClassHeader(k) {
|
||||
r[k] = v
|
||||
} else {
|
||||
r["x-amz-meta-"+k] = v
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
@@ -373,15 +377,6 @@ func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo
|
||||
fmt.Sprintf("Client side encryption is used in source object %s/%s", src.bucket, src.object))
|
||||
}
|
||||
|
||||
// Since we did a HEAD to get size, we use the ETag
|
||||
// value to make sure the object has not changed by
|
||||
// the time we perform the copy. This is done, only if
|
||||
// the user has not set their own ETag match
|
||||
// condition.
|
||||
if src.Headers.Get("x-amz-copy-source-if-match") == "" {
|
||||
src.SetMatchETagCond(etag)
|
||||
}
|
||||
|
||||
// Check if a segment is specified, and if so, is the
|
||||
// segment within object bounds?
|
||||
if src.start != -1 {
|
||||
@@ -429,7 +424,15 @@ func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo
|
||||
|
||||
// Now, handle multipart-copy cases.
|
||||
|
||||
// 1. Initiate a new multipart upload.
|
||||
// 1. Ensure that the object has not been changed while
|
||||
// we are copying data.
|
||||
for _, src := range srcs {
|
||||
if src.Headers.Get("x-amz-copy-source-if-match") == "" {
|
||||
src.SetMatchETagCond(etag)
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Initiate a new multipart upload.
|
||||
|
||||
// Set user-metadata on the destination object. If no
|
||||
// user-metadata is specified, and there is only one source,
|
||||
@@ -449,13 +452,13 @@ func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo
|
||||
return err
|
||||
}
|
||||
|
||||
// 2. Perform copy part uploads
|
||||
// 3. Perform copy part uploads
|
||||
objParts := []CompletePart{}
|
||||
partIndex := 1
|
||||
for i, src := range srcs {
|
||||
h := src.Headers
|
||||
if src.encryption != nil {
|
||||
src.encryption.Marshal(h)
|
||||
encrypt.SSECopy(src.encryption).Marshal(h)
|
||||
}
|
||||
// Add destination encryption headers
|
||||
if dst.encryption != nil {
|
||||
@@ -480,14 +483,14 @@ func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo
|
||||
return err
|
||||
}
|
||||
if progress != nil {
|
||||
io.CopyN(ioutil.Discard, progress, start+end-1)
|
||||
io.CopyN(ioutil.Discard, progress, end-start+1)
|
||||
}
|
||||
objParts = append(objParts, complPart)
|
||||
partIndex++
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Make final complete-multipart request.
|
||||
// 4. Make final complete-multipart request.
|
||||
_, err = c.completeMultipartUpload(ctx, dst.bucket, dst.object, uploadID,
|
||||
completeMultipartUpload{Parts: objParts})
|
||||
if err != nil {
|
||||
|
||||
77
vendor/github.com/minio/minio-go/api-get-lifecycle.go
сгенерированный
поставляемый
Обычный файл
77
vendor/github.com/minio/minio-go/api-get-lifecycle.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,77 @@
|
||||
/*
|
||||
* Minio Go Library for Amazon S3 Compatible Cloud Storage
|
||||
* Copyright 2015-2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package minio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
"github.com/minio/minio-go/pkg/s3utils"
|
||||
)
|
||||
|
||||
// GetBucketLifecycle - get bucket lifecycle.
|
||||
func (c Client) GetBucketLifecycle(bucketName string) (string, error) {
|
||||
// Input validation.
|
||||
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
|
||||
return "", err
|
||||
}
|
||||
bucketLifecycle, err := c.getBucketLifecycle(bucketName)
|
||||
if err != nil {
|
||||
errResponse := ToErrorResponse(err)
|
||||
if errResponse.Code == "NoSuchLifecycleConfiguration" {
|
||||
return "", nil
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
return bucketLifecycle, nil
|
||||
}
|
||||
|
||||
// Request server for current bucket lifecycle.
|
||||
func (c Client) getBucketLifecycle(bucketName string) (string, error) {
|
||||
// Get resources properly escaped and lined up before
|
||||
// using them in http request.
|
||||
urlValues := make(url.Values)
|
||||
urlValues.Set("lifecycle", "")
|
||||
|
||||
// Execute GET on bucket to get lifecycle.
|
||||
resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{
|
||||
bucketName: bucketName,
|
||||
queryValues: urlValues,
|
||||
})
|
||||
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if resp != nil {
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", httpRespToErrorResponse(resp, bucketName, "")
|
||||
}
|
||||
}
|
||||
|
||||
bucketLifecycleBuf, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
lifecycle := string(bucketLifecycleBuf)
|
||||
return lifecycle, err
|
||||
}
|
||||
136
vendor/github.com/minio/minio-go/api-get-object-acl.go
сгенерированный
поставляемый
Обычный файл
136
vendor/github.com/minio/minio-go/api-get-object-acl.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,136 @@
|
||||
/*
|
||||
* Minio Go Library for Amazon S3 Compatible Cloud Storage
|
||||
* Copyright 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package minio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
type accessControlPolicy struct {
|
||||
Owner struct {
|
||||
ID string `xml:"ID"`
|
||||
DisplayName string `xml:"DisplayName"`
|
||||
} `xml:"Owner"`
|
||||
AccessControlList struct {
|
||||
Grant []struct {
|
||||
Grantee struct {
|
||||
ID string `xml:"ID"`
|
||||
DisplayName string `xml:"DisplayName"`
|
||||
URI string `xml:"URI"`
|
||||
} `xml:"Grantee"`
|
||||
Permission string `xml:"Permission"`
|
||||
} `xml:"Grant"`
|
||||
} `xml:"AccessControlList"`
|
||||
}
|
||||
|
||||
//GetObjectACL get object ACLs
|
||||
func (c Client) GetObjectACL(bucketName, objectName string) (*ObjectInfo, error) {
|
||||
|
||||
resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{
|
||||
bucketName: bucketName,
|
||||
objectName: objectName,
|
||||
queryValues: url.Values{
|
||||
"acl": []string{""},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer closeResponse(resp)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, httpRespToErrorResponse(resp, bucketName, objectName)
|
||||
}
|
||||
|
||||
res := &accessControlPolicy{}
|
||||
|
||||
if err := xmlDecoder(resp.Body, res); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
objInfo, err := c.statObject(context.Background(), bucketName, objectName, StatObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cannedACL := getCannedACL(res)
|
||||
if cannedACL != "" {
|
||||
objInfo.Metadata.Add("X-Amz-Acl", cannedACL)
|
||||
return &objInfo, nil
|
||||
}
|
||||
|
||||
grantACL := getAmzGrantACL(res)
|
||||
for k, v := range grantACL {
|
||||
objInfo.Metadata[k] = v
|
||||
}
|
||||
|
||||
return &objInfo, nil
|
||||
}
|
||||
|
||||
func getCannedACL(aCPolicy *accessControlPolicy) string {
|
||||
grants := aCPolicy.AccessControlList.Grant
|
||||
|
||||
switch {
|
||||
case len(grants) == 1:
|
||||
if grants[0].Grantee.URI == "" && grants[0].Permission == "FULL_CONTROL" {
|
||||
return "private"
|
||||
}
|
||||
case len(grants) == 2:
|
||||
for _, g := range grants {
|
||||
if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" && g.Permission == "READ" {
|
||||
return "authenticated-read"
|
||||
}
|
||||
if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "READ" {
|
||||
return "public-read"
|
||||
}
|
||||
if g.Permission == "READ" && g.Grantee.ID == aCPolicy.Owner.ID {
|
||||
return "bucket-owner-read"
|
||||
}
|
||||
}
|
||||
case len(grants) == 3:
|
||||
for _, g := range grants {
|
||||
if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "WRITE" {
|
||||
return "public-read-write"
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func getAmzGrantACL(aCPolicy *accessControlPolicy) map[string][]string {
|
||||
grants := aCPolicy.AccessControlList.Grant
|
||||
res := map[string][]string{}
|
||||
|
||||
for _, g := range grants {
|
||||
switch {
|
||||
case g.Permission == "READ":
|
||||
res["X-Amz-Grant-Read"] = append(res["X-Amz-Grant-Read"], "id="+g.Grantee.ID)
|
||||
case g.Permission == "WRITE":
|
||||
res["X-Amz-Grant-Write"] = append(res["X-Amz-Grant-Write"], "id="+g.Grantee.ID)
|
||||
case g.Permission == "READ_ACP":
|
||||
res["X-Amz-Grant-Read-Acp"] = append(res["X-Amz-Grant-Read-Acp"], "id="+g.Grantee.ID)
|
||||
case g.Permission == "WRITE_ACP":
|
||||
res["X-Amz-Grant-Write-Acp"] = append(res["X-Amz-Grant-Write-Acp"], "id="+g.Grantee.ID)
|
||||
case g.Permission == "FULL_CONTROL":
|
||||
res["X-Amz-Grant-Full-Control"] = append(res["X-Amz-Grant-Full-Control"], "id="+g.Grantee.ID)
|
||||
}
|
||||
}
|
||||
return res
|
||||
}
|
||||
2
vendor/github.com/minio/minio-go/api-get-options.go
сгенерированный
поставляемый
2
vendor/github.com/minio/minio-go/api-get-options.go
сгенерированный
поставляемый
@@ -44,7 +44,7 @@ func (o GetObjectOptions) Header() http.Header {
|
||||
for k, v := range o.headers {
|
||||
headers.Set(k, v)
|
||||
}
|
||||
if o.ServerSideEncryption != nil && o.ServerSideEncryption.Type() != encrypt.S3 {
|
||||
if o.ServerSideEncryption != nil && o.ServerSideEncryption.Type() == encrypt.SSEC {
|
||||
o.ServerSideEncryption.Marshal(headers)
|
||||
}
|
||||
return headers
|
||||
|
||||
81
vendor/github.com/minio/minio-go/api-put-bucket.go
сгенерированный
поставляемый
81
vendor/github.com/minio/minio-go/api-put-bucket.go
сгенерированный
поставляемый
@@ -178,6 +178,87 @@ func (c Client) removeBucketPolicy(bucketName string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetBucketLifecycle set the lifecycle on an existing bucket.
|
||||
func (c Client) SetBucketLifecycle(bucketName, lifecycle string) error {
|
||||
// Input validation.
|
||||
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// If lifecycle is empty then delete it.
|
||||
if lifecycle == "" {
|
||||
return c.removeBucketLifecycle(bucketName)
|
||||
}
|
||||
|
||||
// Save the updated lifecycle.
|
||||
return c.putBucketLifecycle(bucketName, lifecycle)
|
||||
}
|
||||
|
||||
// Saves a new bucket lifecycle.
|
||||
func (c Client) putBucketLifecycle(bucketName, lifecycle string) error {
|
||||
// Input validation.
|
||||
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Get resources properly escaped and lined up before
|
||||
// using them in http request.
|
||||
urlValues := make(url.Values)
|
||||
urlValues.Set("lifecycle", "")
|
||||
|
||||
// Content-length is mandatory for put lifecycle request
|
||||
lifecycleReader := strings.NewReader(lifecycle)
|
||||
b, err := ioutil.ReadAll(lifecycleReader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
reqMetadata := requestMetadata{
|
||||
bucketName: bucketName,
|
||||
queryValues: urlValues,
|
||||
contentBody: lifecycleReader,
|
||||
contentLength: int64(len(b)),
|
||||
contentMD5Base64: sumMD5Base64(b),
|
||||
}
|
||||
|
||||
// Execute PUT to upload a new bucket lifecycle.
|
||||
resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata)
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp != nil {
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return httpRespToErrorResponse(resp, bucketName, "")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove lifecycle from a bucket.
|
||||
func (c Client) removeBucketLifecycle(bucketName string) error {
|
||||
// Input validation.
|
||||
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
|
||||
return err
|
||||
}
|
||||
// Get resources properly escaped and lined up before
|
||||
// using them in http request.
|
||||
urlValues := make(url.Values)
|
||||
urlValues.Set("lifecycle", "")
|
||||
|
||||
// Execute DELETE on objectName.
|
||||
resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{
|
||||
bucketName: bucketName,
|
||||
queryValues: urlValues,
|
||||
contentSHA256Hex: emptySHA256Hex,
|
||||
})
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetBucketNotification saves a new bucket notification.
|
||||
func (c Client) SetBucketNotification(bucketName string, bucketNotification BucketNotification) error {
|
||||
// Input validation.
|
||||
|
||||
6
vendor/github.com/minio/minio-go/api-put-object-multipart.go
сгенерированный
поставляемый
6
vendor/github.com/minio/minio-go/api-put-object-multipart.go
сгенерированный
поставляемый
@@ -259,7 +259,11 @@ func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID
|
||||
|
||||
// Set encryption headers, if any.
|
||||
customHeader := make(http.Header)
|
||||
if sse != nil && sse.Type() != encrypt.S3 && sse.Type() != encrypt.KMS {
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html
|
||||
// Server-side encryption is supported by the S3 Multipart Upload actions.
|
||||
// Unless you are using a customer-provided encryption key, you don't need
|
||||
// to specify the encryption parameters in each UploadPart request.
|
||||
if sse != nil && sse.Type() == encrypt.SSEC {
|
||||
sse.Marshal(customHeader)
|
||||
}
|
||||
|
||||
|
||||
520
vendor/github.com/minio/minio-go/api-select.go
сгенерированный
поставляемый
Обычный файл
520
vendor/github.com/minio/minio-go/api-select.go
сгенерированный
поставляемый
Обычный файл
@@ -0,0 +1,520 @@
|
||||
/*
|
||||
* Minio Go Library for Amazon S3 Compatible Cloud Storage
|
||||
* (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package minio
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go/pkg/encrypt"
|
||||
"github.com/minio/minio-go/pkg/s3utils"
|
||||
)
|
||||
|
||||
// CSVFileHeaderInfo - is the parameter for whether to utilize headers.
|
||||
type CSVFileHeaderInfo string
|
||||
|
||||
// Constants for file header info.
|
||||
const (
|
||||
CSVFileHeaderInfoNone CSVFileHeaderInfo = "NONE"
|
||||
CSVFileHeaderInfoIgnore = "IGNORE"
|
||||
CSVFileHeaderInfoUse = "USE"
|
||||
)
|
||||
|
||||
// SelectCompressionType - is the parameter for what type of compression is
|
||||
// present
|
||||
type SelectCompressionType string
|
||||
|
||||
// Constants for compression types under select API.
|
||||
const (
|
||||
SelectCompressionNONE SelectCompressionType = "NONE"
|
||||
SelectCompressionGZIP = "GZIP"
|
||||
SelectCompressionBZIP = "BZIP2"
|
||||
)
|
||||
|
||||
// CSVQuoteFields - is the parameter for how CSV fields are quoted.
|
||||
type CSVQuoteFields string
|
||||
|
||||
// Constants for csv quote styles.
|
||||
const (
|
||||
CSVQuoteFieldsAlways CSVQuoteFields = "Always"
|
||||
CSVQuoteFieldsAsNeeded = "AsNeeded"
|
||||
)
|
||||
|
||||
// QueryExpressionType - is of what syntax the expression is, this should only
|
||||
// be SQL
|
||||
type QueryExpressionType string
|
||||
|
||||
// Constants for expression type.
|
||||
const (
|
||||
QueryExpressionTypeSQL QueryExpressionType = "SQL"
|
||||
)
|
||||
|
||||
// JSONType determines json input serialization type.
|
||||
type JSONType string
|
||||
|
||||
// Constants for JSONTypes.
|
||||
const (
|
||||
JSONDocumentType JSONType = "DOCUMENT"
|
||||
JSONLinesType = "LINES"
|
||||
)
|
||||
|
||||
// ParquetInputOptions parquet input specific options
|
||||
type ParquetInputOptions struct{}
|
||||
|
||||
// CSVInputOptions csv input specific options
|
||||
type CSVInputOptions struct {
|
||||
FileHeaderInfo CSVFileHeaderInfo
|
||||
RecordDelimiter string
|
||||
FieldDelimiter string
|
||||
QuoteCharacter string
|
||||
QuoteEscapeCharacter string
|
||||
Comments string
|
||||
}
|
||||
|
||||
// CSVOutputOptions csv output specific options
|
||||
type CSVOutputOptions struct {
|
||||
QuoteFields CSVQuoteFields
|
||||
RecordDelimiter string
|
||||
FieldDelimiter string
|
||||
QuoteCharacter string
|
||||
QuoteEscapeCharacter string
|
||||
}
|
||||
|
||||
// JSONInputOptions json input specific options
|
||||
type JSONInputOptions struct {
|
||||
Type JSONType
|
||||
}
|
||||
|
||||
// JSONOutputOptions - json output specific options
|
||||
type JSONOutputOptions struct {
|
||||
RecordDelimiter string
|
||||
}
|
||||
|
||||
// SelectObjectInputSerialization - input serialization parameters
|
||||
type SelectObjectInputSerialization struct {
|
||||
CompressionType SelectCompressionType
|
||||
Parquet *ParquetInputOptions `xml:"Parquet,omitempty"`
|
||||
CSV *CSVInputOptions `xml:"CSV,omitempty"`
|
||||
JSON *JSONInputOptions `xml:"JSON,omitempty"`
|
||||
}
|
||||
|
||||
// SelectObjectOutputSerialization - output serialization parameters.
|
||||
type SelectObjectOutputSerialization struct {
|
||||
CSV *CSVOutputOptions `xml:"CSV,omitempty"`
|
||||
JSON *JSONOutputOptions `xml:"JSON,omitempty"`
|
||||
}
|
||||
|
||||
// SelectObjectOptions - represents the input select body
|
||||
type SelectObjectOptions struct {
|
||||
XMLName xml.Name `xml:"SelectObjectContentRequest" json:"-"`
|
||||
ServerSideEncryption encrypt.ServerSide `xml:"-"`
|
||||
Expression string
|
||||
ExpressionType QueryExpressionType
|
||||
InputSerialization SelectObjectInputSerialization
|
||||
OutputSerialization SelectObjectOutputSerialization
|
||||
RequestProgress struct {
|
||||
Enabled bool
|
||||
}
|
||||
}
|
||||
|
||||
// Header returns the http.Header representation of the SelectObject options.
|
||||
func (o SelectObjectOptions) Header() http.Header {
|
||||
headers := make(http.Header)
|
||||
if o.ServerSideEncryption != nil && o.ServerSideEncryption.Type() == encrypt.SSEC {
|
||||
o.ServerSideEncryption.Marshal(headers)
|
||||
}
|
||||
return headers
|
||||
}
|
||||
|
||||
// SelectObjectType - is the parameter which defines what type of object the
|
||||
// operation is being performed on.
|
||||
type SelectObjectType string
|
||||
|
||||
// Constants for input data types.
|
||||
const (
|
||||
SelectObjectTypeCSV SelectObjectType = "CSV"
|
||||
SelectObjectTypeJSON = "JSON"
|
||||
SelectObjectTypeParquet = "Parquet"
|
||||
)
|
||||
|
||||
// preludeInfo is used for keeping track of necessary information from the
|
||||
// prelude.
|
||||
type preludeInfo struct {
|
||||
totalLen uint32
|
||||
headerLen uint32
|
||||
}
|
||||
|
||||
// SelectResults is used for the streaming responses from the server.
|
||||
type SelectResults struct {
|
||||
pipeReader *io.PipeReader
|
||||
resp *http.Response
|
||||
stats *StatsMessage
|
||||
progress *ProgressMessage
|
||||
}
|
||||
|
||||
// ProgressMessage is a struct for progress xml message.
|
||||
type ProgressMessage struct {
|
||||
XMLName xml.Name `xml:"Progress" json:"-"`
|
||||
StatsMessage
|
||||
}
|
||||
|
||||
// StatsMessage is a struct for stat xml message.
|
||||
type StatsMessage struct {
|
||||
XMLName xml.Name `xml:"Stats" json:"-"`
|
||||
BytesScanned int64
|
||||
BytesProcessed int64
|
||||
BytesReturned int64
|
||||
}
|
||||
|
||||
// eventType represents the type of event.
|
||||
type eventType string
|
||||
|
||||
// list of event-types returned by Select API.
|
||||
const (
|
||||
endEvent eventType = "End"
|
||||
errorEvent = "Error"
|
||||
recordsEvent = "Records"
|
||||
progressEvent = "Progress"
|
||||
statsEvent = "Stats"
|
||||
)
|
||||
|
||||
// contentType represents content type of event.
|
||||
type contentType string
|
||||
|
||||
const (
|
||||
xmlContent contentType = "text/xml"
|
||||
)
|
||||
|
||||
// SelectObjectContent is a implementation of http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html AWS S3 API.
|
||||
func (c Client) SelectObjectContent(ctx context.Context, bucketName, objectName string, opts SelectObjectOptions) (*SelectResults, error) {
|
||||
// Input validation.
|
||||
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s3utils.CheckValidObjectName(objectName); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
selectReqBytes, err := xml.Marshal(opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
urlValues := make(url.Values)
|
||||
urlValues.Set("select", "")
|
||||
urlValues.Set("select-type", "2")
|
||||
|
||||
// Execute POST on bucket/object.
|
||||
resp, err := c.executeMethod(ctx, "POST", requestMetadata{
|
||||
bucketName: bucketName,
|
||||
objectName: objectName,
|
||||
queryValues: urlValues,
|
||||
customHeader: opts.Header(),
|
||||
contentMD5Base64: sumMD5Base64(selectReqBytes),
|
||||
contentSHA256Hex: sum256Hex(selectReqBytes),
|
||||
contentBody: bytes.NewReader(selectReqBytes),
|
||||
contentLength: int64(len(selectReqBytes)),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, httpRespToErrorResponse(resp, bucketName, "")
|
||||
}
|
||||
|
||||
pipeReader, pipeWriter := io.Pipe()
|
||||
streamer := &SelectResults{
|
||||
resp: resp,
|
||||
stats: &StatsMessage{},
|
||||
progress: &ProgressMessage{},
|
||||
pipeReader: pipeReader,
|
||||
}
|
||||
streamer.start(pipeWriter)
|
||||
return streamer, nil
|
||||
}
|
||||
|
||||
// Close - closes the underlying response body and the stream reader.
|
||||
func (s *SelectResults) Close() error {
|
||||
defer closeResponse(s.resp)
|
||||
return s.pipeReader.Close()
|
||||
}
|
||||
|
||||
// Read - is a reader compatible implementation for SelectObjectContent records.
|
||||
func (s *SelectResults) Read(b []byte) (n int, err error) {
|
||||
return s.pipeReader.Read(b)
|
||||
}
|
||||
|
||||
// Stats - information about a request's stats when processing is complete.
|
||||
func (s *SelectResults) Stats() *StatsMessage {
|
||||
return s.stats
|
||||
}
|
||||
|
||||
// Progress - information about the progress of a request.
|
||||
func (s *SelectResults) Progress() *ProgressMessage {
|
||||
return s.progress
|
||||
}
|
||||
|
||||
// start is the main function that decodes the large byte array into
|
||||
// several events that are sent through the eventstream.
|
||||
func (s *SelectResults) start(pipeWriter *io.PipeWriter) {
|
||||
go func() {
|
||||
for {
|
||||
var prelude preludeInfo
|
||||
var headers = make(http.Header)
|
||||
var err error
|
||||
|
||||
// Create CRC code
|
||||
crc := crc32.New(crc32.IEEETable)
|
||||
crcReader := io.TeeReader(s.resp.Body, crc)
|
||||
|
||||
// Extract the prelude(12 bytes) into a struct to extract relevant information.
|
||||
prelude, err = processPrelude(crcReader, crc)
|
||||
if err != nil {
|
||||
pipeWriter.CloseWithError(err)
|
||||
closeResponse(s.resp)
|
||||
return
|
||||
}
|
||||
|
||||
// Extract the headers(variable bytes) into a struct to extract relevant information
|
||||
if prelude.headerLen > 0 {
|
||||
if err = extractHeader(io.LimitReader(crcReader, int64(prelude.headerLen)), headers); err != nil {
|
||||
pipeWriter.CloseWithError(err)
|
||||
closeResponse(s.resp)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Get the actual payload length so that the appropriate amount of
|
||||
// bytes can be read or parsed.
|
||||
payloadLen := prelude.PayloadLen()
|
||||
|
||||
// Get content-type of the payload.
|
||||
c := contentType(headers.Get("content-type"))
|
||||
|
||||
// Get event type of the payload.
|
||||
e := eventType(headers.Get("event-type"))
|
||||
|
||||
// Handle all supported events.
|
||||
switch e {
|
||||
case endEvent:
|
||||
pipeWriter.Close()
|
||||
closeResponse(s.resp)
|
||||
return
|
||||
case errorEvent:
|
||||
pipeWriter.CloseWithError(errors.New("Error Type of " + headers.Get("error-type") + " " + headers.Get("error-message")))
|
||||
closeResponse(s.resp)
|
||||
return
|
||||
case recordsEvent:
|
||||
if _, err = io.Copy(pipeWriter, io.LimitReader(crcReader, payloadLen)); err != nil {
|
||||
pipeWriter.CloseWithError(err)
|
||||
closeResponse(s.resp)
|
||||
return
|
||||
}
|
||||
case progressEvent:
|
||||
switch c {
|
||||
case xmlContent:
|
||||
if err = xmlDecoder(io.LimitReader(crcReader, payloadLen), s.progress); err != nil {
|
||||
pipeWriter.CloseWithError(err)
|
||||
closeResponse(s.resp)
|
||||
return
|
||||
}
|
||||
default:
|
||||
pipeWriter.CloseWithError(fmt.Errorf("Unexpected content-type %s sent for event-type %s", c, progressEvent))
|
||||
closeResponse(s.resp)
|
||||
return
|
||||
}
|
||||
case statsEvent:
|
||||
switch c {
|
||||
case xmlContent:
|
||||
if err = xmlDecoder(io.LimitReader(crcReader, payloadLen), s.stats); err != nil {
|
||||
pipeWriter.CloseWithError(err)
|
||||
closeResponse(s.resp)
|
||||
return
|
||||
}
|
||||
default:
|
||||
pipeWriter.CloseWithError(fmt.Errorf("Unexpected content-type %s sent for event-type %s", c, statsEvent))
|
||||
closeResponse(s.resp)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Ensures that the full message's CRC is correct and
|
||||
// that the message is not corrupted
|
||||
if err := checkCRC(s.resp.Body, crc.Sum32()); err != nil {
|
||||
pipeWriter.CloseWithError(err)
|
||||
closeResponse(s.resp)
|
||||
return
|
||||
}
|
||||
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// PayloadLen is a function that calculates the length of the payload.
|
||||
func (p preludeInfo) PayloadLen() int64 {
|
||||
return int64(p.totalLen - p.headerLen - 16)
|
||||
}
|
||||
|
||||
// processPrelude is the function that reads the 12 bytes of the prelude and
|
||||
// ensures the CRC is correct while also extracting relevant information into
|
||||
// the struct,
|
||||
func processPrelude(prelude io.Reader, crc hash.Hash32) (preludeInfo, error) {
|
||||
var err error
|
||||
var pInfo = preludeInfo{}
|
||||
|
||||
// reads total length of the message (first 4 bytes)
|
||||
pInfo.totalLen, err = extractUint32(prelude)
|
||||
if err != nil {
|
||||
return pInfo, err
|
||||
}
|
||||
|
||||
// reads total header length of the message (2nd 4 bytes)
|
||||
pInfo.headerLen, err = extractUint32(prelude)
|
||||
if err != nil {
|
||||
return pInfo, err
|
||||
}
|
||||
|
||||
// checks that the CRC is correct (3rd 4 bytes)
|
||||
preCRC := crc.Sum32()
|
||||
if err := checkCRC(prelude, preCRC); err != nil {
|
||||
return pInfo, err
|
||||
}
|
||||
|
||||
return pInfo, nil
|
||||
}
|
||||
|
||||
// extracts the relevant information from the Headers.
|
||||
func extractHeader(body io.Reader, myHeaders http.Header) error {
|
||||
for {
|
||||
// extracts the first part of the header,
|
||||
headerTypeName, err := extractHeaderType(body)
|
||||
if err != nil {
|
||||
// Since end of file, we have read all of our headers
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// reads the 7 present in the header and ignores it.
|
||||
extractUint8(body)
|
||||
|
||||
headerValueName, err := extractHeaderValue(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
myHeaders.Set(headerTypeName, headerValueName)
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// extractHeaderType extracts the first half of the header message, the header type.
|
||||
func extractHeaderType(body io.Reader) (string, error) {
|
||||
// extracts 2 bit integer
|
||||
headerNameLen, err := extractUint8(body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// extracts the string with the appropriate number of bytes
|
||||
headerName, err := extractString(body, int(headerNameLen))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimPrefix(headerName, ":"), nil
|
||||
}
|
||||
|
||||
// extractsHeaderValue extracts the second half of the header message, the
|
||||
// header value
|
||||
func extractHeaderValue(body io.Reader) (string, error) {
|
||||
bodyLen, err := extractUint16(body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
bodyName, err := extractString(body, int(bodyLen))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return bodyName, nil
|
||||
}
|
||||
|
||||
// extracts a string from byte array of a particular number of bytes.
|
||||
func extractString(source io.Reader, lenBytes int) (string, error) {
|
||||
myVal := make([]byte, lenBytes)
|
||||
_, err := source.Read(myVal)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(myVal), nil
|
||||
}
|
||||
|
||||
// extractUint32 extracts a 4 byte integer from the byte array.
|
||||
func extractUint32(r io.Reader) (uint32, error) {
|
||||
buf := make([]byte, 4)
|
||||
_, err := io.ReadFull(r, buf)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return binary.BigEndian.Uint32(buf), nil
|
||||
}
|
||||
|
||||
// extractUint16 extracts a 2 byte integer from the byte array.
|
||||
func extractUint16(r io.Reader) (uint16, error) {
|
||||
buf := make([]byte, 2)
|
||||
_, err := io.ReadFull(r, buf)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return binary.BigEndian.Uint16(buf), nil
|
||||
}
|
||||
|
||||
// extractUint8 extracts a 1 byte integer from the byte array.
|
||||
func extractUint8(r io.Reader) (uint8, error) {
|
||||
buf := make([]byte, 1)
|
||||
_, err := io.ReadFull(r, buf)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return buf[0], nil
|
||||
}
|
||||
|
||||
// checkCRC ensures that the CRC matches with the one from the reader.
|
||||
func checkCRC(r io.Reader, expect uint32) error {
|
||||
msgCRC, err := extractUint32(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if msgCRC != expect {
|
||||
return fmt.Errorf("Checksum Mismatch, MessageCRC of 0x%X does not equal expected CRC of 0x%X", msgCRC, expect)
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
25
vendor/github.com/minio/minio-go/api.go
сгенерированный
поставляемый
25
vendor/github.com/minio/minio-go/api.go
сгенерированный
поставляемый
@@ -99,7 +99,7 @@ type Options struct {
|
||||
// Global constants.
|
||||
const (
|
||||
libraryName = "minio-go"
|
||||
libraryVersion = "v6.0.5"
|
||||
libraryVersion = "v6.0.7"
|
||||
)
|
||||
|
||||
// User Agent should always following the below style.
|
||||
@@ -455,22 +455,9 @@ func (c Client) dumpHTTP(req *http.Request, resp *http.Response) error {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// WORKAROUND for https://github.com/golang/go/issues/13942.
|
||||
// httputil.DumpResponse does not print response headers for
|
||||
// all successful calls which have response ContentLength set
|
||||
// to zero. Keep this workaround until the above bug is fixed.
|
||||
if resp.ContentLength == 0 {
|
||||
var buffer bytes.Buffer
|
||||
if err = resp.Header.Write(&buffer); err != nil {
|
||||
return err
|
||||
}
|
||||
respTrace = buffer.Bytes()
|
||||
respTrace = append(respTrace, []byte("\r\n")...)
|
||||
} else {
|
||||
respTrace, err = httputil.DumpResponse(resp, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
respTrace, err = httputil.DumpResponse(resp, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -599,8 +586,8 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque
|
||||
// Initiate the request.
|
||||
res, err = c.do(req)
|
||||
if err != nil {
|
||||
// For supported network errors verify.
|
||||
if isNetErrorRetryable(err) {
|
||||
// For supported http requests errors verify.
|
||||
if isHTTPReqErrorRetryable(err) {
|
||||
continue // Retry.
|
||||
}
|
||||
// For other errors, return here no need to retry.
|
||||
|
||||
9
vendor/github.com/minio/minio-go/core.go
сгенерированный
поставляемый
9
vendor/github.com/minio/minio-go/core.go
сгенерированный
поставляемый
@@ -21,6 +21,8 @@ import (
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go/pkg/encrypt"
|
||||
)
|
||||
|
||||
// Core - Inherits Client and adds new methods to expose the low level S3 APIs.
|
||||
@@ -68,7 +70,7 @@ func (c Core) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string
|
||||
}
|
||||
|
||||
// PutObject - Upload object. Uploads using single PUT call.
|
||||
func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, metadata map[string]string) (ObjectInfo, error) {
|
||||
func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, metadata map[string]string, sse encrypt.ServerSide) (ObjectInfo, error) {
|
||||
opts := PutObjectOptions{}
|
||||
m := make(map[string]string)
|
||||
for k, v := range metadata {
|
||||
@@ -89,6 +91,7 @@ func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Ba
|
||||
}
|
||||
}
|
||||
opts.UserMetadata = m
|
||||
opts.ServerSideEncryption = sse
|
||||
return c.putObjectDo(context.Background(), bucket, object, data, md5Base64, sha256Hex, size, opts)
|
||||
}
|
||||
|
||||
@@ -104,8 +107,8 @@ func (c Core) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, de
|
||||
}
|
||||
|
||||
// PutObjectPart - Upload an object part.
|
||||
func (c Core) PutObjectPart(bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string) (ObjectPart, error) {
|
||||
return c.uploadPart(context.Background(), bucket, object, uploadID, data, partID, md5Base64, sha256Hex, size, nil)
|
||||
func (c Core) PutObjectPart(bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string, sse encrypt.ServerSide) (ObjectPart, error) {
|
||||
return c.uploadPart(context.Background(), bucket, object, uploadID, data, partID, md5Base64, sha256Hex, size, sse)
|
||||
}
|
||||
|
||||
// ListObjectParts - List uploaded parts of an incomplete upload.x
|
||||
|
||||
165
vendor/github.com/minio/minio-go/functional_tests.go
сгенерированный
поставляемый
165
vendor/github.com/minio/minio-go/functional_tests.go
сгенерированный
поставляемый
@@ -2271,7 +2271,8 @@ func testPresignedPostPolicy() {
|
||||
defer reader.Close()
|
||||
|
||||
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
||||
metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
||||
// Azure requires the key to not start with a number
|
||||
metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "user")
|
||||
metadataValue := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
||||
|
||||
buf, err := ioutil.ReadAll(reader)
|
||||
@@ -7023,6 +7024,167 @@ func testFGetObjectWithContext() {
|
||||
|
||||
}
|
||||
|
||||
// Test get object ACLs with GetObjectACL
|
||||
func testGetObjectACL() {
|
||||
// initialize logging params
|
||||
startTime := time.Now()
|
||||
testName := getFuncName()
|
||||
function := "GetObjectACL(bucketName, objectName)"
|
||||
args := map[string]interface{}{
|
||||
"bucketName": "",
|
||||
"objectName": "",
|
||||
}
|
||||
// Seed random based on current time.
|
||||
rand.Seed(time.Now().Unix())
|
||||
|
||||
// skipping region functional tests for non s3 runs
|
||||
if os.Getenv(serverEndpoint) != "s3.amazonaws.com" {
|
||||
ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info()
|
||||
return
|
||||
}
|
||||
|
||||
// Instantiate new minio client object.
|
||||
c, err := minio.NewV4(
|
||||
os.Getenv(serverEndpoint),
|
||||
os.Getenv(accessKey),
|
||||
os.Getenv(secretKey),
|
||||
mustParseBool(os.Getenv(enableHTTPS)),
|
||||
)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "Minio client v4 object creation failed", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Enable tracing, write to stderr.
|
||||
// c.TraceOn(os.Stderr)
|
||||
|
||||
// Set user agent.
|
||||
c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0")
|
||||
|
||||
// Generate a new random bucket name.
|
||||
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
|
||||
args["bucketName"] = bucketName
|
||||
|
||||
// Make a new bucket.
|
||||
err = c.MakeBucket(bucketName, "us-east-1")
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
|
||||
return
|
||||
}
|
||||
|
||||
bufSize := dataFileMap["datafile-1-MB"]
|
||||
var reader = getDataReader("datafile-1-MB")
|
||||
defer reader.Close()
|
||||
// Save the data
|
||||
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
||||
args["objectName"] = objectName
|
||||
|
||||
// Add meta data to add a canned acl
|
||||
metaData := map[string]string{
|
||||
"X-Amz-Acl": "public-read-write",
|
||||
}
|
||||
|
||||
_, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", UserMetadata: metaData})
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Read the data back
|
||||
objectInfo, getObjectACLErr := c.GetObjectACL(bucketName, objectName)
|
||||
if getObjectACLErr == nil {
|
||||
logError(testName, function, args, startTime, "", "GetObjectACL fail", getObjectACLErr)
|
||||
return
|
||||
}
|
||||
|
||||
s, ok := objectInfo.Metadata["X-Amz-Acl"]
|
||||
if !ok {
|
||||
logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Acl\"", nil)
|
||||
return
|
||||
}
|
||||
|
||||
if len(s) != 1 {
|
||||
logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" canned acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil)
|
||||
return
|
||||
}
|
||||
|
||||
if s[0] != "public-read-write" {
|
||||
logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" expected \"public-read-write\" but got"+fmt.Sprintf("%q", s[0]), nil)
|
||||
return
|
||||
}
|
||||
|
||||
bufSize = dataFileMap["datafile-1-MB"]
|
||||
var reader2 = getDataReader("datafile-1-MB")
|
||||
defer reader2.Close()
|
||||
// Save the data
|
||||
objectName = randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
||||
args["objectName"] = objectName
|
||||
|
||||
// Add meta data to add a canned acl
|
||||
metaData = map[string]string{
|
||||
"X-Amz-Grant-Read": "id=fooread@minio.go",
|
||||
"X-Amz-Grant-Write": "id=foowrite@minio.go",
|
||||
}
|
||||
|
||||
_, err = c.PutObject(bucketName, objectName, reader2, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", UserMetadata: metaData})
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Read the data back
|
||||
objectInfo, getObjectACLErr = c.GetObjectACL(bucketName, objectName)
|
||||
if getObjectACLErr == nil {
|
||||
logError(testName, function, args, startTime, "", "GetObjectACL fail", getObjectACLErr)
|
||||
return
|
||||
}
|
||||
|
||||
if len(objectInfo.Metadata) != 3 {
|
||||
logError(testName, function, args, startTime, "", "GetObjectACL fail expected \"3\" ACLs but got "+fmt.Sprintf(`"%d"`, len(objectInfo.Metadata)), nil)
|
||||
return
|
||||
}
|
||||
|
||||
s, ok = objectInfo.Metadata["X-Amz-Grant-Read"]
|
||||
if !ok {
|
||||
logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Read\"", nil)
|
||||
return
|
||||
}
|
||||
|
||||
if len(s) != 1 {
|
||||
logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil)
|
||||
return
|
||||
}
|
||||
|
||||
if s[0] != "fooread@minio.go" {
|
||||
logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"fooread@minio.go\" got "+fmt.Sprintf("%q", s), nil)
|
||||
return
|
||||
}
|
||||
|
||||
s, ok = objectInfo.Metadata["X-Amz-Grant-Write"]
|
||||
if !ok {
|
||||
logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Write\"", nil)
|
||||
return
|
||||
}
|
||||
|
||||
if len(s) != 1 {
|
||||
logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil)
|
||||
return
|
||||
}
|
||||
|
||||
if s[0] != "foowrite@minio.go" {
|
||||
logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"foowrite@minio.go\" got "+fmt.Sprintf("%q", s), nil)
|
||||
return
|
||||
}
|
||||
|
||||
// Delete all objects and buckets
|
||||
if err = cleanupBucket(bucketName, c); err != nil {
|
||||
logError(testName, function, args, startTime, "", "Cleanup failed", err)
|
||||
return
|
||||
}
|
||||
|
||||
successLogger(testName, function, args, startTime).Info()
|
||||
}
|
||||
|
||||
// Test validates putObject with context to see if request cancellation is honored for V2.
|
||||
func testPutObjectWithContextV2() {
|
||||
// initialize logging params
|
||||
@@ -7481,6 +7643,7 @@ func main() {
|
||||
testGetObjectWithContext()
|
||||
testFPutObjectWithContext()
|
||||
testFGetObjectWithContext()
|
||||
testGetObjectACL()
|
||||
testPutObjectWithContext()
|
||||
testStorageClassMetadataPutObject()
|
||||
testStorageClassInvalidMetadataPutObject()
|
||||
|
||||
18
vendor/github.com/minio/minio-go/pkg/credentials/file_minio_client.go
сгенерированный
поставляемый
18
vendor/github.com/minio/minio-go/pkg/credentials/file_minio_client.go
сгенерированный
поставляемый
@@ -62,13 +62,17 @@ func NewFileMinioClient(filename string, alias string) *Credentials {
|
||||
// users home directory.
|
||||
func (p *FileMinioClient) Retrieve() (Value, error) {
|
||||
if p.filename == "" {
|
||||
homeDir, err := homedir.Dir()
|
||||
if err != nil {
|
||||
return Value{}, err
|
||||
}
|
||||
p.filename = filepath.Join(homeDir, ".mc", "config.json")
|
||||
if runtime.GOOS == "windows" {
|
||||
p.filename = filepath.Join(homeDir, "mc", "config.json")
|
||||
if value, ok := os.LookupEnv("MINIO_SHARED_CREDENTIALS_FILE"); ok {
|
||||
p.filename = value
|
||||
} else {
|
||||
homeDir, err := homedir.Dir()
|
||||
if err != nil {
|
||||
return Value{}, err
|
||||
}
|
||||
p.filename = filepath.Join(homeDir, ".mc", "config.json")
|
||||
if runtime.GOOS == "windows" {
|
||||
p.filename = filepath.Join(homeDir, "mc", "config.json")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
46
vendor/github.com/minio/minio-go/retry.go
сгенерированный
поставляемый
46
vendor/github.com/minio/minio-go/retry.go
сгенерированный
поставляемый
@@ -85,36 +85,32 @@ func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duratio
|
||||
return attemptCh
|
||||
}
|
||||
|
||||
// isNetErrorRetryable - is network error retryable.
|
||||
func isNetErrorRetryable(err error) bool {
|
||||
// isHTTPReqErrorRetryable - is http requests error retryable, such
|
||||
// as i/o timeout, connection broken etc..
|
||||
func isHTTPReqErrorRetryable(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
switch err.(type) {
|
||||
case net.Error:
|
||||
switch err.(type) {
|
||||
switch e := err.(type) {
|
||||
case *url.Error:
|
||||
switch e.Err.(type) {
|
||||
case *net.DNSError, *net.OpError, net.UnknownNetworkError:
|
||||
return true
|
||||
case *url.Error:
|
||||
// For a URL error, where it replies back "connection closed"
|
||||
// retry again.
|
||||
if strings.Contains(err.Error(), "Connection closed by foreign host") {
|
||||
return true
|
||||
}
|
||||
default:
|
||||
if strings.Contains(err.Error(), "net/http: TLS handshake timeout") {
|
||||
// If error is - tlsHandshakeTimeoutError, retry.
|
||||
return true
|
||||
} else if strings.Contains(err.Error(), "i/o timeout") {
|
||||
// If error is - tcp timeoutError, retry.
|
||||
return true
|
||||
} else if strings.Contains(err.Error(), "connection timed out") {
|
||||
// If err is a net.Dial timeout, retry.
|
||||
return true
|
||||
} else if strings.Contains(err.Error(), "net/http: HTTP/1.x transport connection broken") {
|
||||
// If error is transport connection broken, retry.
|
||||
return true
|
||||
}
|
||||
}
|
||||
if strings.Contains(err.Error(), "Connection closed by foreign host") {
|
||||
return true
|
||||
} else if strings.Contains(err.Error(), "net/http: TLS handshake timeout") {
|
||||
// If error is - tlsHandshakeTimeoutError, retry.
|
||||
return true
|
||||
} else if strings.Contains(err.Error(), "i/o timeout") {
|
||||
// If error is - tcp timeoutError, retry.
|
||||
return true
|
||||
} else if strings.Contains(err.Error(), "connection timed out") {
|
||||
// If err is a net.Dial timeout, retry.
|
||||
return true
|
||||
} else if strings.Contains(err.Error(), "net/http: HTTP/1.x transport connection broken") {
|
||||
// If error is transport connection broken, retry.
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
|
||||
3
vendor/github.com/minio/minio-go/utils.go
сгенерированный
поставляемый
3
vendor/github.com/minio/minio-go/utils.go
сгенерированный
поставляемый
@@ -223,6 +223,7 @@ var supportedHeaders = []string{
|
||||
"content-disposition",
|
||||
"content-language",
|
||||
"x-amz-website-redirect-location",
|
||||
"expires",
|
||||
// Add more supported headers here.
|
||||
}
|
||||
|
||||
@@ -267,5 +268,5 @@ func isSSEHeader(headerKey string) bool {
|
||||
func isAmzHeader(headerKey string) bool {
|
||||
key := strings.ToLower(headerKey)
|
||||
|
||||
return strings.HasPrefix(key, "x-amz-meta-") || key == "x-amz-acl"
|
||||
return strings.HasPrefix(key, "x-amz-meta-") || strings.HasPrefix(key, "x-amz-grant-") || key == "x-amz-acl" || isSSEHeader(headerKey)
|
||||
}
|
||||
|
||||
Ссылка в новой задаче
Block a user