diff --git a/go.mod b/go.mod index e53ebb3476..d38fbc01b9 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( github.com/mattn/go-runewidth v0.0.8 // indirect github.com/mattn/go-sqlite3 v1.11.0 github.com/miekg/dns v1.1.27 // indirect - github.com/minio/minio-go/v6 v6.0.45 + github.com/minio/minio-go/v6 v6.0.53 github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/muesli/smartcrop v0.3.0 // indirect github.com/oklog/run v1.1.0 // indirect @@ -73,11 +73,11 @@ require ( github.com/prometheus/client_model v0.2.0 // indirect github.com/rs/cors v1.7.0 github.com/rudderlabs/analytics-go v3.1.0+incompatible - github.com/russellhaering/goxmldsig v0.0.0-20180430223755-7acd5e4a6ef7 + github.com/russellhaering/goxmldsig v0.0.0-20180430223755-7acd5e4a6ef7 // indirect github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd github.com/segmentio/analytics-go v3.1.0+incompatible github.com/segmentio/backo-go v0.0.0-20160424052352-204274ad699c // indirect - github.com/sirupsen/logrus v1.4.2 + github.com/sirupsen/logrus v1.5.0 github.com/smartystreets/assertions v1.0.0 // indirect github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945 // indirect github.com/spf13/afero v1.2.2 // indirect diff --git a/go.sum b/go.sum index 970aaafc04..2039d35819 100644 --- a/go.sum +++ b/go.sum @@ -310,6 +310,8 @@ github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM= github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/minio/minio-go/v6 v6.0.45 h1:aY4NI/DOgSbZiwGN3fEF4NAkC9An4bhaIWuJrQrRYew= github.com/minio/minio-go/v6 v6.0.45/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= +github.com/minio/minio-go/v6 v6.0.53 h1:8jzpwiOzZ5Iz7/goFWqNZRICbyWYShbb5rARjrnSCNI= +github.com/minio/minio-go/v6 v6.0.53/go.mod h1:DIvC/IApeHX8q1BAMVCXSXwpmrmM+I+iBvhvztQorfI= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -454,6 +456,8 @@ github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5k github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= +github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= diff --git a/vendor/github.com/minio/minio-go/v6/README.md b/vendor/github.com/minio/minio-go/v6/README.md index 810a93941c..33eb6c44de 100644 --- a/vendor/github.com/minio/minio-go/v6/README.md +++ b/vendor/github.com/minio/minio-go/v6/README.md @@ -190,6 +190,11 @@ The full API Reference is available here. * [setbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketlifecycle.go) * [getbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketlifecycle.go) +### Full Examples : Bucket encryption Operations +* [setbucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketencryption.go) +* [getbucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketencryption.go) +* [deletebucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/deletebucketencryption.go) + ### Full Examples : Bucket notification Operations * [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go) * [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go) diff --git a/vendor/github.com/minio/minio-go/v6/api-compose-object.go b/vendor/github.com/minio/minio-go/v6/api-compose-object.go index c9e44c556e..15e3b6f38a 100644 --- a/vendor/github.com/minio/minio-go/v6/api-compose-object.go +++ b/vendor/github.com/minio/minio-go/v6/api-compose-object.go @@ -60,6 +60,13 @@ type DestInfoOptions struct { // Otherwise this field is ignored UserTags map[string]string ReplaceTags bool + + // Specifies whether you want to apply a Legal Hold to the copied object. + LegalHold LegalHoldStatus + + // Object Retention related fields + Mode RetentionMode + RetainUntilDate time.Time } // Process custom-metadata to remove a `x-amz-meta-` prefix if @@ -107,6 +114,8 @@ func NewDestinationInfo(bucket, object string, sse encrypt.ServerSide, userMeta UserMeta: m, UserTags: nil, ReplaceTags: false, + LegalHold: LegalHoldStatus(""), + Mode: RetentionMode(""), } return DestinationInfo{ bucket: bucket, diff --git a/vendor/github.com/minio/minio-go/v6/api-datatypes.go b/vendor/github.com/minio/minio-go/v6/api-datatypes.go index e81a3bd41c..58e3c5e0e7 100644 --- a/vendor/github.com/minio/minio-go/v6/api-datatypes.go +++ b/vendor/github.com/minio/minio-go/v6/api-datatypes.go @@ -82,6 +82,9 @@ type ObjectInfo struct { // x-amz-meta-* headers stripped "x-amz-meta-" prefix containing the first value. UserMetadata StringMap `json:"userMetadata"` + // x-amz-tagging values in their k/v values. + UserTags map[string]string `json:"userTags"` + // Owner name. Owner struct { DisplayName string `json:"name"` diff --git a/vendor/github.com/minio/minio-go/v6/api-get-bucket-encryption.go b/vendor/github.com/minio/minio-go/v6/api-get-bucket-encryption.go new file mode 100644 index 0000000000..213abb8fc2 --- /dev/null +++ b/vendor/github.com/minio/minio-go/v6/api-get-bucket-encryption.go @@ -0,0 +1,71 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "encoding/xml" + "io/ioutil" + "net/http" + "net/url" + + "github.com/minio/minio-go/v6/pkg/s3utils" +) + +// GetBucketEncryption - get default encryption configuration for a bucket. +func (c Client) GetBucketEncryption(bucketName string) (ServerSideEncryptionConfiguration, error) { + return c.GetBucketEncryptionWithContext(context.Background(), bucketName) +} + +// GetBucketEncryptionWithContext gets the default encryption configuration on an existing bucket with a context to control cancellations and timeouts. +func (c Client) GetBucketEncryptionWithContext(ctx context.Context, bucketName string) (ServerSideEncryptionConfiguration, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return ServerSideEncryptionConfiguration{}, err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("encryption", "") + + // Execute GET on bucket to get the default encryption configuration. + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + }) + + defer closeResponse(resp) + if err != nil { + return ServerSideEncryptionConfiguration{}, err + } + + if resp.StatusCode != http.StatusOK { + return ServerSideEncryptionConfiguration{}, httpRespToErrorResponse(resp, bucketName, "") + } + + bucketEncryptionBuf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return ServerSideEncryptionConfiguration{}, err + } + + encryptionConfig := ServerSideEncryptionConfiguration{} + if err := xml.Unmarshal(bucketEncryptionBuf, &encryptionConfig); err != nil { + return ServerSideEncryptionConfiguration{}, err + } + return encryptionConfig, nil +} diff --git a/vendor/github.com/minio/minio-go/v6/api-get-bucket-versioning.go b/vendor/github.com/minio/minio-go/v6/api-get-bucket-versioning.go new file mode 100644 index 0000000000..9b13d6730e --- /dev/null +++ b/vendor/github.com/minio/minio-go/v6/api-get-bucket-versioning.go @@ -0,0 +1,78 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "encoding/xml" + "io/ioutil" + "net/http" + "net/url" + + "github.com/minio/minio-go/v6/pkg/s3utils" +) + +// BucketVersioningConfiguration is the versioning configuration structure +type BucketVersioningConfiguration struct { + XMLName xml.Name `xml:"VersioningConfiguration"` + Status string `xml:"Status"` + MfaDelete string `xml:"MfaDelete,omitempty"` +} + +// GetBucketVersioning - get versioning configuration for a bucket. +func (c Client) GetBucketVersioning(bucketName string) (BucketVersioningConfiguration, error) { + return c.GetBucketVersioningWithContext(context.Background(), bucketName) +} + +// GetBucketVersioningWithContext gets the versioning configuration on an existing bucket with a context to control cancellations and timeouts. +func (c Client) GetBucketVersioningWithContext(ctx context.Context, bucketName string) (BucketVersioningConfiguration, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return BucketVersioningConfiguration{}, err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("versioning", "") + + // Execute GET on bucket to get the versioning configuration. + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + }) + + defer closeResponse(resp) + if err != nil { + return BucketVersioningConfiguration{}, err + } + + if resp.StatusCode != http.StatusOK { + return BucketVersioningConfiguration{}, httpRespToErrorResponse(resp, bucketName, "") + } + + bucketVersioningBuf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return BucketVersioningConfiguration{}, err + } + + versioningConfig := BucketVersioningConfiguration{} + if err := xml.Unmarshal(bucketVersioningBuf, &versioningConfig); err != nil { + return BucketVersioningConfiguration{}, err + } + return versioningConfig, nil +} diff --git a/vendor/github.com/minio/minio-go/v6/api-get-object-file.go b/vendor/github.com/minio/minio-go/v6/api-get-object-file.go index 0684b2b3ae..c2d277514c 100644 --- a/vendor/github.com/minio/minio-go/v6/api-get-object-file.go +++ b/vendor/github.com/minio/minio-go/v6/api-get-object-file.go @@ -87,6 +87,17 @@ func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectNam return err } + // If we return early with an error, be sure to close and delete + // filePart. If we have an error along the way there is a chance + // that filePart is somehow damaged, and we should discard it. + closeAndRemove := true + defer func() { + if closeAndRemove { + _ = filePart.Close() + _ = os.Remove(filePartPath) + } + }() + // Issue Stat to get the current offset. st, err = filePart.Stat() if err != nil { @@ -111,6 +122,7 @@ func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectNam } // Close the file before rename, this is specifically needed for Windows users. + closeAndRemove = false if err = filePart.Close(); err != nil { return err } diff --git a/vendor/github.com/minio/minio-go/v6/api-get-object.go b/vendor/github.com/minio/minio-go/v6/api-get-object.go index c73f4a6b4f..14370b0cdb 100644 --- a/vendor/github.com/minio/minio-go/v6/api-get-object.go +++ b/vendor/github.com/minio/minio-go/v6/api-get-object.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,9 +23,7 @@ import ( "fmt" "io" "net/http" - "strings" "sync" - "time" "github.com/minio/minio-go/v6/pkg/s3utils" ) @@ -635,38 +633,10 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, op } } - // Trim off the odd double quotes from ETag in the beginning and end. - md5sum := trimEtag(resp.Header.Get("ETag")) - - // Parse the date. - date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified")) + objectStat, err := ToObjectInfo(bucketName, objectName, resp.Header) if err != nil { - msg := "Last-Modified time format not recognized. " + reportIssue - return nil, ObjectInfo{}, nil, ErrorResponse{ - Code: "InternalError", - Message: msg, - RequestID: resp.Header.Get("x-amz-request-id"), - HostID: resp.Header.Get("x-amz-id-2"), - Region: resp.Header.Get("x-amz-bucket-region"), - } - } - - // Get content-type. - contentType := strings.TrimSpace(resp.Header.Get("Content-Type")) - if contentType == "" { - contentType = "application/octet-stream" - } - - objectStat := ObjectInfo{ - ETag: md5sum, - Key: objectName, - Size: resp.ContentLength, - LastModified: date, - ContentType: contentType, - // Extract only the relevant header keys describing the object. - // following function filters out a list of standard set of keys - // which are not part of object metadata. - Metadata: extractObjMetadata(resp.Header), + closeResponse(resp) + return nil, objectStat, resp.Header, nil } // do not close body here, caller will close diff --git a/vendor/github.com/minio/minio-go/v6/api-list.go b/vendor/github.com/minio/minio-go/v6/api-list.go index c48201ce76..1bcb64f3dd 100644 --- a/vendor/github.com/minio/minio-go/v6/api-list.go +++ b/vendor/github.com/minio/minio-go/v6/api-list.go @@ -317,14 +317,14 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s } for i, obj := range listBucketResult.Contents { - listBucketResult.Contents[i].Key, err = url.QueryUnescape(obj.Key) + listBucketResult.Contents[i].Key, err = decodeS3Name(obj.Key, listBucketResult.EncodingType) if err != nil { return listBucketResult, err } } for i, obj := range listBucketResult.CommonPrefixes { - listBucketResult.CommonPrefixes[i].Prefix, err = url.QueryUnescape(obj.Prefix) + listBucketResult.CommonPrefixes[i].Prefix, err = decodeS3Name(obj.Prefix, listBucketResult.EncodingType) if err != nil { return listBucketResult, err } @@ -500,21 +500,21 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit } for i, obj := range listBucketResult.Contents { - listBucketResult.Contents[i].Key, err = url.QueryUnescape(obj.Key) + listBucketResult.Contents[i].Key, err = decodeS3Name(obj.Key, listBucketResult.EncodingType) if err != nil { return listBucketResult, err } } for i, obj := range listBucketResult.CommonPrefixes { - listBucketResult.CommonPrefixes[i].Prefix, err = url.QueryUnescape(obj.Prefix) + listBucketResult.CommonPrefixes[i].Prefix, err = decodeS3Name(obj.Prefix, listBucketResult.EncodingType) if err != nil { return listBucketResult, err } } if listBucketResult.NextMarker != "" { - listBucketResult.NextMarker, err = url.QueryUnescape(listBucketResult.NextMarker) + listBucketResult.NextMarker, err = decodeS3Name(listBucketResult.NextMarker, listBucketResult.EncodingType) if err != nil { return listBucketResult, err } @@ -697,25 +697,25 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, return listMultipartUploadsResult, err } - listMultipartUploadsResult.NextKeyMarker, err = url.QueryUnescape(listMultipartUploadsResult.NextKeyMarker) + listMultipartUploadsResult.NextKeyMarker, err = decodeS3Name(listMultipartUploadsResult.NextKeyMarker, listMultipartUploadsResult.EncodingType) if err != nil { return listMultipartUploadsResult, err } - listMultipartUploadsResult.NextUploadIDMarker, err = url.QueryUnescape(listMultipartUploadsResult.NextUploadIDMarker) + listMultipartUploadsResult.NextUploadIDMarker, err = decodeS3Name(listMultipartUploadsResult.NextUploadIDMarker, listMultipartUploadsResult.EncodingType) if err != nil { return listMultipartUploadsResult, err } for i, obj := range listMultipartUploadsResult.Uploads { - listMultipartUploadsResult.Uploads[i].Key, err = url.QueryUnescape(obj.Key) + listMultipartUploadsResult.Uploads[i].Key, err = decodeS3Name(obj.Key, listMultipartUploadsResult.EncodingType) if err != nil { return listMultipartUploadsResult, err } } for i, obj := range listMultipartUploadsResult.CommonPrefixes { - listMultipartUploadsResult.CommonPrefixes[i].Prefix, err = url.QueryUnescape(obj.Prefix) + listMultipartUploadsResult.CommonPrefixes[i].Prefix, err = decodeS3Name(obj.Prefix, listMultipartUploadsResult.EncodingType) if err != nil { return listMultipartUploadsResult, err } @@ -837,3 +837,13 @@ func (c Client) listObjectPartsQuery(bucketName, objectName, uploadID string, pa } return listObjectPartsResult, nil } + +// Decode an S3 object name according to the encoding type +func decodeS3Name(name, encodingType string) (string, error) { + switch encodingType { + case "url": + return url.QueryUnescape(name) + default: + return name, nil + } +} diff --git a/vendor/github.com/minio/minio-go/v6/api-notification.go b/vendor/github.com/minio/minio-go/v6/api-notification.go index 0480c21eb5..b08ad993a8 100644 --- a/vendor/github.com/minio/minio-go/v6/api-notification.go +++ b/vendor/github.com/minio/minio-go/v6/api-notification.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 MinIO, Inc. + * Copyright 2017-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -89,11 +89,13 @@ type bucketMeta struct { // Notification event object metadata. type objectMeta struct { - Key string `json:"key"` - Size int64 `json:"size,omitempty"` - ETag string `json:"eTag,omitempty"` - VersionID string `json:"versionId,omitempty"` - Sequencer string `json:"sequencer"` + Key string `json:"key"` + Size int64 `json:"size,omitempty"` + ETag string `json:"eTag,omitempty"` + ContentType string `json:"contentType,omitempty"` + UserMetadata map[string]string `json:"userMetadata,omitempty"` + VersionID string `json:"versionId,omitempty"` + Sequencer string `json:"sequencer"` } // Notification event server specific metadata. diff --git a/vendor/github.com/minio/minio-go/v6/api-object-legal-hold.go b/vendor/github.com/minio/minio-go/v6/api-object-legal-hold.go new file mode 100644 index 0000000000..2b43575ace --- /dev/null +++ b/vendor/github.com/minio/minio-go/v6/api-object-legal-hold.go @@ -0,0 +1,176 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/xml" + "fmt" + "net/http" + "net/url" + + "github.com/minio/minio-go/v6/pkg/s3utils" +) + +// objectLegalHold - object legal hold specified in +// https://docs.aws.amazon.com/AmazonS3/latest/API/archive-RESTObjectPUTLegalHold.html +type objectLegalHold struct { + XMLNS string `xml:"xmlns,attr,omitempty"` + XMLName xml.Name `xml:"LegalHold"` + Status LegalHoldStatus `xml:"Status,omitempty"` +} + +// PutObjectLegalHoldOptions represents options specified by user for PutObjectLegalHold call +type PutObjectLegalHoldOptions struct { + VersionID string + Status *LegalHoldStatus +} + +// GetObjectLegalHoldOptions represents options specified by user for GetObjectLegalHold call +type GetObjectLegalHoldOptions struct { + VersionID string +} + +// LegalHoldStatus - object legal hold status. +type LegalHoldStatus string + +const ( + // LegalHoldEnabled indicates legal hold is enabled + LegalHoldEnabled LegalHoldStatus = "ON" + + // LegalHoldDisabled indicates legal hold is disabled + LegalHoldDisabled LegalHoldStatus = "OFF" +) + +func (r LegalHoldStatus) String() string { + return string(r) +} + +// IsValid - check whether this legal hold status is valid or not. +func (r LegalHoldStatus) IsValid() bool { + return r == LegalHoldEnabled || r == LegalHoldDisabled +} + +func newObjectLegalHold(status *LegalHoldStatus) (*objectLegalHold, error) { + if status == nil { + return nil, fmt.Errorf("Status not set") + } + if !status.IsValid() { + return nil, fmt.Errorf("invalid legal hold status `%v`", status) + } + legalHold := &objectLegalHold{ + Status: *status, + } + return legalHold, nil +} + +// PutObjectLegalHold : sets object legal hold for a given object and versionID. +func (c Client) PutObjectLegalHold(bucketName, objectName string, opts PutObjectLegalHoldOptions) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("legal-hold", "") + + if opts.VersionID != "" { + urlValues.Set("versionId", opts.VersionID) + } + + lh, err := newObjectLegalHold(opts.Status) + if err != nil { + return err + } + + lhData, err := xml.Marshal(lh) + if err != nil { + return err + } + + reqMetadata := requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + contentBody: bytes.NewReader(lhData), + contentLength: int64(len(lhData)), + contentMD5Base64: sumMD5Base64(lhData), + contentSHA256Hex: sum256Hex(lhData), + } + + // Execute PUT Object Legal Hold. + resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return httpRespToErrorResponse(resp, bucketName, objectName) + } + } + return nil +} + +// GetObjectLegalHold gets legal-hold status of given object. +func (c Client) GetObjectLegalHold(bucketName, objectName string, opts GetObjectLegalHoldOptions) (status *LegalHoldStatus, err error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return nil, err + } + + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return nil, err + } + urlValues := make(url.Values) + urlValues.Set("legal-hold", "") + + if opts.VersionID != "" { + urlValues.Set("versionId", opts.VersionID) + } + + // Execute GET on bucket to list objects. + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return nil, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return nil, httpRespToErrorResponse(resp, bucketName, objectName) + } + } + lh := &objectLegalHold{} + if err = xml.NewDecoder(resp.Body).Decode(lh); err != nil { + return nil, err + } + + return &lh.Status, nil +} diff --git a/vendor/github.com/minio/minio-go/v6/api-object-lock.go b/vendor/github.com/minio/minio-go/v6/api-object-lock.go index c30ab32589..14168ef39b 100644 --- a/vendor/github.com/minio/minio-go/v6/api-object-lock.go +++ b/vendor/github.com/minio/minio-go/v6/api-object-lock.go @@ -183,11 +183,11 @@ func (c Client) SetBucketObjectLockConfig(bucketName string, mode *RetentionMode return nil } -// GetBucketObjectLockConfig gets object lock configuration of given bucket. -func (c Client) GetBucketObjectLockConfig(bucketName string) (mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { +// GetObjectLockConfig gets object lock configuration of given bucket. +func (c Client) GetObjectLockConfig(bucketName string) (objectLock string, mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return nil, nil, nil, err + return "", nil, nil, nil, err } urlValues := make(url.Values) @@ -201,16 +201,16 @@ func (c Client) GetBucketObjectLockConfig(bucketName string) (mode *RetentionMod }) defer closeResponse(resp) if err != nil { - return nil, nil, nil, err + return "", nil, nil, nil, err } if resp != nil { if resp.StatusCode != http.StatusOK { - return nil, nil, nil, httpRespToErrorResponse(resp, bucketName, "") + return "", nil, nil, nil, httpRespToErrorResponse(resp, bucketName, "") } } config := &objectLockConfig{} if err = xml.NewDecoder(resp.Body).Decode(config); err != nil { - return nil, nil, nil, err + return "", nil, nil, nil, err } if config.Rule != nil { @@ -224,9 +224,18 @@ func (c Client) GetBucketObjectLockConfig(bucketName string) (mode *RetentionMod years := Years unit = &years } - - return mode, validity, unit, nil + return config.ObjectLockEnabled, mode, validity, unit, nil } - - return nil, nil, nil, nil + return config.ObjectLockEnabled, nil, nil, nil, nil +} + +// GetBucketObjectLockConfig gets object lock configuration of given bucket. +func (c Client) GetBucketObjectLockConfig(bucketName string) (mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { + _, mode, validity, unit, err = c.GetObjectLockConfig(bucketName) + return mode, validity, unit, err +} + +// SetObjectLockConfig sets object lock configuration in given bucket. mode, validity and unit are either all set or all nil. +func (c Client) SetObjectLockConfig(bucketName string, mode *RetentionMode, validity *uint, unit *ValidityUnit) error { + return c.SetBucketObjectLockConfig(bucketName, mode, validity, unit) } diff --git a/vendor/github.com/minio/minio-go/v6/api-object-retention.go b/vendor/github.com/minio/minio-go/v6/api-object-retention.go index 8aa08f073a..822bcc6042 100644 --- a/vendor/github.com/minio/minio-go/v6/api-object-retention.go +++ b/vendor/github.com/minio/minio-go/v6/api-object-retention.go @@ -100,7 +100,7 @@ func (c Client) PutObjectRetention(bucketName, objectName string, opts PutObject if opts.GovernanceBypass { // Set the bypass goverenance retention header - headers.Set("x-amz-bypass-governance-retention", "True") + headers.Set(amzBypassGovernance, "true") } reqMetadata := requestMetadata{ diff --git a/vendor/github.com/minio/minio-go/v6/api-object-tagging.go b/vendor/github.com/minio/minio-go/v6/api-object-tagging.go index fdf8fab91a..66568063cd 100644 --- a/vendor/github.com/minio/minio-go/v6/api-object-tagging.go +++ b/vendor/github.com/minio/minio-go/v6/api-object-tagging.go @@ -155,7 +155,8 @@ func (c Client) RemoveObjectTaggingWithContext(ctx context.Context, bucketName, } if resp != nil { - if resp.StatusCode != http.StatusOK { + // S3 returns "204 No content" after Object tag deletion. + if resp.StatusCode != http.StatusNoContent { return httpRespToErrorResponse(resp, bucketName, objectName) } } diff --git a/vendor/github.com/minio/minio-go/v6/api-presigned.go b/vendor/github.com/minio/minio-go/v6/api-presigned.go index e2d68b0ece..2e27c3f97a 100644 --- a/vendor/github.com/minio/minio-go/v6/api-presigned.go +++ b/vendor/github.com/minio/minio-go/v6/api-presigned.go @@ -23,8 +23,8 @@ import ( "net/url" "time" - "github.com/minio/minio-go/v6/pkg/s3signer" "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/signer" ) // presignURL - Returns a presigned URL for an input 'method'. @@ -157,7 +157,7 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[str p.formData["AWSAccessKeyId"] = accessKeyID } // Sign the policy. - p.formData["signature"] = s3signer.PostPresignSignatureV2(policyBase64, secretAccessKey) + p.formData["signature"] = signer.PostPresignSignatureV2(policyBase64, secretAccessKey) return u, p.formData, nil } @@ -180,7 +180,7 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[str } // Add a credential policy. - credential := s3signer.GetCredential(accessKeyID, location, t) + credential := signer.GetCredential(accessKeyID, location, t, signer.ServiceTypeS3) if err = p.addNewPolicy(policyCondition{ matchType: "eq", condition: "$x-amz-credential", @@ -210,6 +210,6 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[str if sessionToken != "" { p.formData["x-amz-security-token"] = sessionToken } - p.formData["x-amz-signature"] = s3signer.PostPresignSignatureV4(policyBase64, t, secretAccessKey, location) + p.formData["x-amz-signature"] = signer.PostPresignSignatureV4(policyBase64, t, secretAccessKey, location) return u, p.formData, nil } diff --git a/vendor/github.com/minio/minio-go/v6/api-put-bucket.go b/vendor/github.com/minio/minio-go/v6/api-put-bucket.go index 68371986ae..8d9671b1e1 100644 --- a/vendor/github.com/minio/minio-go/v6/api-put-bucket.go +++ b/vendor/github.com/minio/minio-go/v6/api-put-bucket.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,6 +29,25 @@ import ( "github.com/minio/minio-go/v6/pkg/s3utils" ) +// ApplyServerSideEncryptionByDefault defines default encryption configuration, KMS or SSE. To activate +// KMS, SSEAlgoritm needs to be set to "aws:kms" +// Minio currently does not support Kms. +type ApplyServerSideEncryptionByDefault struct { + KmsMasterKeyID string `xml:"KMSMasterKeyID,omitempty"` + SSEAlgorithm string `xml:"SSEAlgorithm"` +} + +// Rule layer encapsulates default encryption configuration +type Rule struct { + Apply ApplyServerSideEncryptionByDefault `xml:"ApplyServerSideEncryptionByDefault"` +} + +// ServerSideEncryptionConfiguration is the default encryption configuration structure +type ServerSideEncryptionConfiguration struct { + XMLName xml.Name `xml:"ServerSideEncryptionConfiguration"` + Rules []Rule `xml:"Rule"` +} + /// Bucket operations func (c Client) makeBucket(ctx context.Context, bucketName string, location string, objectLockEnabled bool) (err error) { @@ -313,6 +332,82 @@ func (c Client) removeBucketLifecycle(ctx context.Context, bucketName string) er return nil } +// SetBucketEncryption sets the default encryption configuration on an existing bucket. +func (c Client) SetBucketEncryption(bucketName string, configuration ServerSideEncryptionConfiguration) error { + return c.SetBucketEncryptionWithContext(context.Background(), bucketName, configuration) +} + +// SetBucketEncryptionWithContext sets the default encryption configuration on an existing bucket with a context to control cancellations and timeouts. +func (c Client) SetBucketEncryptionWithContext(ctx context.Context, bucketName string, configuration ServerSideEncryptionConfiguration) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + buf, err := xml.Marshal(&configuration) + if err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("encryption", "") + + // Content-length is mandatory to set a default encryption configuration + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: bytes.NewReader(buf), + contentLength: int64(len(buf)), + contentMD5Base64: sumMD5Base64(buf), + } + + // Execute PUT to upload a new bucket default encryption configuration. + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + return nil +} + +// DeleteBucketEncryption removes the default encryption configuration on a bucket. +func (c Client) DeleteBucketEncryption(bucketName string) error { + return c.DeleteBucketEncryptionWithContext(context.Background(), bucketName) +} + +// DeleteBucketEncryptionWithContext removes the default encryption configuration on a bucket with a context to control cancellations and timeouts. +func (c Client) DeleteBucketEncryptionWithContext(ctx context.Context, bucketName string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("encryption", "") + + // DELETE default encryption configuration on a bucket. + resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return httpRespToErrorResponse(resp, bucketName, "") + } + return nil +} + // SetBucketNotification saves a new bucket notification. func (c Client) SetBucketNotification(bucketName string, bucketNotification BucketNotification) error { return c.SetBucketNotificationWithContext(context.Background(), bucketName, bucketNotification) diff --git a/vendor/github.com/minio/minio-go/v6/api-put-object-copy.go b/vendor/github.com/minio/minio-go/v6/api-put-object-copy.go index ce48674b04..aa5a36c4de 100644 --- a/vendor/github.com/minio/minio-go/v6/api-put-object-copy.go +++ b/vendor/github.com/minio/minio-go/v6/api-put-object-copy.go @@ -22,6 +22,7 @@ import ( "io" "io/ioutil" "net/http" + "time" "github.com/minio/minio-go/v6/pkg/encrypt" "github.com/minio/minio-go/v6/pkg/s3utils" @@ -45,6 +46,15 @@ func (c Client) CopyObjectWithProgress(dst DestinationInfo, src SourceInfo, prog header.Set(amzTaggingHeader, s3utils.TagEncode(dst.opts.UserTags)) } + if dst.opts.LegalHold != LegalHoldStatus("") { + header.Set(amzLegalHoldHeader, dst.opts.LegalHold.String()) + } + + if dst.opts.Mode != RetentionMode("") && !dst.opts.RetainUntilDate.IsZero() { + header.Set(amzLockMode, dst.opts.Mode.String()) + header.Set(amzLockRetainUntil, dst.opts.RetainUntilDate.Format(time.RFC3339)) + } + var err error var size int64 // If progress bar is specified, size should be requested as well initiate a StatObject request. diff --git a/vendor/github.com/minio/minio-go/v6/api-put-object-multipart.go b/vendor/github.com/minio/minio-go/v6/api-put-object-multipart.go index 864acc8bcb..e5bd37d5d4 100644 --- a/vendor/github.com/minio/minio-go/v6/api-put-object-multipart.go +++ b/vendor/github.com/minio/minio-go/v6/api-put-object-multipart.go @@ -50,7 +50,7 @@ func (c Client) putObjectMultipart(ctx context.Context, bucketName, objectName s return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) } // Fall back to uploading as single PutObject operation. - return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) + return c.putObject(ctx, bucketName, objectName, reader, size, opts) } } return n, err @@ -104,7 +104,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje // Choose hash algorithms to be calculated by hashCopyN, // avoid sha256 with non-v4 signature request or // HTTPS connection. - hashAlgos, hashSums := c.hashMaterials() + hashAlgos, hashSums := c.hashMaterials(opts.SendContentMd5) length, rErr := io.ReadFull(reader, buf) if rErr == io.EOF { diff --git a/vendor/github.com/minio/minio-go/v6/api-put-object-streaming.go b/vendor/github.com/minio/minio-go/v6/api-put-object-streaming.go index 5add2afcfc..9463062a18 100644 --- a/vendor/github.com/minio/minio-go/v6/api-put-object-streaming.go +++ b/vendor/github.com/minio/minio-go/v6/api-put-object-streaming.go @@ -18,10 +18,14 @@ package minio import ( + "bytes" "context" + "crypto/md5" + "encoding/base64" "fmt" "io" "net/http" + "runtime/debug" "sort" "strings" @@ -40,11 +44,11 @@ import ( func (c Client) putObjectMultipartStream(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { - if !isObject(reader) && isReadAt(reader) { + if !isObject(reader) && isReadAt(reader) && !opts.SendContentMd5 { // Verify if the reader implements ReadAt and it is not a *minio.Object then we will use parallel uploader. n, err = c.putObjectMultipartStreamFromReadAt(ctx, bucketName, objectName, reader.(io.ReaderAt), size, opts) } else { - n, err = c.putObjectMultipartStreamNoChecksum(ctx, bucketName, objectName, reader, size, opts) + n, err = c.putObjectMultipartStreamOptionalChecksum(ctx, bucketName, objectName, reader, size, opts) } if err != nil { errResp := ToErrorResponse(err) @@ -56,7 +60,7 @@ func (c Client) putObjectMultipartStream(ctx context.Context, bucketName, object return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) } // Fall back to uploading as single PutObject operation. - return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) + return c.putObject(ctx, bucketName, objectName, reader, size, opts) } } return n, err @@ -220,7 +224,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa return totalUploadedSize, nil } -func (c Client) putObjectMultipartStreamNoChecksum(ctx context.Context, bucketName, objectName string, +func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { // Input validation. if err = s3utils.CheckValidBucketName(bucketName); err != nil { @@ -257,20 +261,47 @@ func (c Client) putObjectMultipartStreamNoChecksum(ctx context.Context, bucketNa // Initialize parts uploaded map. partsInfo := make(map[int]ObjectPart) + // Create a buffer. + buf := make([]byte, partSize) + defer debug.FreeOSMemory() + + // Avoid declaring variables in the for loop + var md5Base64 string + var hookReader io.Reader + // Part number always starts with '1'. var partNumber int for partNumber = 1; partNumber <= totalPartsCount; partNumber++ { - // Update progress reader appropriately to the latest offset - // as we read from the source. - hookReader := newHook(reader, opts.Progress) // Proceed to upload the part. if partNumber == totalPartsCount { partSize = lastPartSize } + + if opts.SendContentMd5 { + length, rerr := io.ReadFull(reader, buf) + if rerr == io.EOF && partNumber > 1 { + break + } + if rerr != nil && rerr != io.ErrUnexpectedEOF && rerr != io.EOF { + return 0, rerr + } + // Calculate md5sum. + hash := md5.New() + hash.Write(buf[:length]) + md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil)) + // Update progress reader appropriately to the latest offset + // as we read from the source. + hookReader = newHook(bytes.NewReader(buf[:length]), opts.Progress) + } else { + // Update progress reader appropriately to the latest offset + // as we read from the source. + hookReader = newHook(reader, opts.Progress) + } + objPart, uerr := c.uploadPart(ctx, bucketName, objectName, uploadID, io.LimitReader(hookReader, partSize), - partNumber, "", "", partSize, opts.ServerSideEncryption) + partNumber, md5Base64, "", partSize, opts.ServerSideEncryption) if uerr != nil { return totalUploadedSize, uerr } @@ -316,9 +347,9 @@ func (c Client) putObjectMultipartStreamNoChecksum(ctx context.Context, bucketNa return totalUploadedSize, nil } -// putObjectNoChecksum special function used Google Cloud Storage. This special function +// putObject special function used Google Cloud Storage. This special function // is used for Google Cloud Storage since Google's multipart API is not S3 compatible. -func (c Client) putObjectNoChecksum(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { +func (c Client) putObject(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return 0, err @@ -343,13 +374,30 @@ func (c Client) putObjectNoChecksum(ctx context.Context, bucketName, objectName } } + var md5Base64 string + if opts.SendContentMd5 { + // Create a buffer. + buf := make([]byte, size) + defer debug.FreeOSMemory() + + length, rErr := io.ReadFull(reader, buf) + if rErr != nil && rErr != io.ErrUnexpectedEOF { + return 0, rErr + } + + // Calculate md5sum. + hash := md5.New() + hash.Write(buf[:length]) + md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil)) + } + // Update progress reader appropriately to the latest offset as we // read from the source. readSeeker := newHook(reader, opts.Progress) // This function does not calculate sha256 and md5sum for payload. // Execute put object. - st, err := c.putObjectDo(ctx, bucketName, objectName, readSeeker, "", "", size, opts) + st, err := c.putObjectDo(ctx, bucketName, objectName, readSeeker, md5Base64, "", size, opts) if err != nil { return 0, err } diff --git a/vendor/github.com/minio/minio-go/v6/api-put-object.go b/vendor/github.com/minio/minio-go/v6/api-put-object.go index c760056cb6..eabe788da8 100644 --- a/vendor/github.com/minio/minio-go/v6/api-put-object.go +++ b/vendor/github.com/minio/minio-go/v6/api-put-object.go @@ -20,6 +20,9 @@ package minio import ( "bytes" "context" + "crypto/md5" + "encoding/base64" + "errors" "fmt" "io" "net/http" @@ -49,6 +52,9 @@ type PutObjectOptions struct { StorageClass string WebsiteRedirectLocation string PartSize uint64 + LegalHold LegalHoldStatus + SendContentMd5 bool + DisableMultipart bool } // getNumThreads - gets the number of threads to be used in the multipart @@ -67,48 +73,58 @@ func (opts PutObjectOptions) getNumThreads() (numThreads int) { func (opts PutObjectOptions) Header() (header http.Header) { header = make(http.Header) - if opts.ContentType != "" { - header["Content-Type"] = []string{opts.ContentType} - } else { - header["Content-Type"] = []string{"application/octet-stream"} + contentType := opts.ContentType + if contentType == "" { + contentType = "application/octet-stream" } + header.Set("Content-Type", contentType) + if opts.ContentEncoding != "" { - header["Content-Encoding"] = []string{opts.ContentEncoding} + header.Set("Content-Encoding", opts.ContentEncoding) } if opts.ContentDisposition != "" { - header["Content-Disposition"] = []string{opts.ContentDisposition} + header.Set("Content-Disposition", opts.ContentDisposition) } if opts.ContentLanguage != "" { - header["Content-Language"] = []string{opts.ContentLanguage} + header.Set("Content-Language", opts.ContentLanguage) } if opts.CacheControl != "" { - header["Cache-Control"] = []string{opts.CacheControl} + header.Set("Cache-Control", opts.CacheControl) } if opts.Mode != nil { - header["x-amz-object-lock-mode"] = []string{opts.Mode.String()} + header.Set(amzLockMode, opts.Mode.String()) } + if opts.RetainUntilDate != nil { - header["x-amz-object-lock-retain-until-date"] = []string{opts.RetainUntilDate.Format(time.RFC3339)} + header.Set("X-Amz-Object-Lock-Retain-Until-Date", opts.RetainUntilDate.Format(time.RFC3339)) + } + + if opts.LegalHold != "" { + header.Set(amzLegalHoldHeader, opts.LegalHold.String()) } if opts.ServerSideEncryption != nil { opts.ServerSideEncryption.Marshal(header) } + if opts.StorageClass != "" { - header[amzStorageClass] = []string{opts.StorageClass} + header.Set(amzStorageClass, opts.StorageClass) } + if opts.WebsiteRedirectLocation != "" { - header[amzWebsiteRedirectLocation] = []string{opts.WebsiteRedirectLocation} + header.Set(amzWebsiteRedirectLocation, opts.WebsiteRedirectLocation) } + if len(opts.UserTags) != 0 { - header[amzTaggingHeader] = []string{s3utils.TagEncode(opts.UserTags)} + header.Set(amzTaggingHeader, s3utils.TagEncode(opts.UserTags)) } + for k, v := range opts.UserMetadata { if !isAmzHeader(k) && !isStandardHeader(k) && !isStorageClassHeader(k) { - header["X-Amz-Meta-"+k] = []string{v} + header.Set("X-Amz-Meta-"+k, v) } else { - header[k] = []string{v} + header.Set(k, v) } } return @@ -129,6 +145,9 @@ func (opts PutObjectOptions) validate() (err error) { return ErrInvalidArgument(opts.Mode.String() + " unsupported retention mode") } } + if opts.LegalHold != "" && !opts.LegalHold.IsValid() { + return ErrInvalidArgument(opts.LegalHold.String() + " unsupported legal-hold status") + } return nil } @@ -153,6 +172,10 @@ func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].Part // be uploaded through this operation will be 5TiB. func (c Client) PutObject(bucketName, objectName string, reader io.Reader, objectSize int64, opts PutObjectOptions) (n int64, err error) { + if objectSize < 0 && opts.DisableMultipart { + return 0, errors.New("object size must be provided with disable multipart upload") + } + return c.PutObjectWithContext(context.Background(), bucketName, objectName, reader, objectSize, opts) } @@ -164,8 +187,7 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri // NOTE: Streaming signature is not supported by GCS. if s3utils.IsGoogleEndpoint(*c.endpointURL) { - // Do not compute MD5 for Google Cloud Storage. - return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) + return c.putObject(ctx, bucketName, objectName, reader, size, opts) } partSize := opts.PartSize @@ -174,8 +196,8 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri } if c.overrideSignerType.IsV2() { - if size >= 0 && size < int64(partSize) { - return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) + if size >= 0 && size < int64(partSize) || opts.DisableMultipart { + return c.putObject(ctx, bucketName, objectName, reader, size, opts) } return c.putObjectMultipart(ctx, bucketName, objectName, reader, size, opts) } @@ -183,11 +205,10 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri return c.putObjectMultipartStreamNoLength(ctx, bucketName, objectName, reader, opts) } - if size < int64(partSize) { - return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) + if size < int64(partSize) || opts.DisableMultipart { + return c.putObject(ctx, bucketName, objectName, reader, size, opts) } - // For all sizes greater than 128MiB do multipart. return c.putObjectMultipartStream(ctx, bucketName, objectName, reader, size, opts) } @@ -243,13 +264,21 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName return 0, rerr } + var md5Base64 string + if opts.SendContentMd5 { + // Calculate md5sum. + hash := md5.New() + hash.Write(buf[:length]) + md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil)) + } + // Update progress reader appropriately to the latest offset // as we read from the source. rd := newHook(bytes.NewReader(buf[:length]), opts.Progress) // Proceed to upload the part. objPart, uerr := c.uploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber, - "", "", int64(length), opts.ServerSideEncryption) + md5Base64, "", int64(length), opts.ServerSideEncryption) if uerr != nil { return totalUploadedSize, uerr } diff --git a/vendor/github.com/minio/minio-go/v6/api-remove.go b/vendor/github.com/minio/minio-go/v6/api-remove.go index 4c8c335c31..f2879e8b58 100644 --- a/vendor/github.com/minio/minio-go/v6/api-remove.go +++ b/vendor/github.com/minio/minio-go/v6/api-remove.go @@ -63,7 +63,7 @@ func (c Client) RemoveObject(bucketName, objectName string) error { return c.RemoveObjectWithOptions(bucketName, objectName, RemoveObjectOptions{}) } -// RemoveObjectOptions represents options specified by user for PutObject call +// RemoveObjectOptions represents options specified by user for RemoveObject call type RemoveObjectOptions struct { GovernanceBypass bool VersionID string @@ -92,7 +92,7 @@ func (c Client) RemoveObjectWithOptions(bucketName, objectName string, opts Remo if opts.GovernanceBypass { // Set the bypass goverenance retention header - headers.Set("x-amz-bypass-governance-retention", "True") + headers.Set(amzBypassGovernance, "true") } // Execute DELETE on objectName. resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{ @@ -179,73 +179,109 @@ func (c Client) RemoveObjectsWithContext(ctx context.Context, bucketName string, return errorCh } - // Generate and call MultiDelete S3 requests based on entries received from objectsCh - go func(errorCh chan<- RemoveObjectError) { - maxEntries := 1000 - finish := false - urlValues := make(url.Values) - urlValues.Set("delete", "") - - // Close error channel when Multi delete finishes. - defer close(errorCh) - - // Loop over entries by 1000 and call MultiDelete requests - for { - if finish { - break - } - count := 0 - var batch []string - - // Try to gather 1000 entries - for object := range objectsCh { - batch = append(batch, object) - if count++; count >= maxEntries { - break - } - } - if count == 0 { - // Multi Objects Delete API doesn't accept empty object list, quit immediately - break - } - if count < maxEntries { - // We didn't have 1000 entries, so this is the last batch - finish = true - } - - // Generate remove multi objects XML request - removeBytes := generateRemoveMultiObjectsRequest(batch) - // Execute GET on bucket to list objects. - resp, err := c.executeMethod(ctx, "POST", requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentBody: bytes.NewReader(removeBytes), - contentLength: int64(len(removeBytes)), - contentMD5Base64: sumMD5Base64(removeBytes), - contentSHA256Hex: sum256Hex(removeBytes), - }) - if resp != nil { - if resp.StatusCode != http.StatusOK { - e := httpRespToErrorResponse(resp, bucketName, "") - errorCh <- RemoveObjectError{ObjectName: "", Err: e} - } - } - if err != nil { - for _, b := range batch { - errorCh <- RemoveObjectError{ObjectName: b, Err: err} - } - continue - } - - // Process multiobjects remove xml response - processRemoveMultiObjectsResponse(resp.Body, batch, errorCh) - - closeResponse(resp) - } - }(errorCh) + go c.removeObjects(ctx, bucketName, objectsCh, errorCh, RemoveObjectsOptions{}) return errorCh } +// RemoveObjectsWithOptionsContext - Identical to RemoveObjects call, but accepts context to +// facilitate request cancellation and options to bypass governance retention +func (c Client) RemoveObjectsWithOptionsContext(ctx context.Context, bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError { + errorCh := make(chan RemoveObjectError, 1) + + // Validate if bucket name is valid. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + defer close(errorCh) + errorCh <- RemoveObjectError{ + Err: err, + } + return errorCh + } + // Validate objects channel to be properly allocated. + if objectsCh == nil { + defer close(errorCh) + errorCh <- RemoveObjectError{ + Err: ErrInvalidArgument("Objects channel cannot be nil"), + } + return errorCh + } + + go c.removeObjects(ctx, bucketName, objectsCh, errorCh, opts) + return errorCh +} + +// Generate and call MultiDelete S3 requests based on entries received from objectsCh +func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh <-chan string, errorCh chan<- RemoveObjectError, opts RemoveObjectsOptions) { + maxEntries := 1000 + finish := false + urlValues := make(url.Values) + urlValues.Set("delete", "") + + // Close error channel when Multi delete finishes. + defer close(errorCh) + + // Loop over entries by 1000 and call MultiDelete requests + for { + if finish { + break + } + count := 0 + var batch []string + + // Try to gather 1000 entries + for object := range objectsCh { + batch = append(batch, object) + if count++; count >= maxEntries { + break + } + } + if count == 0 { + // Multi Objects Delete API doesn't accept empty object list, quit immediately + break + } + if count < maxEntries { + // We didn't have 1000 entries, so this is the last batch + finish = true + } + + // Build headers. + headers := make(http.Header) + if opts.GovernanceBypass { + // Set the bypass goverenance retention header + headers.Set(amzBypassGovernance, "true") + } + + // Generate remove multi objects XML request + removeBytes := generateRemoveMultiObjectsRequest(batch) + // Execute GET on bucket to list objects. + resp, err := c.executeMethod(ctx, http.MethodPost, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: bytes.NewReader(removeBytes), + contentLength: int64(len(removeBytes)), + contentMD5Base64: sumMD5Base64(removeBytes), + contentSHA256Hex: sum256Hex(removeBytes), + customHeader: headers, + }) + if resp != nil { + if resp.StatusCode != http.StatusOK { + e := httpRespToErrorResponse(resp, bucketName, "") + errorCh <- RemoveObjectError{ObjectName: "", Err: e} + } + } + if err != nil { + for _, b := range batch { + errorCh <- RemoveObjectError{ObjectName: b, Err: err} + } + continue + } + + // Process multiobjects remove xml response + processRemoveMultiObjectsResponse(resp.Body, batch, errorCh) + + closeResponse(resp) + } +} + // RemoveObjects removes multiple objects from a bucket. // The list of objects to remove are received from objectsCh. // Remove failures are sent back via error channel. @@ -253,6 +289,18 @@ func (c Client) RemoveObjects(bucketName string, objectsCh <-chan string) <-chan return c.RemoveObjectsWithContext(context.Background(), bucketName, objectsCh) } +// RemoveObjectsOptions represents options specified by user for RemoveObjects call +type RemoveObjectsOptions struct { + GovernanceBypass bool +} + +// RemoveObjectsWithOptions removes multiple objects from a bucket. +// The list of objects to remove are received from objectsCh. +// Remove failures are sent back via error channel. +func (c Client) RemoveObjectsWithOptions(bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError { + return c.RemoveObjectsWithOptionsContext(context.Background(), bucketName, objectsCh, opts) +} + // RemoveIncompleteUpload aborts an partially uploaded object. func (c Client) RemoveIncompleteUpload(bucketName, objectName string) error { // Input validation. diff --git a/vendor/github.com/minio/minio-go/v6/api-stat.go b/vendor/github.com/minio/minio-go/v6/api-stat.go index 45844c9cb1..5da981b6c3 100644 --- a/vendor/github.com/minio/minio-go/v6/api-stat.go +++ b/vendor/github.com/minio/minio-go/v6/api-stat.go @@ -20,9 +20,6 @@ package minio import ( "context" "net/http" - "strconv" - "strings" - "time" "github.com/minio/minio-go/v6/pkg/s3utils" ) @@ -64,39 +61,6 @@ func (c Client) BucketExistsWithContext(ctx context.Context, bucketName string) return true, nil } -// List of header keys to be filtered, usually -// from all S3 API http responses. -var defaultFilterKeys = []string{ - "Connection", - "Transfer-Encoding", - "Accept-Ranges", - "Date", - "Server", - "Vary", - "x-amz-bucket-region", - "x-amz-request-id", - "x-amz-id-2", - "Content-Security-Policy", - "X-Xss-Protection", - amzTaggingHeaderCount, - // Add new headers to be ignored. -} - -// Extract only necessary metadata header key/values by -// filtering them out with a list of custom header keys. -func extractObjMetadata(header http.Header) http.Header { - filterKeys := append([]string{ - "ETag", - "Content-Length", - "Last-Modified", - "Content-Type", - "Expires", - "X-Cache", - "X-Cache-Lookup", - }, defaultFilterKeys...) - return filterHeader(header, filterKeys) -} - // StatObject verifies if object exists and you have permission to access. func (c Client) StatObject(bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { return c.StatObjectWithContext(context.Background(), bucketName, objectName, opts) @@ -142,76 +106,5 @@ func (c Client) statObject(ctx context.Context, bucketName, objectName string, o } } - // Trim off the odd double quotes from ETag in the beginning and end. - md5sum := trimEtag(resp.Header.Get("ETag")) - - // Parse content length is exists - var size int64 = -1 - contentLengthStr := resp.Header.Get("Content-Length") - if contentLengthStr != "" { - size, err = strconv.ParseInt(contentLengthStr, 10, 64) - if err != nil { - // Content-Length is not valid - return ObjectInfo{}, ErrorResponse{ - Code: "InternalError", - Message: "Content-Length is invalid. " + reportIssue, - BucketName: bucketName, - Key: objectName, - RequestID: resp.Header.Get("x-amz-request-id"), - HostID: resp.Header.Get("x-amz-id-2"), - Region: resp.Header.Get("x-amz-bucket-region"), - } - } - } - - // Parse Last-Modified has http time format. - date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified")) - if err != nil { - return ObjectInfo{}, ErrorResponse{ - Code: "InternalError", - Message: "Last-Modified time format is invalid. " + reportIssue, - BucketName: bucketName, - Key: objectName, - RequestID: resp.Header.Get("x-amz-request-id"), - HostID: resp.Header.Get("x-amz-id-2"), - Region: resp.Header.Get("x-amz-bucket-region"), - } - } - - // Fetch content type if any present. - contentType := strings.TrimSpace(resp.Header.Get("Content-Type")) - if contentType == "" { - contentType = "application/octet-stream" - } - - expiryStr := resp.Header.Get("Expires") - var expTime time.Time - if t, err := time.Parse(http.TimeFormat, expiryStr); err == nil { - expTime = t.UTC() - } - - metadata := extractObjMetadata(resp.Header) - userMetadata := map[string]string{} - const xamzmeta = "x-amz-meta-" - const xamzmetaLen = len(xamzmeta) - for k, v := range metadata { - if strings.HasPrefix(strings.ToLower(k), xamzmeta) { - userMetadata[k[xamzmetaLen:]] = v[0] - } - } - - // Save object metadata info. - return ObjectInfo{ - ETag: md5sum, - Key: objectName, - Size: size, - LastModified: date, - ContentType: contentType, - Expires: expTime, - // Extract only the relevant header keys describing the object. - // following function filters out a list of standard set of keys - // which are not part of object metadata. - Metadata: metadata, - UserMetadata: userMetadata, - }, nil + return ToObjectInfo(bucketName, objectName, resp.Header) } diff --git a/vendor/github.com/minio/minio-go/v6/api.go b/vendor/github.com/minio/minio-go/v6/api.go index 2354072901..cb094ea32b 100644 --- a/vendor/github.com/minio/minio-go/v6/api.go +++ b/vendor/github.com/minio/minio-go/v6/api.go @@ -43,8 +43,8 @@ import ( "golang.org/x/net/publicsuffix" "github.com/minio/minio-go/v6/pkg/credentials" - "github.com/minio/minio-go/v6/pkg/s3signer" "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/signer" ) // Client implements Amazon S3 compatible methods. @@ -104,7 +104,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.45" + libraryVersion = "v6.0.53" ) // User Agent should always following the below style. @@ -271,7 +271,7 @@ func (c *Client) redirectHeaders(req *http.Request, via []*http.Request) error { case signerType.IsV2(): return errors.New("signature V2 cannot support redirection") case signerType.IsV4(): - s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, getDefaultLocation(*c.endpointURL, region)) + signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, getDefaultLocation(*c.endpointURL, region)) } } return nil @@ -413,7 +413,7 @@ func (c *Client) SetS3TransferAccelerate(accelerateEndpoint string) { // - For signature v4 request if the connection is insecure compute only sha256. // - For signature v4 request if the connection is secure compute only md5. // - For anonymous request compute md5. -func (c *Client) hashMaterials() (hashAlgos map[string]hash.Hash, hashSums map[string][]byte) { +func (c *Client) hashMaterials(isMd5Requested bool) (hashAlgos map[string]hash.Hash, hashSums map[string][]byte) { hashSums = make(map[string][]byte) hashAlgos = make(map[string]hash.Hash) if c.overrideSignerType.IsV4() { @@ -427,6 +427,9 @@ func (c *Client) hashMaterials() (hashAlgos map[string]hash.Hash, hashSums map[s hashAlgos["md5"] = md5.New() } } + if isMd5Requested { + hashAlgos["md5"] = md5.New() + } return hashAlgos, hashSums } @@ -588,16 +591,16 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque } } - // Create a done channel to control 'newRetryTimer' go routine. - doneCh := make(chan struct{}, 1) + // Create cancel context to control 'newRetryTimer' go routine. + retryCtx, cancel := context.WithCancel(ctx) // Indicate to our routine to exit cleanly upon return. - defer close(doneCh) + defer cancel() // Blank indentifier is kept here on purpose since 'range' without // blank identifiers is only supported since go1.4 // https://golang.org/doc/go1.4#forrange. - for range c.newRetryTimer(reqRetry, DefaultRetryUnit, DefaultRetryCap, MaxJitter, doneCh) { + for range c.newRetryTimer(retryCtx, reqRetry, DefaultRetryUnit, DefaultRetryCap, MaxJitter) { // Retry executes the following function body if request has an // error until maxRetries have been exhausted, retry attempts are // performed after waiting for a given period of time in a @@ -627,6 +630,9 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque // Initiate the request. res, err = c.do(req) if err != nil { + if err == context.Canceled || err == context.DeadlineExceeded { + return nil, err + } continue } @@ -669,20 +675,23 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque case "InvalidRegion": fallthrough case "AccessDenied": - if metadata.bucketName != "" && errResponse.Region != "" { + if errResponse.Region == "" { + // Region is empty we simply return the error. + return res, err + } + // Region is not empty figure out a way to + // handle this appropriately. + if metadata.bucketName != "" { // Gather Cached location only if bucketName is present. if _, cachedOk := c.bucketLocCache.Get(metadata.bucketName); cachedOk { c.bucketLocCache.Set(metadata.bucketName, errResponse.Region) continue // Retry. } } else { - // Most probably for ListBuckets() + // This is for ListBuckets() fallback. if errResponse.Region != metadata.bucketLocation { - // Retry if the error - // response has a - // different region - // than the request we - // just made. + // Retry if the error response has a different region + // than the request we just made. metadata.bucketLocation = errResponse.Region continue // Retry } @@ -703,6 +712,12 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque // For all other cases break out of the retry loop. break } + + // Return an error when retry is canceled or deadlined + if e := retryCtx.Err(); e != nil { + return nil, e + } + return res, err } @@ -777,10 +792,10 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R } if signerType.IsV2() { // Presign URL with signature v2. - req = s3signer.PreSignV2(*req, accessKeyID, secretAccessKey, metadata.expires, isVirtualHost) + req = signer.PreSignV2(*req, accessKeyID, secretAccessKey, metadata.expires, isVirtualHost) } else if signerType.IsV4() { // Presign URL with signature v4. - req = s3signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires) + req = signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires) } return req, nil } @@ -823,12 +838,12 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R switch { case signerType.IsV2(): // Add signature version '2' authorization header. - req = s3signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualHost) + req = signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualHost) case metadata.objectName != "" && metadata.queryValues == nil && method == "PUT" && metadata.customHeader.Get("X-Amz-Copy-Source") == "" && !c.secure: // Streaming signature is used by default for a PUT object request. Additionally we also // look if the initialized client is secure, if yes then we don't need to perform // streaming signature. - req = s3signer.StreamingSignV4(req, accessKeyID, + req = signer.StreamingSignV4(req, accessKeyID, secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC()) default: // Set sha256 sum for signature calculation only with signature version '4'. @@ -839,7 +854,7 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R req.Header.Set("X-Amz-Content-Sha256", shaHeader) // Add signature version '4' authorization header. - req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, location) + req = signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, location) } // Return request. diff --git a/vendor/github.com/minio/minio-go/v6/bucket-cache.go b/vendor/github.com/minio/minio-go/v6/bucket-cache.go index 5e46bf9a73..6727840ae2 100644 --- a/vendor/github.com/minio/minio-go/v6/bucket-cache.go +++ b/vendor/github.com/minio/minio-go/v6/bucket-cache.go @@ -25,8 +25,8 @@ import ( "sync" "github.com/minio/minio-go/v6/pkg/credentials" - "github.com/minio/minio-go/v6/pkg/s3signer" "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/signer" ) // bucketLocationCache - Provides simple mechanism to hold bucket @@ -183,11 +183,21 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro } } - targetURL.Path = path.Join(bucketName, "") + "/" - targetURL.RawQuery = urlValues.Encode() + isVirtualHost := s3utils.IsVirtualHostSupported(targetURL, bucketName) + + var urlStr string + + //only suport Aliyun OSS for virtual hosted path, compatible Amazon & Google Endpoint + if isVirtualHost && s3utils.IsAliyunOSSEndpoint(targetURL) { + urlStr = c.endpointURL.Scheme + "://" + bucketName + "." + targetURL.Host + "/?location" + } else { + targetURL.Path = path.Join(bucketName, "") + "/" + targetURL.RawQuery = urlValues.Encode() + urlStr = targetURL.String() + } // Get a new HTTP request for the method. - req, err := http.NewRequest("GET", targetURL.String(), nil) + req, err := http.NewRequest("GET", urlStr, nil) if err != nil { return nil, err } @@ -226,7 +236,7 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro if signerType.IsV2() { // Get Bucket Location calls should be always path style isVirtualHost := false - req = s3signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualHost) + req = signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualHost) return req, nil } @@ -237,6 +247,6 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro } req.Header.Set("X-Amz-Content-Sha256", contentSha256) - req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1") + req = signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1") return req, nil } diff --git a/vendor/github.com/minio/minio-go/v6/bucket-notification.go b/vendor/github.com/minio/minio-go/v6/bucket-notification.go index 4714eadad5..7503b01b9a 100644 --- a/vendor/github.com/minio/minio-go/v6/bucket-notification.go +++ b/vendor/github.com/minio/minio-go/v6/bucket-notification.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,8 @@ package minio import ( "encoding/xml" + "errors" + "fmt" "github.com/minio/minio-go/v6/pkg/set" ) @@ -81,7 +83,7 @@ func NewArn(partition, service, region, accountID, resource string) Arn { Resource: resource} } -// Return the string format of the ARN +// String returns the string format of the ARN func (arn Arn) String() string { return "arn:" + arn.Partition + ":" + arn.Service + ":" + arn.Region + ":" + arn.AccountID + ":" + arn.Resource } @@ -137,6 +139,62 @@ func (t *NotificationConfig) AddFilterPrefix(prefix string) { t.Filter.S3Key.FilterRules = append(t.Filter.S3Key.FilterRules, newFilterRule) } +// EqualNotificationEventTypeList tells whether a and b contain the same events +func EqualNotificationEventTypeList(a, b []NotificationEventType) bool { + if len(a) != len(b) { + return false + } + setA := set.NewStringSet() + for _, i := range a { + setA.Add(string(i)) + } + + setB := set.NewStringSet() + for _, i := range b { + setB.Add(string(i)) + } + + return setA.Difference(setB).IsEmpty() +} + +// EqualFilterRuleList tells whether a and b contain the same filters +func EqualFilterRuleList(a, b []FilterRule) bool { + if len(a) != len(b) { + return false + } + + setA := set.NewStringSet() + for _, i := range a { + setA.Add(fmt.Sprintf("%s-%s", i.Name, i.Value)) + } + + setB := set.NewStringSet() + for _, i := range b { + setB.Add(fmt.Sprintf("%s-%s", i.Name, i.Value)) + } + + return setA.Difference(setB).IsEmpty() +} + +// Equal returns whether this `NotificationConfig` is equal to another defined by the passed parameters +func (t *NotificationConfig) Equal(events []NotificationEventType, prefix, suffix string) bool { + //Compare events + passEvents := EqualNotificationEventTypeList(t.Events, events) + + //Compare filters + var newFilter []FilterRule + if prefix != "" { + newFilter = append(newFilter, FilterRule{Name: "prefix", Value: prefix}) + } + if suffix != "" { + newFilter = append(newFilter, FilterRule{Name: "suffix", Value: suffix}) + } + + passFilters := EqualFilterRuleList(t.Filter.S3Key.FilterRules, newFilter) + // if it matches events and filters, mark the index for deletion + return passEvents && passFilters +} + // TopicConfig carries one single topic notification configuration type TopicConfig struct { NotificationConfig @@ -250,6 +308,26 @@ func (b *BucketNotification) RemoveTopicByArn(arn Arn) { b.TopicConfigs = topics } +// ErrNoNotificationConfigMatch is returned when a notification configuration (sqs,sns,lambda) is not found when trying to delete +var ErrNoNotificationConfigMatch = errors.New("no notification configuration matched") + +// RemoveTopicByArnEventsPrefixSuffix removes a topic configuration that match the exact specified ARN, events, prefix and suffix +func (b *BucketNotification) RemoveTopicByArnEventsPrefixSuffix(arn Arn, events []NotificationEventType, prefix, suffix string) error { + removeIndex := -1 + for i, v := range b.TopicConfigs { + // if it matches events and filters, mark the index for deletion + if v.Topic == arn.String() && v.NotificationConfig.Equal(events, prefix, suffix) { + removeIndex = i + break // since we have at most one matching config + } + } + if removeIndex >= 0 { + b.TopicConfigs = append(b.TopicConfigs[:removeIndex], b.TopicConfigs[removeIndex+1:]...) + return nil + } + return ErrNoNotificationConfigMatch +} + // RemoveQueueByArn removes all queue configurations that match the exact specified ARN func (b *BucketNotification) RemoveQueueByArn(arn Arn) { var queues []QueueConfig @@ -261,6 +339,23 @@ func (b *BucketNotification) RemoveQueueByArn(arn Arn) { b.QueueConfigs = queues } +// RemoveQueueByArnEventsPrefixSuffix removes a queue configuration that match the exact specified ARN, events, prefix and suffix +func (b *BucketNotification) RemoveQueueByArnEventsPrefixSuffix(arn Arn, events []NotificationEventType, prefix, suffix string) error { + removeIndex := -1 + for i, v := range b.QueueConfigs { + // if it matches events and filters, mark the index for deletion + if v.Queue == arn.String() && v.NotificationConfig.Equal(events, prefix, suffix) { + removeIndex = i + break // since we have at most one matching config + } + } + if removeIndex >= 0 { + b.QueueConfigs = append(b.QueueConfigs[:removeIndex], b.QueueConfigs[removeIndex+1:]...) + return nil + } + return ErrNoNotificationConfigMatch +} + // RemoveLambdaByArn removes all lambda configurations that match the exact specified ARN func (b *BucketNotification) RemoveLambdaByArn(arn Arn) { var lambdas []LambdaConfig @@ -271,3 +366,20 @@ func (b *BucketNotification) RemoveLambdaByArn(arn Arn) { } b.LambdaConfigs = lambdas } + +// RemoveLambdaByArnEventsPrefixSuffix removes a topic configuration that match the exact specified ARN, events, prefix and suffix +func (b *BucketNotification) RemoveLambdaByArnEventsPrefixSuffix(arn Arn, events []NotificationEventType, prefix, suffix string) error { + removeIndex := -1 + for i, v := range b.LambdaConfigs { + // if it matches events and filters, mark the index for deletion + if v.Lambda == arn.String() && v.NotificationConfig.Equal(events, prefix, suffix) { + removeIndex = i + break // since we have at most one matching config + } + } + if removeIndex >= 0 { + b.LambdaConfigs = append(b.LambdaConfigs[:removeIndex], b.LambdaConfigs[removeIndex+1:]...) + return nil + } + return ErrNoNotificationConfigMatch +} diff --git a/vendor/github.com/minio/minio-go/v6/constants.go b/vendor/github.com/minio/minio-go/v6/constants.go index 7f907c8b2c..78ba5a114e 100644 --- a/vendor/github.com/minio/minio-go/v6/constants.go +++ b/vendor/github.com/minio/minio-go/v6/constants.go @@ -55,13 +55,22 @@ const ( iso8601DateFormat = "20060102T150405Z" ) -// Storage class header constant. -const amzStorageClass = "X-Amz-Storage-Class" +const ( + // Storage class header. + amzStorageClass = "X-Amz-Storage-Class" -// Website redirect location header constant -const amzWebsiteRedirectLocation = "X-Amz-Website-Redirect-Location" + // Website redirect location header + amzWebsiteRedirectLocation = "X-Amz-Website-Redirect-Location" -// Tagging header constants -const amzTaggingHeader = "X-Amz-Tagging" -const amzTaggingHeaderDirective = "X-Amz-Tagging-Directive" -const amzTaggingHeaderCount = "X-Amz-Tagging-Count" + // Object Tagging headers + amzTaggingHeader = "X-Amz-Tagging" + amzTaggingHeaderDirective = "X-Amz-Tagging-Directive" + + // Object legal hold header + amzLegalHoldHeader = "X-Amz-Object-Lock-Legal-Hold" + + // Object retention header + amzLockMode = "X-Amz-Object-Lock-Mode" + amzLockRetainUntil = "X-Amz-Object-Lock-Retain-Until-Date" + amzBypassGovernance = "X-Amz-Bypass-Governance-Retention" +) diff --git a/vendor/github.com/minio/minio-go/v6/go.mod b/vendor/github.com/minio/minio-go/v6/go.mod index 2dc189d1aa..fe6d38bc6f 100644 --- a/vendor/github.com/minio/minio-go/v6/go.mod +++ b/vendor/github.com/minio/minio-go/v6/go.mod @@ -6,7 +6,7 @@ require ( github.com/dustin/go-humanize v1.0.0 // indirect github.com/minio/sha256-simd v0.1.1 github.com/mitchellh/go-homedir v1.1.0 - github.com/sirupsen/logrus v1.4.2 // indirect + github.com/sirupsen/logrus v1.5.0 // indirect github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f golang.org/x/net v0.0.0-20190522155817-f3200d17e092 diff --git a/vendor/github.com/minio/minio-go/v6/go.sum b/vendor/github.com/minio/minio-go/v6/go.sum index cd02277ed3..57de3a2b95 100644 --- a/vendor/github.com/minio/minio-go/v6/go.sum +++ b/vendor/github.com/minio/minio-go/v6/go.sum @@ -1,4 +1,3 @@ -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -11,16 +10,13 @@ github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKU github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= +github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= diff --git a/vendor/github.com/minio/minio-go/v6/pkg/credentials/assume_role.go b/vendor/github.com/minio/minio-go/v6/pkg/credentials/assume_role.go new file mode 100644 index 0000000000..f3702d8d67 --- /dev/null +++ b/vendor/github.com/minio/minio-go/v6/pkg/credentials/assume_role.go @@ -0,0 +1,214 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "encoding/hex" + "encoding/xml" + "errors" + "io" + "io/ioutil" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/minio/minio-go/v6/pkg/signer" + sha256 "github.com/minio/sha256-simd" +) + +// AssumeRoleResponse contains the result of successful AssumeRole request. +type AssumeRoleResponse struct { + XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleResponse" json:"-"` + + Result AssumeRoleResult `xml:"AssumeRoleResult"` + ResponseMetadata struct { + RequestID string `xml:"RequestId,omitempty"` + } `xml:"ResponseMetadata,omitempty"` +} + +// AssumeRoleResult - Contains the response to a successful AssumeRole +// request, including temporary credentials that can be used to make +// MinIO API requests. +type AssumeRoleResult struct { + // The identifiers for the temporary security credentials that the operation + // returns. + AssumedRoleUser AssumedRoleUser `xml:",omitempty"` + + // The temporary security credentials, which include an access key ID, a secret + // access key, and a security (or session) token. + // + // Note: The size of the security token that STS APIs return is not fixed. We + // strongly recommend that you make no assumptions about the maximum size. As + // of this writing, the typical size is less than 4096 bytes, but that can vary. + // Also, future updates to AWS might require larger sizes. + Credentials struct { + AccessKey string `xml:"AccessKeyId" json:"accessKey,omitempty"` + SecretKey string `xml:"SecretAccessKey" json:"secretKey,omitempty"` + Expiration time.Time `xml:"Expiration" json:"expiration,omitempty"` + SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty"` + } `xml:",omitempty"` + + // A percentage value that indicates the size of the policy in packed form. + // The service rejects any policy with a packed size greater than 100 percent, + // which means the policy exceeded the allowed space. + PackedPolicySize int `xml:",omitempty"` +} + +// A STSAssumeRole retrieves credentials from MinIO service, and keeps track if +// those credentials are expired. +type STSAssumeRole struct { + Expiry + + // Required http Client to use when connecting to MinIO STS service. + Client *http.Client + + // STS endpoint to fetch STS credentials. + STSEndpoint string + + // various options for this request. + Options STSAssumeRoleOptions +} + +// STSAssumeRoleOptions collection of various input options +// to obtain AssumeRole credentials. +type STSAssumeRoleOptions struct { + // Mandatory inputs. + AccessKey string + SecretKey string + + Location string // Optional commonly needed with AWS STS. + DurationSeconds int // Optional defaults to 1 hour. + + // Optional only valid if using with AWS STS + RoleARN string + RoleSessionName string +} + +// NewSTSAssumeRole returns a pointer to a new +// Credentials object wrapping the STSAssumeRole. +func NewSTSAssumeRole(stsEndpoint string, opts STSAssumeRoleOptions) (*Credentials, error) { + if stsEndpoint == "" { + return nil, errors.New("STS endpoint cannot be empty") + } + if opts.AccessKey == "" || opts.SecretKey == "" { + return nil, errors.New("AssumeRole credentials access/secretkey is mandatory") + } + return New(&STSAssumeRole{ + Client: &http.Client{ + Transport: http.DefaultTransport, + }, + STSEndpoint: stsEndpoint, + Options: opts, + }), nil +} + +const defaultDurationSeconds = 3600 + +// closeResponse close non nil response with any response Body. +// convenient wrapper to drain any remaining data on response body. +// +// Subsequently this allows golang http RoundTripper +// to re-use the same connection for future requests. +func closeResponse(resp *http.Response) { + // Callers should close resp.Body when done reading from it. + // If resp.Body is not closed, the Client's underlying RoundTripper + // (typically Transport) may not be able to re-use a persistent TCP + // connection to the server for a subsequent "keep-alive" request. + if resp != nil && resp.Body != nil { + // Drain any remaining Body and then close the connection. + // Without this closing connection would disallow re-using + // the same connection for future uses. + // - http://stackoverflow.com/a/17961593/4465767 + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + } +} + +func getAssumeRoleCredentials(clnt *http.Client, endpoint string, opts STSAssumeRoleOptions) (AssumeRoleResponse, error) { + v := url.Values{} + v.Set("Action", "AssumeRole") + v.Set("Version", "2011-06-15") + if opts.RoleARN != "" { + v.Set("RoleArn", opts.RoleARN) + } + if opts.RoleSessionName != "" { + v.Set("RoleSessionName", opts.RoleSessionName) + } + if opts.DurationSeconds > defaultDurationSeconds { + v.Set("DurationSeconds", strconv.Itoa(opts.DurationSeconds)) + } else { + v.Set("DurationSeconds", strconv.Itoa(defaultDurationSeconds)) + } + + u, err := url.Parse(endpoint) + if err != nil { + return AssumeRoleResponse{}, err + } + u.Path = "/" + + postBody := strings.NewReader(v.Encode()) + hash := sha256.New() + if _, err = io.Copy(hash, postBody); err != nil { + return AssumeRoleResponse{}, err + } + postBody.Seek(0, 0) + + req, err := http.NewRequest("POST", u.String(), postBody) + if err != nil { + return AssumeRoleResponse{}, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(hash.Sum(nil))) + req = signer.SignV4STS(*req, opts.AccessKey, opts.SecretKey, opts.Location) + + resp, err := clnt.Do(req) + if err != nil { + return AssumeRoleResponse{}, err + } + defer closeResponse(resp) + if resp.StatusCode != http.StatusOK { + return AssumeRoleResponse{}, errors.New(resp.Status) + } + + a := AssumeRoleResponse{} + if err = xml.NewDecoder(resp.Body).Decode(&a); err != nil { + return AssumeRoleResponse{}, err + } + return a, nil +} + +// Retrieve retrieves credentials from the MinIO service. +// Error will be returned if the request fails. +func (m *STSAssumeRole) Retrieve() (Value, error) { + a, err := getAssumeRoleCredentials(m.Client, m.STSEndpoint, m.Options) + if err != nil { + return Value{}, err + } + + // Expiry window is set to 10secs. + m.SetExpiration(a.Result.Credentials.Expiration, DefaultExpiryWindow) + + return Value{ + AccessKeyID: a.Result.Credentials.AccessKey, + SecretAccessKey: a.Result.Credentials.SecretKey, + SessionToken: a.Result.Credentials.SessionToken, + SignerType: SignatureV4, + }, nil +} diff --git a/vendor/github.com/minio/minio-go/v6/pkg/credentials/file_aws_credentials.go b/vendor/github.com/minio/minio-go/v6/pkg/credentials/file_aws_credentials.go index ff07bc55b9..ccc8251f4b 100644 --- a/vendor/github.com/minio/minio-go/v6/pkg/credentials/file_aws_credentials.go +++ b/vendor/github.com/minio/minio-go/v6/pkg/credentials/file_aws_credentials.go @@ -36,12 +36,12 @@ type FileAWSCredentials struct { // env value is empty will default to current user's home directory. // Linux/OSX: "$HOME/.aws/credentials" // Windows: "%USERPROFILE%\.aws\credentials" - filename string + Filename string // AWS Profile to extract credentials from the shared credentials file. If empty // will default to environment variable "AWS_PROFILE" or "default" if // environment variable is also not set. - profile string + Profile string // retrieved states if the credentials have been successfully retrieved. retrieved bool @@ -51,34 +51,34 @@ type FileAWSCredentials struct { // wrapping the Profile file provider. func NewFileAWSCredentials(filename string, profile string) *Credentials { return New(&FileAWSCredentials{ - filename: filename, - profile: profile, + Filename: filename, + Profile: profile, }) } // Retrieve reads and extracts the shared credentials from the current // users home directory. func (p *FileAWSCredentials) Retrieve() (Value, error) { - if p.filename == "" { - p.filename = os.Getenv("AWS_SHARED_CREDENTIALS_FILE") - if p.filename == "" { + if p.Filename == "" { + p.Filename = os.Getenv("AWS_SHARED_CREDENTIALS_FILE") + if p.Filename == "" { homeDir, err := homedir.Dir() if err != nil { return Value{}, err } - p.filename = filepath.Join(homeDir, ".aws", "credentials") + p.Filename = filepath.Join(homeDir, ".aws", "credentials") } } - if p.profile == "" { - p.profile = os.Getenv("AWS_PROFILE") - if p.profile == "" { - p.profile = "default" + if p.Profile == "" { + p.Profile = os.Getenv("AWS_PROFILE") + if p.Profile == "" { + p.Profile = "default" } } p.retrieved = false - iniProfile, err := loadProfile(p.filename, p.profile) + iniProfile, err := loadProfile(p.Filename, p.Profile) if err != nil { return Value{}, err } diff --git a/vendor/github.com/minio/minio-go/v6/pkg/credentials/file_minio_client.go b/vendor/github.com/minio/minio-go/v6/pkg/credentials/file_minio_client.go index 117ceb6efb..8adb5dcb12 100644 --- a/vendor/github.com/minio/minio-go/v6/pkg/credentials/file_minio_client.go +++ b/vendor/github.com/minio/minio-go/v6/pkg/credentials/file_minio_client.go @@ -38,12 +38,12 @@ type FileMinioClient struct { // env value is empty will default to current user's home directory. // Linux/OSX: "$HOME/.mc/config.json" // Windows: "%USERALIAS%\mc\config.json" - filename string + Filename string // MinIO Alias to extract credentials from the shared credentials file. If empty // will default to environment variable "MINIO_ALIAS" or "default" if // environment variable is also not set. - alias string + Alias string // retrieved states if the credentials have been successfully retrieved. retrieved bool @@ -53,39 +53,39 @@ type FileMinioClient struct { // wrapping the Alias file provider. func NewFileMinioClient(filename string, alias string) *Credentials { return New(&FileMinioClient{ - filename: filename, - alias: alias, + Filename: filename, + Alias: alias, }) } // Retrieve reads and extracts the shared credentials from the current // users home directory. func (p *FileMinioClient) Retrieve() (Value, error) { - if p.filename == "" { + if p.Filename == "" { if value, ok := os.LookupEnv("MINIO_SHARED_CREDENTIALS_FILE"); ok { - p.filename = value + p.Filename = value } else { homeDir, err := homedir.Dir() if err != nil { return Value{}, err } - p.filename = filepath.Join(homeDir, ".mc", "config.json") + p.Filename = filepath.Join(homeDir, ".mc", "config.json") if runtime.GOOS == "windows" { - p.filename = filepath.Join(homeDir, "mc", "config.json") + p.Filename = filepath.Join(homeDir, "mc", "config.json") } } } - if p.alias == "" { - p.alias = os.Getenv("MINIO_ALIAS") - if p.alias == "" { - p.alias = "s3" + if p.Alias == "" { + p.Alias = os.Getenv("MINIO_ALIAS") + if p.Alias == "" { + p.Alias = "s3" } } p.retrieved = false - hostCfg, err := loadAlias(p.filename, p.alias) + hostCfg, err := loadAlias(p.Filename, p.Alias) if err != nil { return Value{}, err } diff --git a/vendor/github.com/minio/minio-go/v6/pkg/credentials/iam_aws.go b/vendor/github.com/minio/minio-go/v6/pkg/credentials/iam_aws.go index 6c6be66593..60f78f8084 100644 --- a/vendor/github.com/minio/minio-go/v6/pkg/credentials/iam_aws.go +++ b/vendor/github.com/minio/minio-go/v6/pkg/credentials/iam_aws.go @@ -82,7 +82,7 @@ func (m *IAM) Retrieve() (Value, error) { case len(os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE")) > 0: if len(endpoint) == 0 { if len(os.Getenv("AWS_REGION")) > 0 { - endpoint = "sts." + os.Getenv("AWS_REGION") + ".amazonaws.com" + endpoint = "https://sts." + os.Getenv("AWS_REGION") + ".amazonaws.com" } else { endpoint = defaultSTSRoleEndpoint } @@ -168,6 +168,10 @@ type ec2RoleCredRespBody struct { // be sent to fetch the rolling access credentials. // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html func getIAMRoleURL(endpoint string) (*url.URL, error) { + if endpoint == "" { + endpoint = defaultIAMRoleEndpoint + } + u, err := url.Parse(endpoint) if err != nil { return nil, err diff --git a/vendor/github.com/minio/minio-go/v6/pkg/s3utils/utils.go b/vendor/github.com/minio/minio-go/v6/pkg/s3utils/utils.go index f1f6eea607..aa96a68fbd 100644 --- a/vendor/github.com/minio/minio-go/v6/pkg/s3utils/utils.go +++ b/vendor/github.com/minio/minio-go/v6/pkg/s3utils/utils.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -78,22 +78,28 @@ func IsVirtualHostSupported(endpointURL url.URL, bucketName string) bool { return false } // Return true for all other cases - return IsAmazonEndpoint(endpointURL) || IsGoogleEndpoint(endpointURL) + return IsAmazonEndpoint(endpointURL) || IsGoogleEndpoint(endpointURL) || IsAliyunOSSEndpoint(endpointURL) } // Refer for region styles - https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region // amazonS3HostHyphen - regular expression used to determine if an arg is s3 host in hyphenated style. -var amazonS3HostHyphen = regexp.MustCompile(`^s3-(.*?)\.amazonaws\.com$`) +var amazonS3HostHyphen = regexp.MustCompile(`^s3-(.*?).amazonaws.com$`) // amazonS3HostDualStack - regular expression used to determine if an arg is s3 host dualstack. -var amazonS3HostDualStack = regexp.MustCompile(`^s3\.dualstack\.(.*?)\.amazonaws\.com$`) +var amazonS3HostDualStack = regexp.MustCompile(`^s3.dualstack.(.*?).amazonaws.com$`) // amazonS3HostDot - regular expression used to determine if an arg is s3 host in . style. -var amazonS3HostDot = regexp.MustCompile(`^s3\.(.*?)\.amazonaws\.com$`) +var amazonS3HostDot = regexp.MustCompile(`^s3.(.*?).amazonaws.com$`) // amazonS3ChinaHost - regular expression used to determine if the arg is s3 china host. -var amazonS3ChinaHost = regexp.MustCompile(`^s3\.(cn.*?)\.amazonaws\.com\.cn$`) +var amazonS3ChinaHost = regexp.MustCompile(`^s3.(cn.*?).amazonaws.com.cn$`) + +// Regular expression used to determine if the arg is elb host. +var elbAmazonRegex = regexp.MustCompile(`elb(.*?).amazonaws.com$`) + +// Regular expression used to determine if the arg is elb host in china. +var elbAmazonCnRegex = regexp.MustCompile(`elb(.*?).amazonaws.com.cn$`) // GetRegionFromURL - returns a region from url host. func GetRegionFromURL(endpointURL url.URL) string { @@ -106,6 +112,10 @@ func GetRegionFromURL(endpointURL url.URL) string { if IsAmazonGovCloudEndpoint(endpointURL) { return "us-gov-west-1" } + // if elb's are used we cannot calculate which region it may be, just return empty. + if elbAmazonRegex.MatchString(endpointURL.Host) || elbAmazonCnRegex.MatchString(endpointURL.Host) { + return "" + } parts := amazonS3HostDualStack.FindStringSubmatch(endpointURL.Host) if len(parts) > 1 { return parts[1] @@ -125,6 +135,11 @@ func GetRegionFromURL(endpointURL url.URL) string { return "" } +// IsAliyunOSSEndpoint - Match if it is exactly Aliyun OSS endpoint. +func IsAliyunOSSEndpoint(endpointURL url.URL) bool { + return strings.HasSuffix(endpointURL.Host, "aliyuncs.com") +} + // IsAmazonEndpoint - Match if it is exactly Amazon S3 endpoint. func IsAmazonEndpoint(endpointURL url.URL) bool { if endpointURL.Host == "s3-external-1.amazonaws.com" || endpointURL.Host == "s3.amazonaws.com" { @@ -219,29 +234,39 @@ func QueryEncode(v url.Values) string { return buf.String() } +// TagDecode - decodes canonical tag into map of key and value. +func TagDecode(ctag string) map[string]string { + if ctag == "" { + return map[string]string{} + } + tags := strings.Split(ctag, "&") + tagMap := make(map[string]string, len(tags)) + var err error + for _, tag := range tags { + kvs := strings.SplitN(tag, "=", 2) + if len(kvs) == 0 { + return map[string]string{} + } + if len(kvs) == 1 { + return map[string]string{} + } + tagMap[kvs[0]], err = url.PathUnescape(kvs[1]) + if err != nil { + continue + } + } + return tagMap +} + // TagEncode - encodes tag values in their URL encoded form. In // addition to the percent encoding performed by urlEncodePath() used // here, it also percent encodes '/' (forward slash) func TagEncode(tags map[string]string) string { - if tags == nil { - return "" + values := url.Values{} + for k, v := range tags { + values[k] = []string{v} } - var buf bytes.Buffer - keys := make([]string, 0, len(tags)) - for k := range tags { - keys = append(keys, k) - } - sort.Strings(keys) - for _, k := range keys { - v := tags[k] - prefix := percentEncodeSlash(EncodePath(k)) + "=" - if buf.Len() > 0 { - buf.WriteByte('&') - } - buf.WriteString(prefix) - buf.WriteString(percentEncodeSlash(EncodePath(v))) - } - return buf.String() + return QueryEncode(values) } // if object matches reserved string, no need to encode them @@ -299,10 +324,10 @@ func checkBucketNameCommon(bucketName string, strict bool) (err error) { return errors.New("Bucket name cannot be empty") } if len(bucketName) < 3 { - return errors.New("Bucket name cannot be smaller than 3 characters") + return errors.New("Bucket name cannot be shorter than 3 characters") } if len(bucketName) > 63 { - return errors.New("Bucket name cannot be greater than 63 characters") + return errors.New("Bucket name cannot be longer than 63 characters") } if ipAddress.MatchString(bucketName) { return errors.New("Bucket name cannot be an ip address") @@ -338,7 +363,7 @@ func CheckValidBucketNameStrict(bucketName string) (err error) { // - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html func CheckValidObjectNamePrefix(objectName string) error { if len(objectName) > 1024 { - return errors.New("Object name cannot be greater than 1024 characters") + return errors.New("Object name cannot be longer than 1024 characters") } if !utf8.ValidString(objectName) { return errors.New("Object name with non UTF-8 strings are not supported") diff --git a/vendor/github.com/minio/minio-go/v6/pkg/s3signer/request-signature-streaming.go b/vendor/github.com/minio/minio-go/v6/pkg/signer/request-signature-streaming.go similarity index 97% rename from vendor/github.com/minio/minio-go/v6/pkg/s3signer/request-signature-streaming.go rename to vendor/github.com/minio/minio-go/v6/pkg/signer/request-signature-streaming.go index 810b47c4b3..7b2ca91d12 100644 --- a/vendor/github.com/minio/minio-go/v6/pkg/s3signer/request-signature-streaming.go +++ b/vendor/github.com/minio/minio-go/v6/pkg/signer/request-signature-streaming.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package s3signer +package signer import ( "bytes" @@ -82,7 +82,7 @@ func buildChunkStringToSign(t time.Time, region, previousSig string, chunkData [ stringToSignParts := []string{ streamingPayloadHdr, t.Format(iso8601DateFormat), - getScope(region, t), + getScope(region, t, ServiceTypeS3), previousSig, emptySHA256, hex.EncodeToString(sum256(chunkData)), @@ -118,19 +118,19 @@ func buildChunkSignature(chunkData []byte, reqTime time.Time, region, chunkStringToSign := buildChunkStringToSign(reqTime, region, previousSignature, chunkData) - signingKey := getSigningKey(secretAccessKey, region, reqTime) + signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3) return getSignature(signingKey, chunkStringToSign) } // getSeedSignature - returns the seed signature for a given request. func (s *StreamingReader) setSeedSignature(req *http.Request) { // Get canonical request - canonicalRequest := getCanonicalRequest(*req, ignoredStreamingHeaders) + canonicalRequest := getCanonicalRequest(*req, ignoredStreamingHeaders, getHashedPayload(*req)) // Get string to sign from canonical request. - stringToSign := getStringToSignV4(s.reqTime, s.region, canonicalRequest) + stringToSign := getStringToSignV4(s.reqTime, s.region, canonicalRequest, ServiceTypeS3) - signingKey := getSigningKey(s.secretAccessKey, s.region, s.reqTime) + signingKey := getSigningKey(s.secretAccessKey, s.region, s.reqTime, ServiceTypeS3) // Calculate signature. s.seedSignature = getSignature(signingKey, stringToSign) @@ -185,7 +185,7 @@ func (s *StreamingReader) signChunk(chunkLen int) { // setStreamingAuthHeader - builds and sets authorization header value // for streaming signature. func (s *StreamingReader) setStreamingAuthHeader(req *http.Request) { - credential := GetCredential(s.accessKeyID, s.region, s.reqTime) + credential := GetCredential(s.accessKeyID, s.region, s.reqTime, ServiceTypeS3) authParts := []string{ signV4Algorithm + " Credential=" + credential, "SignedHeaders=" + getSignedHeaders(*req, ignoredStreamingHeaders), diff --git a/vendor/github.com/minio/minio-go/v6/pkg/s3signer/request-signature-v2.go b/vendor/github.com/minio/minio-go/v6/pkg/signer/request-signature-v2.go similarity index 99% rename from vendor/github.com/minio/minio-go/v6/pkg/s3signer/request-signature-v2.go rename to vendor/github.com/minio/minio-go/v6/pkg/signer/request-signature-v2.go index 40ba07130a..3179d62b50 100644 --- a/vendor/github.com/minio/minio-go/v6/pkg/s3signer/request-signature-v2.go +++ b/vendor/github.com/minio/minio-go/v6/pkg/signer/request-signature-v2.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package s3signer +package signer import ( "bytes" diff --git a/vendor/github.com/minio/minio-go/v6/pkg/s3signer/request-signature-v4.go b/vendor/github.com/minio/minio-go/v6/pkg/signer/request-signature-v4.go similarity index 79% rename from vendor/github.com/minio/minio-go/v6/pkg/s3signer/request-signature-v4.go rename to vendor/github.com/minio/minio-go/v6/pkg/signer/request-signature-v4.go index ab96b58c52..0f6a4c0a6d 100644 --- a/vendor/github.com/minio/minio-go/v6/pkg/s3signer/request-signature-v4.go +++ b/vendor/github.com/minio/minio-go/v6/pkg/signer/request-signature-v4.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package s3signer +package signer import ( "bytes" @@ -36,6 +36,12 @@ const ( yyyymmdd = "20060102" ) +// Different service types +const ( + ServiceTypeS3 = "s3" + ServiceTypeSTS = "sts" +) + /// /// Excerpts from @lsegal - /// https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258. @@ -47,42 +53,21 @@ const ( /// by other agents) or when customers pass requests through /// proxies, which may modify the user-agent. /// -/// Content-Length: -/// -/// This is ignored from signing because generating a pre-signed -/// URL should not provide a content-length constraint, -/// specifically when vending a S3 pre-signed PUT URL. The -/// corollary to this is that when sending regular requests -/// (non-pre-signed), the signature contains a checksum of the -/// body, which implicitly validates the payload length (since -/// changing the number of bytes would change the checksum) -/// and therefore this header is not valuable in the signature. -/// -/// Content-Type: -/// -/// Signing this header causes quite a number of problems in -/// browser environments, where browsers like to modify and -/// normalize the content-type header in different ways. There is -/// more information on this in https://goo.gl/2E9gyy. Avoiding -/// this field simplifies logic and reduces the possibility of -/// future bugs. /// /// Authorization: /// /// Is skipped for obvious reasons /// var v4IgnoredHeaders = map[string]bool{ - "Authorization": true, - "Content-Type": true, - "Content-Length": true, - "User-Agent": true, + "Authorization": true, + "User-Agent": true, } // getSigningKey hmac seed to calculate final signature. -func getSigningKey(secret, loc string, t time.Time) []byte { +func getSigningKey(secret, loc string, t time.Time, serviceType string) []byte { date := sumHMAC([]byte("AWS4"+secret), []byte(t.Format(yyyymmdd))) location := sumHMAC(date, []byte(loc)) - service := sumHMAC(location, []byte("s3")) + service := sumHMAC(location, []byte(serviceType)) signingKey := sumHMAC(service, []byte("aws4_request")) return signingKey } @@ -94,19 +79,19 @@ func getSignature(signingKey []byte, stringToSign string) string { // getScope generate a string of a specific date, an AWS region, and a // service. -func getScope(location string, t time.Time) string { +func getScope(location string, t time.Time, serviceType string) string { scope := strings.Join([]string{ t.Format(yyyymmdd), location, - "s3", + serviceType, "aws4_request", }, "/") return scope } // GetCredential generate a credential string. -func GetCredential(accessKeyID, location string, t time.Time) string { - scope := getScope(location, t) +func GetCredential(accessKeyID, location string, t time.Time, serviceType string) string { + scope := getScope(location, t, serviceType) return accessKeyID + "/" + scope } @@ -184,7 +169,7 @@ func getSignedHeaders(req http.Request, ignoredHeaders map[string]bool) string { // \n // \n // -func getCanonicalRequest(req http.Request, ignoredHeaders map[string]bool) string { +func getCanonicalRequest(req http.Request, ignoredHeaders map[string]bool, hashedPayload string) string { req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1) canonicalRequest := strings.Join([]string{ req.Method, @@ -192,15 +177,15 @@ func getCanonicalRequest(req http.Request, ignoredHeaders map[string]bool) strin req.URL.RawQuery, getCanonicalHeaders(req, ignoredHeaders), getSignedHeaders(req, ignoredHeaders), - getHashedPayload(req), + hashedPayload, }, "\n") return canonicalRequest } // getStringToSign a string based on selected query values. -func getStringToSignV4(t time.Time, location, canonicalRequest string) string { +func getStringToSignV4(t time.Time, location, canonicalRequest, serviceType string) string { stringToSign := signV4Algorithm + "\n" + t.Format(iso8601DateFormat) + "\n" - stringToSign = stringToSign + getScope(location, t) + "\n" + stringToSign = stringToSign + getScope(location, t, serviceType) + "\n" stringToSign = stringToSign + hex.EncodeToString(sum256([]byte(canonicalRequest))) return stringToSign } @@ -217,7 +202,7 @@ func PreSignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, loc t := time.Now().UTC() // Get credential string. - credential := GetCredential(accessKeyID, location, t) + credential := GetCredential(accessKeyID, location, t, ServiceTypeS3) // Get all signed headers. signedHeaders := getSignedHeaders(req, v4IgnoredHeaders) @@ -236,13 +221,13 @@ func PreSignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, loc req.URL.RawQuery = query.Encode() // Get canonical request. - canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders) + canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders, getHashedPayload(req)) // Get string to sign from canonical request. - stringToSign := getStringToSignV4(t, location, canonicalRequest) + stringToSign := getStringToSignV4(t, location, canonicalRequest, ServiceTypeS3) // Gext hmac signing key. - signingKey := getSigningKey(secretAccessKey, location, t) + signingKey := getSigningKey(secretAccessKey, location, t, ServiceTypeS3) // Calculate signature. signature := getSignature(signingKey, stringToSign) @@ -257,15 +242,19 @@ func PreSignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, loc // requests. func PostPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string { // Get signining key. - signingkey := getSigningKey(secretAccessKey, location, t) + signingkey := getSigningKey(secretAccessKey, location, t, ServiceTypeS3) // Calculate signature. signature := getSignature(signingkey, policyBase64) return signature } -// SignV4 sign the request before Do(), in accordance with -// http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html. -func SignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string) *http.Request { +// SignV4STS - signature v4 for STS request. +func SignV4STS(req http.Request, accessKeyID, secretAccessKey, location string) *http.Request { + return signV4(req, accessKeyID, secretAccessKey, "", location, ServiceTypeSTS) +} + +// Internal function called for different service types. +func signV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location, serviceType string) *http.Request { // Signature calculation is not needed for anonymous credentials. if accessKeyID == "" || secretAccessKey == "" { return &req @@ -282,17 +271,25 @@ func SignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, locati req.Header.Set("X-Amz-Security-Token", sessionToken) } + hashedPayload := getHashedPayload(req) + if serviceType == ServiceTypeSTS { + // Content sha256 header is not sent with the request + // but it is expected to have sha256 of payload for signature + // in STS service type request. + req.Header.Del("X-Amz-Content-Sha256") + } + // Get canonical request. - canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders) + canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders, hashedPayload) // Get string to sign from canonical request. - stringToSign := getStringToSignV4(t, location, canonicalRequest) + stringToSign := getStringToSignV4(t, location, canonicalRequest, serviceType) // Get hmac signing key. - signingKey := getSigningKey(secretAccessKey, location, t) + signingKey := getSigningKey(secretAccessKey, location, t, serviceType) // Get credential string. - credential := GetCredential(accessKeyID, location, t) + credential := GetCredential(accessKeyID, location, t, serviceType) // Get all signed headers. signedHeaders := getSignedHeaders(req, v4IgnoredHeaders) @@ -313,3 +310,9 @@ func SignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, locati return &req } + +// SignV4 sign the request before Do(), in accordance with +// http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html. +func SignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string) *http.Request { + return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3) +} diff --git a/vendor/github.com/minio/minio-go/v6/pkg/s3signer/utils.go b/vendor/github.com/minio/minio-go/v6/pkg/signer/utils.go similarity index 99% rename from vendor/github.com/minio/minio-go/v6/pkg/s3signer/utils.go rename to vendor/github.com/minio/minio-go/v6/pkg/signer/utils.go index 934e33a489..2192a36933 100644 --- a/vendor/github.com/minio/minio-go/v6/pkg/s3signer/utils.go +++ b/vendor/github.com/minio/minio-go/v6/pkg/signer/utils.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package s3signer +package signer import ( "crypto/hmac" diff --git a/vendor/github.com/minio/minio-go/v6/post-policy.go b/vendor/github.com/minio/minio-go/v6/post-policy.go index 8b0bf092df..017916f062 100644 --- a/vendor/github.com/minio/minio-go/v6/post-policy.go +++ b/vendor/github.com/minio/minio-go/v6/post-policy.go @@ -258,7 +258,7 @@ func (p *PostPolicy) addNewPolicy(policyCond policyCondition) error { return nil } -// Stringer interface for printing policy in json formatted string. +// String function for printing policy in json formatted string. func (p PostPolicy) String() string { return string(p.marshalJSON()) } diff --git a/vendor/github.com/minio/minio-go/v6/retry.go b/vendor/github.com/minio/minio-go/v6/retry.go index f9c1095c05..6e2826fa65 100644 --- a/vendor/github.com/minio/minio-go/v6/retry.go +++ b/vendor/github.com/minio/minio-go/v6/retry.go @@ -18,6 +18,7 @@ package minio import ( + "context" "net/http" "time" ) @@ -41,7 +42,7 @@ const DefaultRetryCap = time.Second * 30 // newRetryTimer creates a timer with exponentially increasing // delays until the maximum retry attempts are reached. -func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duration, jitter float64, doneCh chan struct{}) <-chan int { +func (c Client) newRetryTimer(ctx context.Context, maxRetry int, unit time.Duration, cap time.Duration, jitter float64) <-chan int { attemptCh := make(chan int) // computes the exponential backoff duration according to @@ -70,13 +71,16 @@ func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duratio defer close(attemptCh) for i := 0; i < maxRetry; i++ { select { - // Attempts start from 1. case attemptCh <- i + 1: - case <-doneCh: - // Stop the routine. + case <-ctx.Done(): + return + } + + select { + case <-time.After(exponentialBackoffWait(i)): + case <-ctx.Done(): return } - time.Sleep(exponentialBackoffWait(i)) } }() return attemptCh diff --git a/vendor/github.com/minio/minio-go/v6/s3-endpoints.go b/vendor/github.com/minio/minio-go/v6/s3-endpoints.go index 989f58c7ec..4a8879fd53 100644 --- a/vendor/github.com/minio/minio-go/v6/s3-endpoints.go +++ b/vendor/github.com/minio/minio-go/v6/s3-endpoints.go @@ -35,6 +35,8 @@ var awsS3EndpointMap = map[string]string{ "ap-southeast-2": "s3.dualstack.ap-southeast-2.amazonaws.com", "ap-northeast-1": "s3.dualstack.ap-northeast-1.amazonaws.com", "ap-northeast-2": "s3.dualstack.ap-northeast-2.amazonaws.com", + "ap-northeast-3": "s3.dualstack.ap-northeast-3.amazonaws.com", + "me-south-1": "s3.dualstack.me-south-1.amazonaws.com", "sa-east-1": "s3.dualstack.sa-east-1.amazonaws.com", "us-gov-west-1": "s3.dualstack.us-gov-west-1.amazonaws.com", "us-gov-east-1": "s3.dualstack.us-gov-east-1.amazonaws.com", diff --git a/vendor/github.com/minio/minio-go/v6/transport.go b/vendor/github.com/minio/minio-go/v6/transport.go index 00ee4dad67..fb624854ba 100644 --- a/vendor/github.com/minio/minio-go/v6/transport.go +++ b/vendor/github.com/minio/minio-go/v6/transport.go @@ -38,7 +38,8 @@ var DefaultTransport = func(secure bool) (http.RoundTripper, error) { }).DialContext, MaxIdleConns: 1024, MaxIdleConnsPerHost: 1024, - IdleConnTimeout: 90 * time.Second, + ResponseHeaderTimeout: 60 * time.Second, + IdleConnTimeout: 60 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, // Set this value so that the underlying transport round-tripper diff --git a/vendor/github.com/minio/minio-go/v6/utils.go b/vendor/github.com/minio/minio-go/v6/utils.go index 5b324547df..685e98c70e 100644 --- a/vendor/github.com/minio/minio-go/v6/utils.go +++ b/vendor/github.com/minio/minio-go/v6/utils.go @@ -28,6 +28,7 @@ import ( "net/http" "net/url" "regexp" + "strconv" "strings" "time" @@ -159,29 +160,122 @@ func isValidExpiry(expires time.Duration) error { return nil } -// make a copy of http.Header -func cloneHeader(h http.Header) http.Header { - h2 := make(http.Header, len(h)) - for k, vv := range h { - vv2 := make([]string, len(vv)) - copy(vv2, vv) - h2[k] = vv2 +// Extract only necessary metadata header key/values by +// filtering them out with a list of custom header keys. +func extractObjMetadata(header http.Header) http.Header { + preserveKeys := []string{ + "Content-Type", + "Cache-Control", + "Content-Encoding", + "Content-Language", + "Content-Disposition", + "X-Amz-Storage-Class", + "X-Amz-Object-Lock-Mode", + "X-Amz-Object-Lock-Retain-Until-Date", + "X-Amz-Object-Lock-Legal-Hold", + "X-Amz-Website-Redirect-Location", + "X-Amz-Server-Side-Encryption", + "X-Amz-Meta-", + // Add new headers to be preserved. + // if you add new headers here, please extend + // PutObjectOptions{} to preserve them + // upon upload as well. } - return h2 -} - -// Filter relevant response headers from -// the HEAD, GET http response. The function takes -// a list of headers which are filtered out and -// returned as a new http header. -func filterHeader(header http.Header, filterKeys []string) (filteredHeader http.Header) { - filteredHeader = cloneHeader(header) - for _, key := range filterKeys { - filteredHeader.Del(key) + filteredHeader := make(http.Header) + for k, v := range header { + var found bool + for _, prefix := range preserveKeys { + if !strings.HasPrefix(k, prefix) { + continue + } + found = true + break + } + if found { + filteredHeader[k] = v + } } return filteredHeader } +// ToObjectInfo converts http header values into ObjectInfo type, +// extracts metadata and fills in all the necessary fields in ObjectInfo. +func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectInfo, error) { + var err error + // Trim off the odd double quotes from ETag in the beginning and end. + etag := trimEtag(h.Get("ETag")) + + // Parse content length is exists + var size int64 = -1 + contentLengthStr := h.Get("Content-Length") + if contentLengthStr != "" { + size, err = strconv.ParseInt(contentLengthStr, 10, 64) + if err != nil { + // Content-Length is not valid + return ObjectInfo{}, ErrorResponse{ + Code: "InternalError", + Message: "Content-Length is invalid. " + reportIssue, + BucketName: bucketName, + Key: objectName, + RequestID: h.Get("x-amz-request-id"), + HostID: h.Get("x-amz-id-2"), + Region: h.Get("x-amz-bucket-region"), + } + } + } + + // Parse Last-Modified has http time format. + date, err := time.Parse(http.TimeFormat, h.Get("Last-Modified")) + if err != nil { + return ObjectInfo{}, ErrorResponse{ + Code: "InternalError", + Message: "Last-Modified time format is invalid. " + reportIssue, + BucketName: bucketName, + Key: objectName, + RequestID: h.Get("x-amz-request-id"), + HostID: h.Get("x-amz-id-2"), + Region: h.Get("x-amz-bucket-region"), + } + } + + // Fetch content type if any present. + contentType := strings.TrimSpace(h.Get("Content-Type")) + if contentType == "" { + contentType = "application/octet-stream" + } + + expiryStr := h.Get("Expires") + var expTime time.Time + if t, err := time.Parse(http.TimeFormat, expiryStr); err == nil { + expTime = t.UTC() + } + + metadata := extractObjMetadata(h) + userMetadata := make(map[string]string) + for k, v := range metadata { + if strings.HasPrefix(k, "X-Amz-Meta-") { + userMetadata[strings.TrimPrefix(k, "X-Amz-Meta-")] = v[0] + } + } + userTags := s3utils.TagDecode(h.Get(amzTaggingHeader)) + + // Save object metadata info. + return ObjectInfo{ + ETag: etag, + Key: objectName, + Size: size, + LastModified: date, + ContentType: contentType, + Expires: expTime, + // Extract only the relevant header keys describing the object. + // following function filters out a list of standard set of keys + // which are not part of object metadata. + Metadata: metadata, + UserMetadata: userMetadata, + UserTags: userTags, + }, nil +} + // regCred matches credential string in HTTP header var regCred = regexp.MustCompile("Credential=([A-Z0-9]+)/") diff --git a/vendor/github.com/sirupsen/logrus/.golangci.yml b/vendor/github.com/sirupsen/logrus/.golangci.yml new file mode 100644 index 0000000000..65dc285037 --- /dev/null +++ b/vendor/github.com/sirupsen/logrus/.golangci.yml @@ -0,0 +1,40 @@ +run: + # do not run on test files yet + tests: false + +# all available settings of specific linters +linters-settings: + errcheck: + # report about not checking of errors in type assetions: `a := b.(MyStruct)`; + # default is false: such cases aren't reported by default. + check-type-assertions: false + + # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; + # default is false: such cases aren't reported by default. + check-blank: false + + lll: + line-length: 100 + tab-width: 4 + + prealloc: + simple: false + range-loops: false + for-loops: false + + whitespace: + multi-if: false # Enforces newlines (or comments) after every multi-line if statement + multi-func: false # Enforces newlines (or comments) after every multi-line function signature + +linters: + enable: + - megacheck + - govet + disable: + - maligned + - prealloc + disable-all: false + presets: + - bugs + - unused + fast: false diff --git a/vendor/github.com/sirupsen/logrus/.travis.yml b/vendor/github.com/sirupsen/logrus/.travis.yml index 848938a6d4..5e20aa4140 100644 --- a/vendor/github.com/sirupsen/logrus/.travis.yml +++ b/vendor/github.com/sirupsen/logrus/.travis.yml @@ -4,21 +4,13 @@ git: depth: 1 env: - GO111MODULE=on - - GO111MODULE=off -go: [ 1.11.x, 1.12.x ] -os: [ linux, osx ] -matrix: - exclude: - - go: 1.12.x - env: GO111MODULE=off - - go: 1.11.x - os: osx +go: [1.13.x, 1.14.x] +os: [linux, osx] install: - ./travis/install.sh - - if [[ "$GO111MODULE" == "on" ]]; then go mod download; fi - - if [[ "$GO111MODULE" == "off" ]]; then go get github.com/stretchr/testify/assert golang.org/x/sys/unix github.com/konsorten/go-windows-terminal-sequences; fi script: - ./travis/cross_build.sh + - ./travis/lint.sh - export GOMAXPROCS=4 - export GORACE=halt_on_error=1 - go test -race -v ./... diff --git a/vendor/github.com/sirupsen/logrus/README.md b/vendor/github.com/sirupsen/logrus/README.md index a4796eb07d..5796706dbf 100644 --- a/vendor/github.com/sirupsen/logrus/README.md +++ b/vendor/github.com/sirupsen/logrus/README.md @@ -1,8 +1,28 @@ -# Logrus :walrus: [![Build Status](https://travis-ci.org/sirupsen/logrus.svg?branch=master)](https://travis-ci.org/sirupsen/logrus) [![GoDoc](https://godoc.org/github.com/sirupsen/logrus?status.svg)](https://godoc.org/github.com/sirupsen/logrus) +# Logrus :walrus: [![Build Status](https://travis-ci.org/sirupsen/logrus.svg?branch=master)](https://travis-ci.org/sirupsen/logrus) [![GoDoc](https://godoc.org/github.com/sirupsen/logrus?status.svg)](https://godoc.org/github.com/sirupsen/logrus) Logrus is a structured logger for Go (golang), completely API compatible with the standard library logger. +**Logrus is in maintenance-mode.** We will not be introducing new features. It's +simply too hard to do in a way that won't break many people's projects, which is +the last thing you want from your Logging library (again...). + +This does not mean Logrus is dead. Logrus will continue to be maintained for +security, (backwards compatible) bug fixes, and performance (where we are +limited by the interface). + +I believe Logrus' biggest contribution is to have played a part in today's +widespread use of structured logging in Golang. There doesn't seem to be a +reason to do a major, breaking iteration into Logrus V2, since the fantastic Go +community has built those independently. Many fantastic alternatives have sprung +up. Logrus would look like those, had it been re-designed with what we know +about structured logging in Go today. Check out, for example, +[Zerolog][zerolog], [Zap][zap], and [Apex][apex]. + +[zerolog]: https://github.com/rs/zerolog +[zap]: https://github.com/uber-go/zap +[apex]: https://github.com/apex/log + **Seeing weird case-sensitive problems?** It's in the past been possible to import Logrus as both upper- and lower-case. Due to the Go package environment, this caused issues in the community and we needed a standard. Some environments @@ -15,11 +35,6 @@ comments](https://github.com/sirupsen/logrus/issues/553#issuecomment-306591437). For an in-depth explanation of the casing issue, see [this comment](https://github.com/sirupsen/logrus/issues/570#issuecomment-313933276). -**Are you interested in assisting in maintaining Logrus?** Currently I have a -lot of obligations, and I am unable to provide Logrus with the maintainership it -needs. If you'd like to help, please reach out to me at `simon at author's -username dot com`. - Nicely color-coded in development (when a TTY is attached, otherwise just plain text): @@ -187,7 +202,7 @@ func main() { log.Out = os.Stdout // You could set this to any `io.Writer` such as a file - // file, err := os.OpenFile("logrus.log", os.O_CREATE|os.O_WRONLY, 0666) + // file, err := os.OpenFile("logrus.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) // if err == nil { // log.Out = file // } else { @@ -272,7 +287,7 @@ func init() { ``` Note: Syslog hook also support connecting to local syslog (Ex. "/dev/log" or "/var/run/syslog" or "/var/run/log"). For the detail, please check the [syslog hook README](hooks/syslog/README.md). -A list of currently known of service hook can be found in this wiki [page](https://github.com/sirupsen/logrus/wiki/Hooks) +A list of currently known service hooks can be found in this wiki [page](https://github.com/sirupsen/logrus/wiki/Hooks) #### Level logging @@ -354,6 +369,7 @@ The built-in logging formatters are: [github.com/mattn/go-colorable](https://github.com/mattn/go-colorable). * When colors are enabled, levels are truncated to 4 characters by default. To disable truncation set the `DisableLevelTruncation` field to `true`. + * When outputting to a TTY, it's often helpful to visually scan down a column where all the levels are the same width. Setting the `PadLevelText` field to `true` enables this behavior, by adding padding to the level text. * All options are listed in the [generated docs](https://godoc.org/github.com/sirupsen/logrus#TextFormatter). * `logrus.JSONFormatter`. Logs fields as JSON. * All options are listed in the [generated docs](https://godoc.org/github.com/sirupsen/logrus#JSONFormatter). @@ -364,8 +380,10 @@ Third party logging formatters: * [`GELF`](https://github.com/fabienm/go-logrus-formatters). Formats entries so they comply to Graylog's [GELF 1.1 specification](http://docs.graylog.org/en/2.4/pages/gelf.html). * [`logstash`](https://github.com/bshuster-repo/logrus-logstash-hook). Logs fields as [Logstash](http://logstash.net) Events. * [`prefixed`](https://github.com/x-cray/logrus-prefixed-formatter). Displays log entry source along with alternative layout. -* [`zalgo`](https://github.com/aybabtme/logzalgo). Invoking the P͉̫o̳̼̊w̖͈̰͎e̬͔̭͂r͚̼̹̲ ̫͓͉̳͈ō̠͕͖̚f̝͍̠ ͕̲̞͖͑Z̖̫̤̫ͪa͉̬͈̗l͖͎g̳̥o̰̥̅!̣͔̲̻͊̄ ̙̘̦̹̦. +* [`zalgo`](https://github.com/aybabtme/logzalgo). Invoking the Power of Zalgo. * [`nested-logrus-formatter`](https://github.com/antonfisher/nested-logrus-formatter). Converts logrus fields to a nested structure. +* [`powerful-logrus-formatter`](https://github.com/zput/zxcTool). get fileName, log's line number and the latest function's name when print log; Sava log to files. +* [`caption-json-formatter`](https://github.com/nolleh/caption_json_formatter). logrus's message json formatter with human-readable caption added. You can define your formatter by implementing the `Formatter` interface, requiring a `Format` method. `Format` takes an `*Entry`. `entry.Data` is a @@ -430,14 +448,14 @@ entries. It should not be a feature of the application-level logger. | Tool | Description | | ---- | ----------- | -|[Logrus Mate](https://github.com/gogap/logrus_mate)|Logrus mate is a tool for Logrus to manage loggers, you can initial logger's level, hook and formatter by config file, the logger will generated with different config at different environment.| +|[Logrus Mate](https://github.com/gogap/logrus_mate)|Logrus mate is a tool for Logrus to manage loggers, you can initial logger's level, hook and formatter by config file, the logger will be generated with different configs in different environments.| |[Logrus Viper Helper](https://github.com/heirko/go-contrib/tree/master/logrusHelper)|An Helper around Logrus to wrap with spf13/Viper to load configuration with fangs! And to simplify Logrus configuration use some behavior of [Logrus Mate](https://github.com/gogap/logrus_mate). [sample](https://github.com/heirko/iris-contrib/blob/master/middleware/logrus-logger/example) | #### Testing Logrus has a built in facility for asserting the presence of log messages. This is implemented through the `test` hook and provides: -* decorators for existing logger (`test.NewLocal` and `test.NewGlobal`) which basically just add the `test` hook +* decorators for existing logger (`test.NewLocal` and `test.NewGlobal`) which basically just adds the `test` hook * a test logger (`test.NewNullLogger`) that just records log messages (and does not output any): ```go @@ -465,7 +483,7 @@ func TestSomething(t*testing.T){ Logrus can register one or more functions that will be called when any `fatal` level message is logged. The registered handlers will be executed before -logrus performs a `os.Exit(1)`. This behavior may be helpful if callers need +logrus performs an `os.Exit(1)`. This behavior may be helpful if callers need to gracefully shutdown. Unlike a `panic("Something went wrong...")` call which can be intercepted with a deferred `recover` a call to `os.Exit(1)` can not be intercepted. ``` @@ -490,6 +508,6 @@ Situation when locking is not needed includes: 1) logger.Out is protected by locks. - 2) logger.Out is a os.File handler opened with `O_APPEND` flag, and every write is smaller than 4k. (This allow multi-thread/multi-process writing) + 2) logger.Out is an os.File handler opened with `O_APPEND` flag, and every write is smaller than 4k. (This allows multi-thread/multi-process writing) (Refer to http://www.notthewizard.com/2014/06/17/are-files-appends-really-atomic/) diff --git a/vendor/github.com/sirupsen/logrus/entry.go b/vendor/github.com/sirupsen/logrus/entry.go index 63e25583cb..27b14bfb16 100644 --- a/vendor/github.com/sirupsen/logrus/entry.go +++ b/vendor/github.com/sirupsen/logrus/entry.go @@ -85,10 +85,15 @@ func NewEntry(logger *Logger) *Entry { } } +// Returns the bytes representation of this entry from the formatter. +func (entry *Entry) Bytes() ([]byte, error) { + return entry.Logger.Formatter.Format(entry) +} + // Returns the string representation from the reader and ultimately the // formatter. func (entry *Entry) String() (string, error) { - serialized, err := entry.Logger.Formatter.Format(entry) + serialized, err := entry.Bytes() if err != nil { return "", err } @@ -103,7 +108,11 @@ func (entry *Entry) WithError(err error) *Entry { // Add a context to the Entry. func (entry *Entry) WithContext(ctx context.Context) *Entry { - return &Entry{Logger: entry.Logger, Data: entry.Data, Time: entry.Time, err: entry.err, Context: ctx} + dataCopy := make(Fields, len(entry.Data)) + for k, v := range entry.Data { + dataCopy[k] = v + } + return &Entry{Logger: entry.Logger, Data: dataCopy, Time: entry.Time, err: entry.err, Context: ctx} } // Add a single field to the Entry. @@ -113,6 +122,8 @@ func (entry *Entry) WithField(key string, value interface{}) *Entry { // Add a map of fields to the Entry. func (entry *Entry) WithFields(fields Fields) *Entry { + entry.Logger.mu.Lock() + defer entry.Logger.mu.Unlock() data := make(Fields, len(entry.Data)+len(fields)) for k, v := range entry.Data { data[k] = v @@ -144,7 +155,11 @@ func (entry *Entry) WithFields(fields Fields) *Entry { // Overrides the time of the Entry. func (entry *Entry) WithTime(t time.Time) *Entry { - return &Entry{Logger: entry.Logger, Data: entry.Data, Time: t, err: entry.err, Context: entry.Context} + dataCopy := make(Fields, len(entry.Data)) + for k, v := range entry.Data { + dataCopy[k] = v + } + return &Entry{Logger: entry.Logger, Data: dataCopy, Time: t, err: entry.err, Context: entry.Context} } // getPackageName reduces a fully qualified function name to the package name @@ -165,15 +180,20 @@ func getPackageName(f string) string { // getCaller retrieves the name of the first non-logrus calling function func getCaller() *runtime.Frame { - // cache this package's fully-qualified name callerInitOnce.Do(func() { - pcs := make([]uintptr, 2) + pcs := make([]uintptr, maximumCallerDepth) _ = runtime.Callers(0, pcs) - logrusPackage = getPackageName(runtime.FuncForPC(pcs[1]).Name()) - // now that we have the cache, we can skip a minimum count of known-logrus functions - // XXX this is dubious, the number of frames may vary + // dynamic get the package name and the minimum caller depth + for i := 0; i < maximumCallerDepth; i++ { + funcName := runtime.FuncForPC(pcs[i]).Name() + if strings.Contains(funcName, "getCaller") { + logrusPackage = getPackageName(funcName) + break + } + } + minimumCallerDepth = knownLogrusFrames }) @@ -187,7 +207,7 @@ func getCaller() *runtime.Frame { // If the caller isn't part of this package, we're done if pkg != logrusPackage { - return &f + return &f //nolint:scopelint } } @@ -217,9 +237,11 @@ func (entry Entry) log(level Level, msg string) { entry.Level = level entry.Message = msg + entry.Logger.mu.Lock() if entry.Logger.ReportCaller { entry.Caller = getCaller() } + entry.Logger.mu.Unlock() entry.fireHooks() @@ -255,11 +277,10 @@ func (entry *Entry) write() { serialized, err := entry.Logger.Formatter.Format(entry) if err != nil { fmt.Fprintf(os.Stderr, "Failed to obtain reader, %v\n", err) - } else { - _, err = entry.Logger.Out.Write(serialized) - if err != nil { - fmt.Fprintf(os.Stderr, "Failed to write to log, %v\n", err) - } + return + } + if _, err = entry.Logger.Out.Write(serialized); err != nil { + fmt.Fprintf(os.Stderr, "Failed to write to log, %v\n", err) } } diff --git a/vendor/github.com/sirupsen/logrus/exported.go b/vendor/github.com/sirupsen/logrus/exported.go index 62fc2f2193..42b04f6c80 100644 --- a/vendor/github.com/sirupsen/logrus/exported.go +++ b/vendor/github.com/sirupsen/logrus/exported.go @@ -80,7 +80,7 @@ func WithFields(fields Fields) *Entry { return std.WithFields(fields) } -// WithTime creats an entry from the standard logger and overrides the time of +// WithTime creates an entry from the standard logger and overrides the time of // logs generated with it. // // Note that it doesn't log until you call Debug, Print, Info, Warn, Fatal diff --git a/vendor/github.com/sirupsen/logrus/go.mod b/vendor/github.com/sirupsen/logrus/go.mod index 12fdf98984..9ea6e841b4 100644 --- a/vendor/github.com/sirupsen/logrus/go.mod +++ b/vendor/github.com/sirupsen/logrus/go.mod @@ -4,7 +4,8 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.1 github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/stretchr/objx v0.1.1 // indirect github.com/stretchr/testify v1.2.2 golang.org/x/sys v0.0.0-20190422165155-953cdadca894 ) + +go 1.13 diff --git a/vendor/github.com/sirupsen/logrus/go.sum b/vendor/github.com/sirupsen/logrus/go.sum index 596c318b9f..95a3f07de0 100644 --- a/vendor/github.com/sirupsen/logrus/go.sum +++ b/vendor/github.com/sirupsen/logrus/go.sum @@ -1,16 +1,10 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe h1:CHRGQ8V7OlCYtwaKPJi3iA7J+YdNKdo8j7nG5IgDhjs= -github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/vendor/github.com/sirupsen/logrus/json_formatter.go b/vendor/github.com/sirupsen/logrus/json_formatter.go index 098a21a067..ba7f237112 100644 --- a/vendor/github.com/sirupsen/logrus/json_formatter.go +++ b/vendor/github.com/sirupsen/logrus/json_formatter.go @@ -28,6 +28,9 @@ type JSONFormatter struct { // DisableTimestamp allows disabling automatic timestamps in output DisableTimestamp bool + // DisableHTMLEscape allows disabling html escaping in output + DisableHTMLEscape bool + // DataKey allows users to put all the log entry parameters into a nested dictionary at a given key. DataKey string @@ -110,6 +113,7 @@ func (f *JSONFormatter) Format(entry *Entry) ([]byte, error) { } encoder := json.NewEncoder(b) + encoder.SetEscapeHTML(!f.DisableHTMLEscape) if f.PrettyPrint { encoder.SetIndent("", " ") } diff --git a/vendor/github.com/sirupsen/logrus/logger.go b/vendor/github.com/sirupsen/logrus/logger.go index c0c0b1e559..6fdda748e4 100644 --- a/vendor/github.com/sirupsen/logrus/logger.go +++ b/vendor/github.com/sirupsen/logrus/logger.go @@ -68,10 +68,10 @@ func (mw *MutexWrap) Disable() { // `Out` and `Hooks` directly on the default logger instance. You can also just // instantiate your own: // -// var log = &Logger{ +// var log = &logrus.Logger{ // Out: os.Stderr, -// Formatter: new(JSONFormatter), -// Hooks: make(LevelHooks), +// Formatter: new(logrus.JSONFormatter), +// Hooks: make(logrus.LevelHooks), // Level: logrus.DebugLevel, // } // @@ -100,8 +100,9 @@ func (logger *Logger) releaseEntry(entry *Entry) { logger.entryPool.Put(entry) } -// Adds a field to the log entry, note that it doesn't log until you call -// Debug, Print, Info, Warn, Error, Fatal or Panic. It only creates a log entry. +// WithField allocates a new entry and adds a field to it. +// Debug, Print, Info, Warn, Error, Fatal or Panic must be then applied to +// this new returned entry. // If you want multiple fields, use `WithFields`. func (logger *Logger) WithField(key string, value interface{}) *Entry { entry := logger.newEntry() diff --git a/vendor/github.com/sirupsen/logrus/logrus.go b/vendor/github.com/sirupsen/logrus/logrus.go index 8644761f73..2f16224cb9 100644 --- a/vendor/github.com/sirupsen/logrus/logrus.go +++ b/vendor/github.com/sirupsen/logrus/logrus.go @@ -51,7 +51,7 @@ func (level *Level) UnmarshalText(text []byte) error { return err } - *level = Level(l) + *level = l return nil } diff --git a/vendor/github.com/sirupsen/logrus/terminal_check_bsd.go b/vendor/github.com/sirupsen/logrus/terminal_check_bsd.go index 3c4f43f91c..499789984d 100644 --- a/vendor/github.com/sirupsen/logrus/terminal_check_bsd.go +++ b/vendor/github.com/sirupsen/logrus/terminal_check_bsd.go @@ -1,4 +1,5 @@ // +build darwin dragonfly freebsd netbsd openbsd +// +build !js package logrus @@ -10,4 +11,3 @@ func isTerminal(fd int) bool { _, err := unix.IoctlGetTermios(fd, ioctlReadTermios) return err == nil } - diff --git a/vendor/github.com/sirupsen/logrus/terminal_check_js.go b/vendor/github.com/sirupsen/logrus/terminal_check_js.go new file mode 100644 index 0000000000..ebdae3ec62 --- /dev/null +++ b/vendor/github.com/sirupsen/logrus/terminal_check_js.go @@ -0,0 +1,7 @@ +// +build js + +package logrus + +func isTerminal(fd int) bool { + return false +} diff --git a/vendor/github.com/sirupsen/logrus/terminal_check_unix.go b/vendor/github.com/sirupsen/logrus/terminal_check_unix.go index 355dc966f0..cc4fe6e317 100644 --- a/vendor/github.com/sirupsen/logrus/terminal_check_unix.go +++ b/vendor/github.com/sirupsen/logrus/terminal_check_unix.go @@ -1,4 +1,5 @@ // +build linux aix +// +build !js package logrus @@ -10,4 +11,3 @@ func isTerminal(fd int) bool { _, err := unix.IoctlGetTermios(fd, ioctlReadTermios) return err == nil } - diff --git a/vendor/github.com/sirupsen/logrus/text_formatter.go b/vendor/github.com/sirupsen/logrus/text_formatter.go index e01587c437..2d15a239fe 100644 --- a/vendor/github.com/sirupsen/logrus/text_formatter.go +++ b/vendor/github.com/sirupsen/logrus/text_formatter.go @@ -6,9 +6,11 @@ import ( "os" "runtime" "sort" + "strconv" "strings" "sync" "time" + "unicode/utf8" ) const ( @@ -32,6 +34,9 @@ type TextFormatter struct { // Force disabling colors. DisableColors bool + // Force quoting of all values + ForceQuote bool + // Override coloring based on CLICOLOR and CLICOLOR_FORCE. - https://bixense.com/clicolors/ EnvironmentOverrideColors bool @@ -57,6 +62,10 @@ type TextFormatter struct { // Disables the truncation of the level text to 4 characters. DisableLevelTruncation bool + // PadLevelText Adds padding the level text so that all the levels output at the same length + // PadLevelText is a superset of the DisableLevelTruncation option + PadLevelText bool + // QuoteEmptyFields will wrap empty fields in quotes if true QuoteEmptyFields bool @@ -79,23 +88,32 @@ type TextFormatter struct { CallerPrettyfier func(*runtime.Frame) (function string, file string) terminalInitOnce sync.Once + + // The max length of the level text, generated dynamically on init + levelTextMaxLength int } func (f *TextFormatter) init(entry *Entry) { if entry.Logger != nil { f.isTerminal = checkIfTerminal(entry.Logger.Out) } + // Get the max length of the level text + for _, level := range AllLevels { + levelTextLength := utf8.RuneCount([]byte(level.String())) + if levelTextLength > f.levelTextMaxLength { + f.levelTextMaxLength = levelTextLength + } + } } func (f *TextFormatter) isColored() bool { isColored := f.ForceColors || (f.isTerminal && (runtime.GOOS != "windows")) if f.EnvironmentOverrideColors { - if force, ok := os.LookupEnv("CLICOLOR_FORCE"); ok && force != "0" { + switch force, ok := os.LookupEnv("CLICOLOR_FORCE"); { + case ok && force != "0": isColored = true - } else if ok && force == "0" { - isColored = false - } else if os.Getenv("CLICOLOR") == "0" { + case ok && force == "0", os.Getenv("CLICOLOR") == "0": isColored = false } } @@ -217,9 +235,18 @@ func (f *TextFormatter) printColored(b *bytes.Buffer, entry *Entry, keys []strin } levelText := strings.ToUpper(entry.Level.String()) - if !f.DisableLevelTruncation { + if !f.DisableLevelTruncation && !f.PadLevelText { levelText = levelText[0:4] } + if f.PadLevelText { + // Generates the format string used in the next line, for example "%-6s" or "%-7s". + // Based on the max level text length. + formatString := "%-" + strconv.Itoa(f.levelTextMaxLength) + "s" + // Formats the level text by appending spaces up to the max length, for example: + // - "INFO " + // - "WARNING" + levelText = fmt.Sprintf(formatString, levelText) + } // Remove a single newline if it already exists in the message to keep // the behavior of logrus text_formatter the same as the stdlib log package @@ -243,11 +270,12 @@ func (f *TextFormatter) printColored(b *bytes.Buffer, entry *Entry, keys []strin } } - if f.DisableTimestamp { + switch { + case f.DisableTimestamp: fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m%s %-44s ", levelColor, levelText, caller, entry.Message) - } else if !f.FullTimestamp { + case !f.FullTimestamp: fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%04d]%s %-44s ", levelColor, levelText, int(entry.Time.Sub(baseTimestamp)/time.Second), caller, entry.Message) - } else { + default: fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%s]%s %-44s ", levelColor, levelText, entry.Time.Format(timestampFormat), caller, entry.Message) } for _, k := range keys { @@ -258,6 +286,9 @@ func (f *TextFormatter) printColored(b *bytes.Buffer, entry *Entry, keys []strin } func (f *TextFormatter) needsQuoting(text string) bool { + if f.ForceQuote { + return true + } if f.QuoteEmptyFields && len(text) == 0 { return true } diff --git a/vendor/github.com/sirupsen/logrus/writer.go b/vendor/github.com/sirupsen/logrus/writer.go index 9e1f751359..72e8e3a1b6 100644 --- a/vendor/github.com/sirupsen/logrus/writer.go +++ b/vendor/github.com/sirupsen/logrus/writer.go @@ -6,10 +6,16 @@ import ( "runtime" ) +// Writer at INFO level. See WriterLevel for details. func (logger *Logger) Writer() *io.PipeWriter { return logger.WriterLevel(InfoLevel) } +// WriterLevel returns an io.Writer that can be used to write arbitrary text to +// the logger at the given log level. Each line written to the writer will be +// printed in the usual way using formatters and hooks. The writer is part of an +// io.Pipe and it is the callers responsibility to close the writer when done. +// This can be used to override the standard library logger easily. func (logger *Logger) WriterLevel(level Level) *io.PipeWriter { return NewEntry(logger).WriterLevel(level) } diff --git a/vendor/modules.txt b/vendor/modules.txt index e43367b6e1..d95b3fbe4f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -220,14 +220,14 @@ github.com/matttproud/golang_protobuf_extensions/pbutil # github.com/miekg/dns v1.1.27 ## explicit github.com/miekg/dns -# github.com/minio/minio-go/v6 v6.0.45 +# github.com/minio/minio-go/v6 v6.0.53 ## explicit github.com/minio/minio-go/v6 github.com/minio/minio-go/v6/pkg/credentials github.com/minio/minio-go/v6/pkg/encrypt -github.com/minio/minio-go/v6/pkg/s3signer github.com/minio/minio-go/v6/pkg/s3utils github.com/minio/minio-go/v6/pkg/set +github.com/minio/minio-go/v6/pkg/signer # github.com/minio/sha256-simd v0.1.1 github.com/minio/sha256-simd # github.com/mitchellh/go-homedir v1.1.0 @@ -309,7 +309,7 @@ github.com/segmentio/analytics-go # github.com/segmentio/backo-go v0.0.0-20160424052352-204274ad699c ## explicit github.com/segmentio/backo-go -# github.com/sirupsen/logrus v1.4.2 +# github.com/sirupsen/logrus v1.5.0 ## explicit github.com/sirupsen/logrus # github.com/smartystreets/assertions v1.0.0