update github.com/minio/minio-go/v6 to v6.0.53 (#14349)

Этот коммит содержится в:
Doug Lauder
2020-04-22 12:15:06 -04:00
коммит произвёл GitHub
родитель 636d168b84
Коммит 50821d1a34
59 изменённых файлов: 1598 добавлений и 521 удалений

5
vendor/github.com/minio/minio-go/v6/README.md сгенерированный поставляемый
Просмотреть файл

@@ -190,6 +190,11 @@ The full API Reference is available here.
* [setbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketlifecycle.go)
* [getbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketlifecycle.go)
### Full Examples : Bucket encryption Operations
* [setbucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketencryption.go)
* [getbucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketencryption.go)
* [deletebucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/deletebucketencryption.go)
### Full Examples : Bucket notification Operations
* [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go)
* [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go)

9
vendor/github.com/minio/minio-go/v6/api-compose-object.go сгенерированный поставляемый
Просмотреть файл

@@ -60,6 +60,13 @@ type DestInfoOptions struct {
// Otherwise this field is ignored
UserTags map[string]string
ReplaceTags bool
// Specifies whether you want to apply a Legal Hold to the copied object.
LegalHold LegalHoldStatus
// Object Retention related fields
Mode RetentionMode
RetainUntilDate time.Time
}
// Process custom-metadata to remove a `x-amz-meta-` prefix if
@@ -107,6 +114,8 @@ func NewDestinationInfo(bucket, object string, sse encrypt.ServerSide, userMeta
UserMeta: m,
UserTags: nil,
ReplaceTags: false,
LegalHold: LegalHoldStatus(""),
Mode: RetentionMode(""),
}
return DestinationInfo{
bucket: bucket,

3
vendor/github.com/minio/minio-go/v6/api-datatypes.go сгенерированный поставляемый
Просмотреть файл

@@ -82,6 +82,9 @@ type ObjectInfo struct {
// x-amz-meta-* headers stripped "x-amz-meta-" prefix containing the first value.
UserMetadata StringMap `json:"userMetadata"`
// x-amz-tagging values in their k/v values.
UserTags map[string]string `json:"userTags"`
// Owner name.
Owner struct {
DisplayName string `json:"name"`

71
vendor/github.com/minio/minio-go/v6/api-get-bucket-encryption.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,71 @@
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2020 MinIO, Inc.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package minio
import (
"context"
"encoding/xml"
"io/ioutil"
"net/http"
"net/url"
"github.com/minio/minio-go/v6/pkg/s3utils"
)
// GetBucketEncryption - get default encryption configuration for a bucket.
func (c Client) GetBucketEncryption(bucketName string) (ServerSideEncryptionConfiguration, error) {
return c.GetBucketEncryptionWithContext(context.Background(), bucketName)
}
// GetBucketEncryptionWithContext gets the default encryption configuration on an existing bucket with a context to control cancellations and timeouts.
func (c Client) GetBucketEncryptionWithContext(ctx context.Context, bucketName string) (ServerSideEncryptionConfiguration, error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return ServerSideEncryptionConfiguration{}, err
}
// Get resources properly escaped and lined up before
// using them in http request.
urlValues := make(url.Values)
urlValues.Set("encryption", "")
// Execute GET on bucket to get the default encryption configuration.
resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
})
defer closeResponse(resp)
if err != nil {
return ServerSideEncryptionConfiguration{}, err
}
if resp.StatusCode != http.StatusOK {
return ServerSideEncryptionConfiguration{}, httpRespToErrorResponse(resp, bucketName, "")
}
bucketEncryptionBuf, err := ioutil.ReadAll(resp.Body)
if err != nil {
return ServerSideEncryptionConfiguration{}, err
}
encryptionConfig := ServerSideEncryptionConfiguration{}
if err := xml.Unmarshal(bucketEncryptionBuf, &encryptionConfig); err != nil {
return ServerSideEncryptionConfiguration{}, err
}
return encryptionConfig, nil
}

78
vendor/github.com/minio/minio-go/v6/api-get-bucket-versioning.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,78 @@
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2020 MinIO, Inc.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package minio
import (
"context"
"encoding/xml"
"io/ioutil"
"net/http"
"net/url"
"github.com/minio/minio-go/v6/pkg/s3utils"
)
// BucketVersioningConfiguration is the versioning configuration structure
type BucketVersioningConfiguration struct {
XMLName xml.Name `xml:"VersioningConfiguration"`
Status string `xml:"Status"`
MfaDelete string `xml:"MfaDelete,omitempty"`
}
// GetBucketVersioning - get versioning configuration for a bucket.
func (c Client) GetBucketVersioning(bucketName string) (BucketVersioningConfiguration, error) {
return c.GetBucketVersioningWithContext(context.Background(), bucketName)
}
// GetBucketVersioningWithContext gets the versioning configuration on an existing bucket with a context to control cancellations and timeouts.
func (c Client) GetBucketVersioningWithContext(ctx context.Context, bucketName string) (BucketVersioningConfiguration, error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return BucketVersioningConfiguration{}, err
}
// Get resources properly escaped and lined up before
// using them in http request.
urlValues := make(url.Values)
urlValues.Set("versioning", "")
// Execute GET on bucket to get the versioning configuration.
resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
})
defer closeResponse(resp)
if err != nil {
return BucketVersioningConfiguration{}, err
}
if resp.StatusCode != http.StatusOK {
return BucketVersioningConfiguration{}, httpRespToErrorResponse(resp, bucketName, "")
}
bucketVersioningBuf, err := ioutil.ReadAll(resp.Body)
if err != nil {
return BucketVersioningConfiguration{}, err
}
versioningConfig := BucketVersioningConfiguration{}
if err := xml.Unmarshal(bucketVersioningBuf, &versioningConfig); err != nil {
return BucketVersioningConfiguration{}, err
}
return versioningConfig, nil
}

12
vendor/github.com/minio/minio-go/v6/api-get-object-file.go сгенерированный поставляемый
Просмотреть файл

@@ -87,6 +87,17 @@ func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectNam
return err
}
// If we return early with an error, be sure to close and delete
// filePart. If we have an error along the way there is a chance
// that filePart is somehow damaged, and we should discard it.
closeAndRemove := true
defer func() {
if closeAndRemove {
_ = filePart.Close()
_ = os.Remove(filePartPath)
}
}()
// Issue Stat to get the current offset.
st, err = filePart.Stat()
if err != nil {
@@ -111,6 +122,7 @@ func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectNam
}
// Close the file before rename, this is specifically needed for Windows users.
closeAndRemove = false
if err = filePart.Close(); err != nil {
return err
}

38
vendor/github.com/minio/minio-go/v6/api-get-object.go сгенерированный поставляемый
Просмотреть файл

@@ -1,6 +1,6 @@
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2015-2017 MinIO, Inc.
* Copyright 2015-2020 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -23,9 +23,7 @@ import (
"fmt"
"io"
"net/http"
"strings"
"sync"
"time"
"github.com/minio/minio-go/v6/pkg/s3utils"
)
@@ -635,38 +633,10 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, op
}
}
// Trim off the odd double quotes from ETag in the beginning and end.
md5sum := trimEtag(resp.Header.Get("ETag"))
// Parse the date.
date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified"))
objectStat, err := ToObjectInfo(bucketName, objectName, resp.Header)
if err != nil {
msg := "Last-Modified time format not recognized. " + reportIssue
return nil, ObjectInfo{}, nil, ErrorResponse{
Code: "InternalError",
Message: msg,
RequestID: resp.Header.Get("x-amz-request-id"),
HostID: resp.Header.Get("x-amz-id-2"),
Region: resp.Header.Get("x-amz-bucket-region"),
}
}
// Get content-type.
contentType := strings.TrimSpace(resp.Header.Get("Content-Type"))
if contentType == "" {
contentType = "application/octet-stream"
}
objectStat := ObjectInfo{
ETag: md5sum,
Key: objectName,
Size: resp.ContentLength,
LastModified: date,
ContentType: contentType,
// Extract only the relevant header keys describing the object.
// following function filters out a list of standard set of keys
// which are not part of object metadata.
Metadata: extractObjMetadata(resp.Header),
closeResponse(resp)
return nil, objectStat, resp.Header, nil
}
// do not close body here, caller will close

28
vendor/github.com/minio/minio-go/v6/api-list.go сгенерированный поставляемый
Просмотреть файл

@@ -317,14 +317,14 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s
}
for i, obj := range listBucketResult.Contents {
listBucketResult.Contents[i].Key, err = url.QueryUnescape(obj.Key)
listBucketResult.Contents[i].Key, err = decodeS3Name(obj.Key, listBucketResult.EncodingType)
if err != nil {
return listBucketResult, err
}
}
for i, obj := range listBucketResult.CommonPrefixes {
listBucketResult.CommonPrefixes[i].Prefix, err = url.QueryUnescape(obj.Prefix)
listBucketResult.CommonPrefixes[i].Prefix, err = decodeS3Name(obj.Prefix, listBucketResult.EncodingType)
if err != nil {
return listBucketResult, err
}
@@ -500,21 +500,21 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit
}
for i, obj := range listBucketResult.Contents {
listBucketResult.Contents[i].Key, err = url.QueryUnescape(obj.Key)
listBucketResult.Contents[i].Key, err = decodeS3Name(obj.Key, listBucketResult.EncodingType)
if err != nil {
return listBucketResult, err
}
}
for i, obj := range listBucketResult.CommonPrefixes {
listBucketResult.CommonPrefixes[i].Prefix, err = url.QueryUnescape(obj.Prefix)
listBucketResult.CommonPrefixes[i].Prefix, err = decodeS3Name(obj.Prefix, listBucketResult.EncodingType)
if err != nil {
return listBucketResult, err
}
}
if listBucketResult.NextMarker != "" {
listBucketResult.NextMarker, err = url.QueryUnescape(listBucketResult.NextMarker)
listBucketResult.NextMarker, err = decodeS3Name(listBucketResult.NextMarker, listBucketResult.EncodingType)
if err != nil {
return listBucketResult, err
}
@@ -697,25 +697,25 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker,
return listMultipartUploadsResult, err
}
listMultipartUploadsResult.NextKeyMarker, err = url.QueryUnescape(listMultipartUploadsResult.NextKeyMarker)
listMultipartUploadsResult.NextKeyMarker, err = decodeS3Name(listMultipartUploadsResult.NextKeyMarker, listMultipartUploadsResult.EncodingType)
if err != nil {
return listMultipartUploadsResult, err
}
listMultipartUploadsResult.NextUploadIDMarker, err = url.QueryUnescape(listMultipartUploadsResult.NextUploadIDMarker)
listMultipartUploadsResult.NextUploadIDMarker, err = decodeS3Name(listMultipartUploadsResult.NextUploadIDMarker, listMultipartUploadsResult.EncodingType)
if err != nil {
return listMultipartUploadsResult, err
}
for i, obj := range listMultipartUploadsResult.Uploads {
listMultipartUploadsResult.Uploads[i].Key, err = url.QueryUnescape(obj.Key)
listMultipartUploadsResult.Uploads[i].Key, err = decodeS3Name(obj.Key, listMultipartUploadsResult.EncodingType)
if err != nil {
return listMultipartUploadsResult, err
}
}
for i, obj := range listMultipartUploadsResult.CommonPrefixes {
listMultipartUploadsResult.CommonPrefixes[i].Prefix, err = url.QueryUnescape(obj.Prefix)
listMultipartUploadsResult.CommonPrefixes[i].Prefix, err = decodeS3Name(obj.Prefix, listMultipartUploadsResult.EncodingType)
if err != nil {
return listMultipartUploadsResult, err
}
@@ -837,3 +837,13 @@ func (c Client) listObjectPartsQuery(bucketName, objectName, uploadID string, pa
}
return listObjectPartsResult, nil
}
// Decode an S3 object name according to the encoding type
func decodeS3Name(name, encodingType string) (string, error) {
switch encodingType {
case "url":
return url.QueryUnescape(name)
default:
return name, nil
}
}

14
vendor/github.com/minio/minio-go/v6/api-notification.go сгенерированный поставляемый
Просмотреть файл

@@ -1,6 +1,6 @@
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2017 MinIO, Inc.
* Copyright 2017-2020 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -89,11 +89,13 @@ type bucketMeta struct {
// Notification event object metadata.
type objectMeta struct {
Key string `json:"key"`
Size int64 `json:"size,omitempty"`
ETag string `json:"eTag,omitempty"`
VersionID string `json:"versionId,omitempty"`
Sequencer string `json:"sequencer"`
Key string `json:"key"`
Size int64 `json:"size,omitempty"`
ETag string `json:"eTag,omitempty"`
ContentType string `json:"contentType,omitempty"`
UserMetadata map[string]string `json:"userMetadata,omitempty"`
VersionID string `json:"versionId,omitempty"`
Sequencer string `json:"sequencer"`
}
// Notification event server specific metadata.

176
vendor/github.com/minio/minio-go/v6/api-object-legal-hold.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,176 @@
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2020 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package minio
import (
"bytes"
"context"
"encoding/xml"
"fmt"
"net/http"
"net/url"
"github.com/minio/minio-go/v6/pkg/s3utils"
)
// objectLegalHold - object legal hold specified in
// https://docs.aws.amazon.com/AmazonS3/latest/API/archive-RESTObjectPUTLegalHold.html
type objectLegalHold struct {
XMLNS string `xml:"xmlns,attr,omitempty"`
XMLName xml.Name `xml:"LegalHold"`
Status LegalHoldStatus `xml:"Status,omitempty"`
}
// PutObjectLegalHoldOptions represents options specified by user for PutObjectLegalHold call
type PutObjectLegalHoldOptions struct {
VersionID string
Status *LegalHoldStatus
}
// GetObjectLegalHoldOptions represents options specified by user for GetObjectLegalHold call
type GetObjectLegalHoldOptions struct {
VersionID string
}
// LegalHoldStatus - object legal hold status.
type LegalHoldStatus string
const (
// LegalHoldEnabled indicates legal hold is enabled
LegalHoldEnabled LegalHoldStatus = "ON"
// LegalHoldDisabled indicates legal hold is disabled
LegalHoldDisabled LegalHoldStatus = "OFF"
)
func (r LegalHoldStatus) String() string {
return string(r)
}
// IsValid - check whether this legal hold status is valid or not.
func (r LegalHoldStatus) IsValid() bool {
return r == LegalHoldEnabled || r == LegalHoldDisabled
}
func newObjectLegalHold(status *LegalHoldStatus) (*objectLegalHold, error) {
if status == nil {
return nil, fmt.Errorf("Status not set")
}
if !status.IsValid() {
return nil, fmt.Errorf("invalid legal hold status `%v`", status)
}
legalHold := &objectLegalHold{
Status: *status,
}
return legalHold, nil
}
// PutObjectLegalHold : sets object legal hold for a given object and versionID.
func (c Client) PutObjectLegalHold(bucketName, objectName string, opts PutObjectLegalHoldOptions) error {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return err
}
if err := s3utils.CheckValidObjectName(objectName); err != nil {
return err
}
// Get resources properly escaped and lined up before
// using them in http request.
urlValues := make(url.Values)
urlValues.Set("legal-hold", "")
if opts.VersionID != "" {
urlValues.Set("versionId", opts.VersionID)
}
lh, err := newObjectLegalHold(opts.Status)
if err != nil {
return err
}
lhData, err := xml.Marshal(lh)
if err != nil {
return err
}
reqMetadata := requestMetadata{
bucketName: bucketName,
objectName: objectName,
queryValues: urlValues,
contentBody: bytes.NewReader(lhData),
contentLength: int64(len(lhData)),
contentMD5Base64: sumMD5Base64(lhData),
contentSHA256Hex: sum256Hex(lhData),
}
// Execute PUT Object Legal Hold.
resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata)
defer closeResponse(resp)
if err != nil {
return err
}
if resp != nil {
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent {
return httpRespToErrorResponse(resp, bucketName, objectName)
}
}
return nil
}
// GetObjectLegalHold gets legal-hold status of given object.
func (c Client) GetObjectLegalHold(bucketName, objectName string, opts GetObjectLegalHoldOptions) (status *LegalHoldStatus, err error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return nil, err
}
if err := s3utils.CheckValidObjectName(objectName); err != nil {
return nil, err
}
urlValues := make(url.Values)
urlValues.Set("legal-hold", "")
if opts.VersionID != "" {
urlValues.Set("versionId", opts.VersionID)
}
// Execute GET on bucket to list objects.
resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{
bucketName: bucketName,
objectName: objectName,
queryValues: urlValues,
contentSHA256Hex: emptySHA256Hex,
})
defer closeResponse(resp)
if err != nil {
return nil, err
}
if resp != nil {
if resp.StatusCode != http.StatusOK {
return nil, httpRespToErrorResponse(resp, bucketName, objectName)
}
}
lh := &objectLegalHold{}
if err = xml.NewDecoder(resp.Body).Decode(lh); err != nil {
return nil, err
}
return &lh.Status, nil
}

29
vendor/github.com/minio/minio-go/v6/api-object-lock.go сгенерированный поставляемый
Просмотреть файл

@@ -183,11 +183,11 @@ func (c Client) SetBucketObjectLockConfig(bucketName string, mode *RetentionMode
return nil
}
// GetBucketObjectLockConfig gets object lock configuration of given bucket.
func (c Client) GetBucketObjectLockConfig(bucketName string) (mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) {
// GetObjectLockConfig gets object lock configuration of given bucket.
func (c Client) GetObjectLockConfig(bucketName string) (objectLock string, mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return nil, nil, nil, err
return "", nil, nil, nil, err
}
urlValues := make(url.Values)
@@ -201,16 +201,16 @@ func (c Client) GetBucketObjectLockConfig(bucketName string) (mode *RetentionMod
})
defer closeResponse(resp)
if err != nil {
return nil, nil, nil, err
return "", nil, nil, nil, err
}
if resp != nil {
if resp.StatusCode != http.StatusOK {
return nil, nil, nil, httpRespToErrorResponse(resp, bucketName, "")
return "", nil, nil, nil, httpRespToErrorResponse(resp, bucketName, "")
}
}
config := &objectLockConfig{}
if err = xml.NewDecoder(resp.Body).Decode(config); err != nil {
return nil, nil, nil, err
return "", nil, nil, nil, err
}
if config.Rule != nil {
@@ -224,9 +224,18 @@ func (c Client) GetBucketObjectLockConfig(bucketName string) (mode *RetentionMod
years := Years
unit = &years
}
return mode, validity, unit, nil
return config.ObjectLockEnabled, mode, validity, unit, nil
}
return nil, nil, nil, nil
return config.ObjectLockEnabled, nil, nil, nil, nil
}
// GetBucketObjectLockConfig gets object lock configuration of given bucket.
func (c Client) GetBucketObjectLockConfig(bucketName string) (mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) {
_, mode, validity, unit, err = c.GetObjectLockConfig(bucketName)
return mode, validity, unit, err
}
// SetObjectLockConfig sets object lock configuration in given bucket. mode, validity and unit are either all set or all nil.
func (c Client) SetObjectLockConfig(bucketName string, mode *RetentionMode, validity *uint, unit *ValidityUnit) error {
return c.SetBucketObjectLockConfig(bucketName, mode, validity, unit)
}

2
vendor/github.com/minio/minio-go/v6/api-object-retention.go сгенерированный поставляемый
Просмотреть файл

@@ -100,7 +100,7 @@ func (c Client) PutObjectRetention(bucketName, objectName string, opts PutObject
if opts.GovernanceBypass {
// Set the bypass goverenance retention header
headers.Set("x-amz-bypass-governance-retention", "True")
headers.Set(amzBypassGovernance, "true")
}
reqMetadata := requestMetadata{

3
vendor/github.com/minio/minio-go/v6/api-object-tagging.go сгенерированный поставляемый
Просмотреть файл

@@ -155,7 +155,8 @@ func (c Client) RemoveObjectTaggingWithContext(ctx context.Context, bucketName,
}
if resp != nil {
if resp.StatusCode != http.StatusOK {
// S3 returns "204 No content" after Object tag deletion.
if resp.StatusCode != http.StatusNoContent {
return httpRespToErrorResponse(resp, bucketName, objectName)
}
}

8
vendor/github.com/minio/minio-go/v6/api-presigned.go сгенерированный поставляемый
Просмотреть файл

@@ -23,8 +23,8 @@ import (
"net/url"
"time"
"github.com/minio/minio-go/v6/pkg/s3signer"
"github.com/minio/minio-go/v6/pkg/s3utils"
"github.com/minio/minio-go/v6/pkg/signer"
)
// presignURL - Returns a presigned URL for an input 'method'.
@@ -157,7 +157,7 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[str
p.formData["AWSAccessKeyId"] = accessKeyID
}
// Sign the policy.
p.formData["signature"] = s3signer.PostPresignSignatureV2(policyBase64, secretAccessKey)
p.formData["signature"] = signer.PostPresignSignatureV2(policyBase64, secretAccessKey)
return u, p.formData, nil
}
@@ -180,7 +180,7 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[str
}
// Add a credential policy.
credential := s3signer.GetCredential(accessKeyID, location, t)
credential := signer.GetCredential(accessKeyID, location, t, signer.ServiceTypeS3)
if err = p.addNewPolicy(policyCondition{
matchType: "eq",
condition: "$x-amz-credential",
@@ -210,6 +210,6 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[str
if sessionToken != "" {
p.formData["x-amz-security-token"] = sessionToken
}
p.formData["x-amz-signature"] = s3signer.PostPresignSignatureV4(policyBase64, t, secretAccessKey, location)
p.formData["x-amz-signature"] = signer.PostPresignSignatureV4(policyBase64, t, secretAccessKey, location)
return u, p.formData, nil
}

97
vendor/github.com/minio/minio-go/v6/api-put-bucket.go сгенерированный поставляемый
Просмотреть файл

@@ -1,6 +1,6 @@
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2015-2017 MinIO, Inc.
* Copyright 2015-2020 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -29,6 +29,25 @@ import (
"github.com/minio/minio-go/v6/pkg/s3utils"
)
// ApplyServerSideEncryptionByDefault defines default encryption configuration, KMS or SSE. To activate
// KMS, SSEAlgoritm needs to be set to "aws:kms"
// Minio currently does not support Kms.
type ApplyServerSideEncryptionByDefault struct {
KmsMasterKeyID string `xml:"KMSMasterKeyID,omitempty"`
SSEAlgorithm string `xml:"SSEAlgorithm"`
}
// Rule layer encapsulates default encryption configuration
type Rule struct {
Apply ApplyServerSideEncryptionByDefault `xml:"ApplyServerSideEncryptionByDefault"`
}
// ServerSideEncryptionConfiguration is the default encryption configuration structure
type ServerSideEncryptionConfiguration struct {
XMLName xml.Name `xml:"ServerSideEncryptionConfiguration"`
Rules []Rule `xml:"Rule"`
}
/// Bucket operations
func (c Client) makeBucket(ctx context.Context, bucketName string, location string, objectLockEnabled bool) (err error) {
@@ -313,6 +332,82 @@ func (c Client) removeBucketLifecycle(ctx context.Context, bucketName string) er
return nil
}
// SetBucketEncryption sets the default encryption configuration on an existing bucket.
func (c Client) SetBucketEncryption(bucketName string, configuration ServerSideEncryptionConfiguration) error {
return c.SetBucketEncryptionWithContext(context.Background(), bucketName, configuration)
}
// SetBucketEncryptionWithContext sets the default encryption configuration on an existing bucket with a context to control cancellations and timeouts.
func (c Client) SetBucketEncryptionWithContext(ctx context.Context, bucketName string, configuration ServerSideEncryptionConfiguration) error {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return err
}
buf, err := xml.Marshal(&configuration)
if err != nil {
return err
}
// Get resources properly escaped and lined up before
// using them in http request.
urlValues := make(url.Values)
urlValues.Set("encryption", "")
// Content-length is mandatory to set a default encryption configuration
reqMetadata := requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
contentBody: bytes.NewReader(buf),
contentLength: int64(len(buf)),
contentMD5Base64: sumMD5Base64(buf),
}
// Execute PUT to upload a new bucket default encryption configuration.
resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata)
defer closeResponse(resp)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return httpRespToErrorResponse(resp, bucketName, "")
}
return nil
}
// DeleteBucketEncryption removes the default encryption configuration on a bucket.
func (c Client) DeleteBucketEncryption(bucketName string) error {
return c.DeleteBucketEncryptionWithContext(context.Background(), bucketName)
}
// DeleteBucketEncryptionWithContext removes the default encryption configuration on a bucket with a context to control cancellations and timeouts.
func (c Client) DeleteBucketEncryptionWithContext(ctx context.Context, bucketName string) error {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return err
}
// Get resources properly escaped and lined up before
// using them in http request.
urlValues := make(url.Values)
urlValues.Set("encryption", "")
// DELETE default encryption configuration on a bucket.
resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Hex: emptySHA256Hex,
})
defer closeResponse(resp)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent {
return httpRespToErrorResponse(resp, bucketName, "")
}
return nil
}
// SetBucketNotification saves a new bucket notification.
func (c Client) SetBucketNotification(bucketName string, bucketNotification BucketNotification) error {
return c.SetBucketNotificationWithContext(context.Background(), bucketName, bucketNotification)

10
vendor/github.com/minio/minio-go/v6/api-put-object-copy.go сгенерированный поставляемый
Просмотреть файл

@@ -22,6 +22,7 @@ import (
"io"
"io/ioutil"
"net/http"
"time"
"github.com/minio/minio-go/v6/pkg/encrypt"
"github.com/minio/minio-go/v6/pkg/s3utils"
@@ -45,6 +46,15 @@ func (c Client) CopyObjectWithProgress(dst DestinationInfo, src SourceInfo, prog
header.Set(amzTaggingHeader, s3utils.TagEncode(dst.opts.UserTags))
}
if dst.opts.LegalHold != LegalHoldStatus("") {
header.Set(amzLegalHoldHeader, dst.opts.LegalHold.String())
}
if dst.opts.Mode != RetentionMode("") && !dst.opts.RetainUntilDate.IsZero() {
header.Set(amzLockMode, dst.opts.Mode.String())
header.Set(amzLockRetainUntil, dst.opts.RetainUntilDate.Format(time.RFC3339))
}
var err error
var size int64
// If progress bar is specified, size should be requested as well initiate a StatObject request.

4
vendor/github.com/minio/minio-go/v6/api-put-object-multipart.go сгенерированный поставляемый
Просмотреть файл

@@ -50,7 +50,7 @@ func (c Client) putObjectMultipart(ctx context.Context, bucketName, objectName s
return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName)
}
// Fall back to uploading as single PutObject operation.
return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts)
return c.putObject(ctx, bucketName, objectName, reader, size, opts)
}
}
return n, err
@@ -104,7 +104,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje
// Choose hash algorithms to be calculated by hashCopyN,
// avoid sha256 with non-v4 signature request or
// HTTPS connection.
hashAlgos, hashSums := c.hashMaterials()
hashAlgos, hashSums := c.hashMaterials(opts.SendContentMd5)
length, rErr := io.ReadFull(reader, buf)
if rErr == io.EOF {

70
vendor/github.com/minio/minio-go/v6/api-put-object-streaming.go сгенерированный поставляемый
Просмотреть файл

@@ -18,10 +18,14 @@
package minio
import (
"bytes"
"context"
"crypto/md5"
"encoding/base64"
"fmt"
"io"
"net/http"
"runtime/debug"
"sort"
"strings"
@@ -40,11 +44,11 @@ import (
func (c Client) putObjectMultipartStream(ctx context.Context, bucketName, objectName string,
reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) {
if !isObject(reader) && isReadAt(reader) {
if !isObject(reader) && isReadAt(reader) && !opts.SendContentMd5 {
// Verify if the reader implements ReadAt and it is not a *minio.Object then we will use parallel uploader.
n, err = c.putObjectMultipartStreamFromReadAt(ctx, bucketName, objectName, reader.(io.ReaderAt), size, opts)
} else {
n, err = c.putObjectMultipartStreamNoChecksum(ctx, bucketName, objectName, reader, size, opts)
n, err = c.putObjectMultipartStreamOptionalChecksum(ctx, bucketName, objectName, reader, size, opts)
}
if err != nil {
errResp := ToErrorResponse(err)
@@ -56,7 +60,7 @@ func (c Client) putObjectMultipartStream(ctx context.Context, bucketName, object
return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName)
}
// Fall back to uploading as single PutObject operation.
return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts)
return c.putObject(ctx, bucketName, objectName, reader, size, opts)
}
}
return n, err
@@ -220,7 +224,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa
return totalUploadedSize, nil
}
func (c Client) putObjectMultipartStreamNoChecksum(ctx context.Context, bucketName, objectName string,
func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bucketName, objectName string,
reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) {
// Input validation.
if err = s3utils.CheckValidBucketName(bucketName); err != nil {
@@ -257,20 +261,47 @@ func (c Client) putObjectMultipartStreamNoChecksum(ctx context.Context, bucketNa
// Initialize parts uploaded map.
partsInfo := make(map[int]ObjectPart)
// Create a buffer.
buf := make([]byte, partSize)
defer debug.FreeOSMemory()
// Avoid declaring variables in the for loop
var md5Base64 string
var hookReader io.Reader
// Part number always starts with '1'.
var partNumber int
for partNumber = 1; partNumber <= totalPartsCount; partNumber++ {
// Update progress reader appropriately to the latest offset
// as we read from the source.
hookReader := newHook(reader, opts.Progress)
// Proceed to upload the part.
if partNumber == totalPartsCount {
partSize = lastPartSize
}
if opts.SendContentMd5 {
length, rerr := io.ReadFull(reader, buf)
if rerr == io.EOF && partNumber > 1 {
break
}
if rerr != nil && rerr != io.ErrUnexpectedEOF && rerr != io.EOF {
return 0, rerr
}
// Calculate md5sum.
hash := md5.New()
hash.Write(buf[:length])
md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil))
// Update progress reader appropriately to the latest offset
// as we read from the source.
hookReader = newHook(bytes.NewReader(buf[:length]), opts.Progress)
} else {
// Update progress reader appropriately to the latest offset
// as we read from the source.
hookReader = newHook(reader, opts.Progress)
}
objPart, uerr := c.uploadPart(ctx, bucketName, objectName, uploadID,
io.LimitReader(hookReader, partSize),
partNumber, "", "", partSize, opts.ServerSideEncryption)
partNumber, md5Base64, "", partSize, opts.ServerSideEncryption)
if uerr != nil {
return totalUploadedSize, uerr
}
@@ -316,9 +347,9 @@ func (c Client) putObjectMultipartStreamNoChecksum(ctx context.Context, bucketNa
return totalUploadedSize, nil
}
// putObjectNoChecksum special function used Google Cloud Storage. This special function
// putObject special function used Google Cloud Storage. This special function
// is used for Google Cloud Storage since Google's multipart API is not S3 compatible.
func (c Client) putObjectNoChecksum(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) {
func (c Client) putObject(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return 0, err
@@ -343,13 +374,30 @@ func (c Client) putObjectNoChecksum(ctx context.Context, bucketName, objectName
}
}
var md5Base64 string
if opts.SendContentMd5 {
// Create a buffer.
buf := make([]byte, size)
defer debug.FreeOSMemory()
length, rErr := io.ReadFull(reader, buf)
if rErr != nil && rErr != io.ErrUnexpectedEOF {
return 0, rErr
}
// Calculate md5sum.
hash := md5.New()
hash.Write(buf[:length])
md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil))
}
// Update progress reader appropriately to the latest offset as we
// read from the source.
readSeeker := newHook(reader, opts.Progress)
// This function does not calculate sha256 and md5sum for payload.
// Execute put object.
st, err := c.putObjectDo(ctx, bucketName, objectName, readSeeker, "", "", size, opts)
st, err := c.putObjectDo(ctx, bucketName, objectName, readSeeker, md5Base64, "", size, opts)
if err != nil {
return 0, err
}

75
vendor/github.com/minio/minio-go/v6/api-put-object.go сгенерированный поставляемый
Просмотреть файл

@@ -20,6 +20,9 @@ package minio
import (
"bytes"
"context"
"crypto/md5"
"encoding/base64"
"errors"
"fmt"
"io"
"net/http"
@@ -49,6 +52,9 @@ type PutObjectOptions struct {
StorageClass string
WebsiteRedirectLocation string
PartSize uint64
LegalHold LegalHoldStatus
SendContentMd5 bool
DisableMultipart bool
}
// getNumThreads - gets the number of threads to be used in the multipart
@@ -67,48 +73,58 @@ func (opts PutObjectOptions) getNumThreads() (numThreads int) {
func (opts PutObjectOptions) Header() (header http.Header) {
header = make(http.Header)
if opts.ContentType != "" {
header["Content-Type"] = []string{opts.ContentType}
} else {
header["Content-Type"] = []string{"application/octet-stream"}
contentType := opts.ContentType
if contentType == "" {
contentType = "application/octet-stream"
}
header.Set("Content-Type", contentType)
if opts.ContentEncoding != "" {
header["Content-Encoding"] = []string{opts.ContentEncoding}
header.Set("Content-Encoding", opts.ContentEncoding)
}
if opts.ContentDisposition != "" {
header["Content-Disposition"] = []string{opts.ContentDisposition}
header.Set("Content-Disposition", opts.ContentDisposition)
}
if opts.ContentLanguage != "" {
header["Content-Language"] = []string{opts.ContentLanguage}
header.Set("Content-Language", opts.ContentLanguage)
}
if opts.CacheControl != "" {
header["Cache-Control"] = []string{opts.CacheControl}
header.Set("Cache-Control", opts.CacheControl)
}
if opts.Mode != nil {
header["x-amz-object-lock-mode"] = []string{opts.Mode.String()}
header.Set(amzLockMode, opts.Mode.String())
}
if opts.RetainUntilDate != nil {
header["x-amz-object-lock-retain-until-date"] = []string{opts.RetainUntilDate.Format(time.RFC3339)}
header.Set("X-Amz-Object-Lock-Retain-Until-Date", opts.RetainUntilDate.Format(time.RFC3339))
}
if opts.LegalHold != "" {
header.Set(amzLegalHoldHeader, opts.LegalHold.String())
}
if opts.ServerSideEncryption != nil {
opts.ServerSideEncryption.Marshal(header)
}
if opts.StorageClass != "" {
header[amzStorageClass] = []string{opts.StorageClass}
header.Set(amzStorageClass, opts.StorageClass)
}
if opts.WebsiteRedirectLocation != "" {
header[amzWebsiteRedirectLocation] = []string{opts.WebsiteRedirectLocation}
header.Set(amzWebsiteRedirectLocation, opts.WebsiteRedirectLocation)
}
if len(opts.UserTags) != 0 {
header[amzTaggingHeader] = []string{s3utils.TagEncode(opts.UserTags)}
header.Set(amzTaggingHeader, s3utils.TagEncode(opts.UserTags))
}
for k, v := range opts.UserMetadata {
if !isAmzHeader(k) && !isStandardHeader(k) && !isStorageClassHeader(k) {
header["X-Amz-Meta-"+k] = []string{v}
header.Set("X-Amz-Meta-"+k, v)
} else {
header[k] = []string{v}
header.Set(k, v)
}
}
return
@@ -129,6 +145,9 @@ func (opts PutObjectOptions) validate() (err error) {
return ErrInvalidArgument(opts.Mode.String() + " unsupported retention mode")
}
}
if opts.LegalHold != "" && !opts.LegalHold.IsValid() {
return ErrInvalidArgument(opts.LegalHold.String() + " unsupported legal-hold status")
}
return nil
}
@@ -153,6 +172,10 @@ func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].Part
// be uploaded through this operation will be 5TiB.
func (c Client) PutObject(bucketName, objectName string, reader io.Reader, objectSize int64,
opts PutObjectOptions) (n int64, err error) {
if objectSize < 0 && opts.DisableMultipart {
return 0, errors.New("object size must be provided with disable multipart upload")
}
return c.PutObjectWithContext(context.Background(), bucketName, objectName, reader, objectSize, opts)
}
@@ -164,8 +187,7 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri
// NOTE: Streaming signature is not supported by GCS.
if s3utils.IsGoogleEndpoint(*c.endpointURL) {
// Do not compute MD5 for Google Cloud Storage.
return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts)
return c.putObject(ctx, bucketName, objectName, reader, size, opts)
}
partSize := opts.PartSize
@@ -174,8 +196,8 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri
}
if c.overrideSignerType.IsV2() {
if size >= 0 && size < int64(partSize) {
return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts)
if size >= 0 && size < int64(partSize) || opts.DisableMultipart {
return c.putObject(ctx, bucketName, objectName, reader, size, opts)
}
return c.putObjectMultipart(ctx, bucketName, objectName, reader, size, opts)
}
@@ -183,11 +205,10 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri
return c.putObjectMultipartStreamNoLength(ctx, bucketName, objectName, reader, opts)
}
if size < int64(partSize) {
return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts)
if size < int64(partSize) || opts.DisableMultipart {
return c.putObject(ctx, bucketName, objectName, reader, size, opts)
}
// For all sizes greater than 128MiB do multipart.
return c.putObjectMultipartStream(ctx, bucketName, objectName, reader, size, opts)
}
@@ -243,13 +264,21 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName
return 0, rerr
}
var md5Base64 string
if opts.SendContentMd5 {
// Calculate md5sum.
hash := md5.New()
hash.Write(buf[:length])
md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil))
}
// Update progress reader appropriately to the latest offset
// as we read from the source.
rd := newHook(bytes.NewReader(buf[:length]), opts.Progress)
// Proceed to upload the part.
objPart, uerr := c.uploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber,
"", "", int64(length), opts.ServerSideEncryption)
md5Base64, "", int64(length), opts.ServerSideEncryption)
if uerr != nil {
return totalUploadedSize, uerr
}

180
vendor/github.com/minio/minio-go/v6/api-remove.go сгенерированный поставляемый
Просмотреть файл

@@ -63,7 +63,7 @@ func (c Client) RemoveObject(bucketName, objectName string) error {
return c.RemoveObjectWithOptions(bucketName, objectName, RemoveObjectOptions{})
}
// RemoveObjectOptions represents options specified by user for PutObject call
// RemoveObjectOptions represents options specified by user for RemoveObject call
type RemoveObjectOptions struct {
GovernanceBypass bool
VersionID string
@@ -92,7 +92,7 @@ func (c Client) RemoveObjectWithOptions(bucketName, objectName string, opts Remo
if opts.GovernanceBypass {
// Set the bypass goverenance retention header
headers.Set("x-amz-bypass-governance-retention", "True")
headers.Set(amzBypassGovernance, "true")
}
// Execute DELETE on objectName.
resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{
@@ -179,73 +179,109 @@ func (c Client) RemoveObjectsWithContext(ctx context.Context, bucketName string,
return errorCh
}
// Generate and call MultiDelete S3 requests based on entries received from objectsCh
go func(errorCh chan<- RemoveObjectError) {
maxEntries := 1000
finish := false
urlValues := make(url.Values)
urlValues.Set("delete", "")
// Close error channel when Multi delete finishes.
defer close(errorCh)
// Loop over entries by 1000 and call MultiDelete requests
for {
if finish {
break
}
count := 0
var batch []string
// Try to gather 1000 entries
for object := range objectsCh {
batch = append(batch, object)
if count++; count >= maxEntries {
break
}
}
if count == 0 {
// Multi Objects Delete API doesn't accept empty object list, quit immediately
break
}
if count < maxEntries {
// We didn't have 1000 entries, so this is the last batch
finish = true
}
// Generate remove multi objects XML request
removeBytes := generateRemoveMultiObjectsRequest(batch)
// Execute GET on bucket to list objects.
resp, err := c.executeMethod(ctx, "POST", requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
contentBody: bytes.NewReader(removeBytes),
contentLength: int64(len(removeBytes)),
contentMD5Base64: sumMD5Base64(removeBytes),
contentSHA256Hex: sum256Hex(removeBytes),
})
if resp != nil {
if resp.StatusCode != http.StatusOK {
e := httpRespToErrorResponse(resp, bucketName, "")
errorCh <- RemoveObjectError{ObjectName: "", Err: e}
}
}
if err != nil {
for _, b := range batch {
errorCh <- RemoveObjectError{ObjectName: b, Err: err}
}
continue
}
// Process multiobjects remove xml response
processRemoveMultiObjectsResponse(resp.Body, batch, errorCh)
closeResponse(resp)
}
}(errorCh)
go c.removeObjects(ctx, bucketName, objectsCh, errorCh, RemoveObjectsOptions{})
return errorCh
}
// RemoveObjectsWithOptionsContext - Identical to RemoveObjects call, but accepts context to
// facilitate request cancellation and options to bypass governance retention
func (c Client) RemoveObjectsWithOptionsContext(ctx context.Context, bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError {
errorCh := make(chan RemoveObjectError, 1)
// Validate if bucket name is valid.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
defer close(errorCh)
errorCh <- RemoveObjectError{
Err: err,
}
return errorCh
}
// Validate objects channel to be properly allocated.
if objectsCh == nil {
defer close(errorCh)
errorCh <- RemoveObjectError{
Err: ErrInvalidArgument("Objects channel cannot be nil"),
}
return errorCh
}
go c.removeObjects(ctx, bucketName, objectsCh, errorCh, opts)
return errorCh
}
// Generate and call MultiDelete S3 requests based on entries received from objectsCh
func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh <-chan string, errorCh chan<- RemoveObjectError, opts RemoveObjectsOptions) {
maxEntries := 1000
finish := false
urlValues := make(url.Values)
urlValues.Set("delete", "")
// Close error channel when Multi delete finishes.
defer close(errorCh)
// Loop over entries by 1000 and call MultiDelete requests
for {
if finish {
break
}
count := 0
var batch []string
// Try to gather 1000 entries
for object := range objectsCh {
batch = append(batch, object)
if count++; count >= maxEntries {
break
}
}
if count == 0 {
// Multi Objects Delete API doesn't accept empty object list, quit immediately
break
}
if count < maxEntries {
// We didn't have 1000 entries, so this is the last batch
finish = true
}
// Build headers.
headers := make(http.Header)
if opts.GovernanceBypass {
// Set the bypass goverenance retention header
headers.Set(amzBypassGovernance, "true")
}
// Generate remove multi objects XML request
removeBytes := generateRemoveMultiObjectsRequest(batch)
// Execute GET on bucket to list objects.
resp, err := c.executeMethod(ctx, http.MethodPost, requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
contentBody: bytes.NewReader(removeBytes),
contentLength: int64(len(removeBytes)),
contentMD5Base64: sumMD5Base64(removeBytes),
contentSHA256Hex: sum256Hex(removeBytes),
customHeader: headers,
})
if resp != nil {
if resp.StatusCode != http.StatusOK {
e := httpRespToErrorResponse(resp, bucketName, "")
errorCh <- RemoveObjectError{ObjectName: "", Err: e}
}
}
if err != nil {
for _, b := range batch {
errorCh <- RemoveObjectError{ObjectName: b, Err: err}
}
continue
}
// Process multiobjects remove xml response
processRemoveMultiObjectsResponse(resp.Body, batch, errorCh)
closeResponse(resp)
}
}
// RemoveObjects removes multiple objects from a bucket.
// The list of objects to remove are received from objectsCh.
// Remove failures are sent back via error channel.
@@ -253,6 +289,18 @@ func (c Client) RemoveObjects(bucketName string, objectsCh <-chan string) <-chan
return c.RemoveObjectsWithContext(context.Background(), bucketName, objectsCh)
}
// RemoveObjectsOptions represents options specified by user for RemoveObjects call
type RemoveObjectsOptions struct {
GovernanceBypass bool
}
// RemoveObjectsWithOptions removes multiple objects from a bucket.
// The list of objects to remove are received from objectsCh.
// Remove failures are sent back via error channel.
func (c Client) RemoveObjectsWithOptions(bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError {
return c.RemoveObjectsWithOptionsContext(context.Background(), bucketName, objectsCh, opts)
}
// RemoveIncompleteUpload aborts an partially uploaded object.
func (c Client) RemoveIncompleteUpload(bucketName, objectName string) error {
// Input validation.

109
vendor/github.com/minio/minio-go/v6/api-stat.go сгенерированный поставляемый
Просмотреть файл

@@ -20,9 +20,6 @@ package minio
import (
"context"
"net/http"
"strconv"
"strings"
"time"
"github.com/minio/minio-go/v6/pkg/s3utils"
)
@@ -64,39 +61,6 @@ func (c Client) BucketExistsWithContext(ctx context.Context, bucketName string)
return true, nil
}
// List of header keys to be filtered, usually
// from all S3 API http responses.
var defaultFilterKeys = []string{
"Connection",
"Transfer-Encoding",
"Accept-Ranges",
"Date",
"Server",
"Vary",
"x-amz-bucket-region",
"x-amz-request-id",
"x-amz-id-2",
"Content-Security-Policy",
"X-Xss-Protection",
amzTaggingHeaderCount,
// Add new headers to be ignored.
}
// Extract only necessary metadata header key/values by
// filtering them out with a list of custom header keys.
func extractObjMetadata(header http.Header) http.Header {
filterKeys := append([]string{
"ETag",
"Content-Length",
"Last-Modified",
"Content-Type",
"Expires",
"X-Cache",
"X-Cache-Lookup",
}, defaultFilterKeys...)
return filterHeader(header, filterKeys)
}
// StatObject verifies if object exists and you have permission to access.
func (c Client) StatObject(bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) {
return c.StatObjectWithContext(context.Background(), bucketName, objectName, opts)
@@ -142,76 +106,5 @@ func (c Client) statObject(ctx context.Context, bucketName, objectName string, o
}
}
// Trim off the odd double quotes from ETag in the beginning and end.
md5sum := trimEtag(resp.Header.Get("ETag"))
// Parse content length is exists
var size int64 = -1
contentLengthStr := resp.Header.Get("Content-Length")
if contentLengthStr != "" {
size, err = strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
// Content-Length is not valid
return ObjectInfo{}, ErrorResponse{
Code: "InternalError",
Message: "Content-Length is invalid. " + reportIssue,
BucketName: bucketName,
Key: objectName,
RequestID: resp.Header.Get("x-amz-request-id"),
HostID: resp.Header.Get("x-amz-id-2"),
Region: resp.Header.Get("x-amz-bucket-region"),
}
}
}
// Parse Last-Modified has http time format.
date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified"))
if err != nil {
return ObjectInfo{}, ErrorResponse{
Code: "InternalError",
Message: "Last-Modified time format is invalid. " + reportIssue,
BucketName: bucketName,
Key: objectName,
RequestID: resp.Header.Get("x-amz-request-id"),
HostID: resp.Header.Get("x-amz-id-2"),
Region: resp.Header.Get("x-amz-bucket-region"),
}
}
// Fetch content type if any present.
contentType := strings.TrimSpace(resp.Header.Get("Content-Type"))
if contentType == "" {
contentType = "application/octet-stream"
}
expiryStr := resp.Header.Get("Expires")
var expTime time.Time
if t, err := time.Parse(http.TimeFormat, expiryStr); err == nil {
expTime = t.UTC()
}
metadata := extractObjMetadata(resp.Header)
userMetadata := map[string]string{}
const xamzmeta = "x-amz-meta-"
const xamzmetaLen = len(xamzmeta)
for k, v := range metadata {
if strings.HasPrefix(strings.ToLower(k), xamzmeta) {
userMetadata[k[xamzmetaLen:]] = v[0]
}
}
// Save object metadata info.
return ObjectInfo{
ETag: md5sum,
Key: objectName,
Size: size,
LastModified: date,
ContentType: contentType,
Expires: expTime,
// Extract only the relevant header keys describing the object.
// following function filters out a list of standard set of keys
// which are not part of object metadata.
Metadata: metadata,
UserMetadata: userMetadata,
}, nil
return ToObjectInfo(bucketName, objectName, resp.Header)
}

55
vendor/github.com/minio/minio-go/v6/api.go сгенерированный поставляемый
Просмотреть файл

@@ -43,8 +43,8 @@ import (
"golang.org/x/net/publicsuffix"
"github.com/minio/minio-go/v6/pkg/credentials"
"github.com/minio/minio-go/v6/pkg/s3signer"
"github.com/minio/minio-go/v6/pkg/s3utils"
"github.com/minio/minio-go/v6/pkg/signer"
)
// Client implements Amazon S3 compatible methods.
@@ -104,7 +104,7 @@ type Options struct {
// Global constants.
const (
libraryName = "minio-go"
libraryVersion = "v6.0.45"
libraryVersion = "v6.0.53"
)
// User Agent should always following the below style.
@@ -271,7 +271,7 @@ func (c *Client) redirectHeaders(req *http.Request, via []*http.Request) error {
case signerType.IsV2():
return errors.New("signature V2 cannot support redirection")
case signerType.IsV4():
s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, getDefaultLocation(*c.endpointURL, region))
signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, getDefaultLocation(*c.endpointURL, region))
}
}
return nil
@@ -413,7 +413,7 @@ func (c *Client) SetS3TransferAccelerate(accelerateEndpoint string) {
// - For signature v4 request if the connection is insecure compute only sha256.
// - For signature v4 request if the connection is secure compute only md5.
// - For anonymous request compute md5.
func (c *Client) hashMaterials() (hashAlgos map[string]hash.Hash, hashSums map[string][]byte) {
func (c *Client) hashMaterials(isMd5Requested bool) (hashAlgos map[string]hash.Hash, hashSums map[string][]byte) {
hashSums = make(map[string][]byte)
hashAlgos = make(map[string]hash.Hash)
if c.overrideSignerType.IsV4() {
@@ -427,6 +427,9 @@ func (c *Client) hashMaterials() (hashAlgos map[string]hash.Hash, hashSums map[s
hashAlgos["md5"] = md5.New()
}
}
if isMd5Requested {
hashAlgos["md5"] = md5.New()
}
return hashAlgos, hashSums
}
@@ -588,16 +591,16 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque
}
}
// Create a done channel to control 'newRetryTimer' go routine.
doneCh := make(chan struct{}, 1)
// Create cancel context to control 'newRetryTimer' go routine.
retryCtx, cancel := context.WithCancel(ctx)
// Indicate to our routine to exit cleanly upon return.
defer close(doneCh)
defer cancel()
// Blank indentifier is kept here on purpose since 'range' without
// blank identifiers is only supported since go1.4
// https://golang.org/doc/go1.4#forrange.
for range c.newRetryTimer(reqRetry, DefaultRetryUnit, DefaultRetryCap, MaxJitter, doneCh) {
for range c.newRetryTimer(retryCtx, reqRetry, DefaultRetryUnit, DefaultRetryCap, MaxJitter) {
// Retry executes the following function body if request has an
// error until maxRetries have been exhausted, retry attempts are
// performed after waiting for a given period of time in a
@@ -627,6 +630,9 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque
// Initiate the request.
res, err = c.do(req)
if err != nil {
if err == context.Canceled || err == context.DeadlineExceeded {
return nil, err
}
continue
}
@@ -669,20 +675,23 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque
case "InvalidRegion":
fallthrough
case "AccessDenied":
if metadata.bucketName != "" && errResponse.Region != "" {
if errResponse.Region == "" {
// Region is empty we simply return the error.
return res, err
}
// Region is not empty figure out a way to
// handle this appropriately.
if metadata.bucketName != "" {
// Gather Cached location only if bucketName is present.
if _, cachedOk := c.bucketLocCache.Get(metadata.bucketName); cachedOk {
c.bucketLocCache.Set(metadata.bucketName, errResponse.Region)
continue // Retry.
}
} else {
// Most probably for ListBuckets()
// This is for ListBuckets() fallback.
if errResponse.Region != metadata.bucketLocation {
// Retry if the error
// response has a
// different region
// than the request we
// just made.
// Retry if the error response has a different region
// than the request we just made.
metadata.bucketLocation = errResponse.Region
continue // Retry
}
@@ -703,6 +712,12 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque
// For all other cases break out of the retry loop.
break
}
// Return an error when retry is canceled or deadlined
if e := retryCtx.Err(); e != nil {
return nil, e
}
return res, err
}
@@ -777,10 +792,10 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R
}
if signerType.IsV2() {
// Presign URL with signature v2.
req = s3signer.PreSignV2(*req, accessKeyID, secretAccessKey, metadata.expires, isVirtualHost)
req = signer.PreSignV2(*req, accessKeyID, secretAccessKey, metadata.expires, isVirtualHost)
} else if signerType.IsV4() {
// Presign URL with signature v4.
req = s3signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires)
req = signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires)
}
return req, nil
}
@@ -823,12 +838,12 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R
switch {
case signerType.IsV2():
// Add signature version '2' authorization header.
req = s3signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualHost)
req = signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualHost)
case metadata.objectName != "" && metadata.queryValues == nil && method == "PUT" && metadata.customHeader.Get("X-Amz-Copy-Source") == "" && !c.secure:
// Streaming signature is used by default for a PUT object request. Additionally we also
// look if the initialized client is secure, if yes then we don't need to perform
// streaming signature.
req = s3signer.StreamingSignV4(req, accessKeyID,
req = signer.StreamingSignV4(req, accessKeyID,
secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC())
default:
// Set sha256 sum for signature calculation only with signature version '4'.
@@ -839,7 +854,7 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R
req.Header.Set("X-Amz-Content-Sha256", shaHeader)
// Add signature version '4' authorization header.
req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, location)
req = signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, location)
}
// Return request.

22
vendor/github.com/minio/minio-go/v6/bucket-cache.go сгенерированный поставляемый
Просмотреть файл

@@ -25,8 +25,8 @@ import (
"sync"
"github.com/minio/minio-go/v6/pkg/credentials"
"github.com/minio/minio-go/v6/pkg/s3signer"
"github.com/minio/minio-go/v6/pkg/s3utils"
"github.com/minio/minio-go/v6/pkg/signer"
)
// bucketLocationCache - Provides simple mechanism to hold bucket
@@ -183,11 +183,21 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro
}
}
targetURL.Path = path.Join(bucketName, "") + "/"
targetURL.RawQuery = urlValues.Encode()
isVirtualHost := s3utils.IsVirtualHostSupported(targetURL, bucketName)
var urlStr string
//only suport Aliyun OSS for virtual hosted path, compatible Amazon & Google Endpoint
if isVirtualHost && s3utils.IsAliyunOSSEndpoint(targetURL) {
urlStr = c.endpointURL.Scheme + "://" + bucketName + "." + targetURL.Host + "/?location"
} else {
targetURL.Path = path.Join(bucketName, "") + "/"
targetURL.RawQuery = urlValues.Encode()
urlStr = targetURL.String()
}
// Get a new HTTP request for the method.
req, err := http.NewRequest("GET", targetURL.String(), nil)
req, err := http.NewRequest("GET", urlStr, nil)
if err != nil {
return nil, err
}
@@ -226,7 +236,7 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro
if signerType.IsV2() {
// Get Bucket Location calls should be always path style
isVirtualHost := false
req = s3signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualHost)
req = signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualHost)
return req, nil
}
@@ -237,6 +247,6 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro
}
req.Header.Set("X-Amz-Content-Sha256", contentSha256)
req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1")
req = signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1")
return req, nil
}

116
vendor/github.com/minio/minio-go/v6/bucket-notification.go сгенерированный поставляемый
Просмотреть файл

@@ -1,6 +1,6 @@
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2015-2017 MinIO, Inc.
* Copyright 2015-2020 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,6 +19,8 @@ package minio
import (
"encoding/xml"
"errors"
"fmt"
"github.com/minio/minio-go/v6/pkg/set"
)
@@ -81,7 +83,7 @@ func NewArn(partition, service, region, accountID, resource string) Arn {
Resource: resource}
}
// Return the string format of the ARN
// String returns the string format of the ARN
func (arn Arn) String() string {
return "arn:" + arn.Partition + ":" + arn.Service + ":" + arn.Region + ":" + arn.AccountID + ":" + arn.Resource
}
@@ -137,6 +139,62 @@ func (t *NotificationConfig) AddFilterPrefix(prefix string) {
t.Filter.S3Key.FilterRules = append(t.Filter.S3Key.FilterRules, newFilterRule)
}
// EqualNotificationEventTypeList tells whether a and b contain the same events
func EqualNotificationEventTypeList(a, b []NotificationEventType) bool {
if len(a) != len(b) {
return false
}
setA := set.NewStringSet()
for _, i := range a {
setA.Add(string(i))
}
setB := set.NewStringSet()
for _, i := range b {
setB.Add(string(i))
}
return setA.Difference(setB).IsEmpty()
}
// EqualFilterRuleList tells whether a and b contain the same filters
func EqualFilterRuleList(a, b []FilterRule) bool {
if len(a) != len(b) {
return false
}
setA := set.NewStringSet()
for _, i := range a {
setA.Add(fmt.Sprintf("%s-%s", i.Name, i.Value))
}
setB := set.NewStringSet()
for _, i := range b {
setB.Add(fmt.Sprintf("%s-%s", i.Name, i.Value))
}
return setA.Difference(setB).IsEmpty()
}
// Equal returns whether this `NotificationConfig` is equal to another defined by the passed parameters
func (t *NotificationConfig) Equal(events []NotificationEventType, prefix, suffix string) bool {
//Compare events
passEvents := EqualNotificationEventTypeList(t.Events, events)
//Compare filters
var newFilter []FilterRule
if prefix != "" {
newFilter = append(newFilter, FilterRule{Name: "prefix", Value: prefix})
}
if suffix != "" {
newFilter = append(newFilter, FilterRule{Name: "suffix", Value: suffix})
}
passFilters := EqualFilterRuleList(t.Filter.S3Key.FilterRules, newFilter)
// if it matches events and filters, mark the index for deletion
return passEvents && passFilters
}
// TopicConfig carries one single topic notification configuration
type TopicConfig struct {
NotificationConfig
@@ -250,6 +308,26 @@ func (b *BucketNotification) RemoveTopicByArn(arn Arn) {
b.TopicConfigs = topics
}
// ErrNoNotificationConfigMatch is returned when a notification configuration (sqs,sns,lambda) is not found when trying to delete
var ErrNoNotificationConfigMatch = errors.New("no notification configuration matched")
// RemoveTopicByArnEventsPrefixSuffix removes a topic configuration that match the exact specified ARN, events, prefix and suffix
func (b *BucketNotification) RemoveTopicByArnEventsPrefixSuffix(arn Arn, events []NotificationEventType, prefix, suffix string) error {
removeIndex := -1
for i, v := range b.TopicConfigs {
// if it matches events and filters, mark the index for deletion
if v.Topic == arn.String() && v.NotificationConfig.Equal(events, prefix, suffix) {
removeIndex = i
break // since we have at most one matching config
}
}
if removeIndex >= 0 {
b.TopicConfigs = append(b.TopicConfigs[:removeIndex], b.TopicConfigs[removeIndex+1:]...)
return nil
}
return ErrNoNotificationConfigMatch
}
// RemoveQueueByArn removes all queue configurations that match the exact specified ARN
func (b *BucketNotification) RemoveQueueByArn(arn Arn) {
var queues []QueueConfig
@@ -261,6 +339,23 @@ func (b *BucketNotification) RemoveQueueByArn(arn Arn) {
b.QueueConfigs = queues
}
// RemoveQueueByArnEventsPrefixSuffix removes a queue configuration that match the exact specified ARN, events, prefix and suffix
func (b *BucketNotification) RemoveQueueByArnEventsPrefixSuffix(arn Arn, events []NotificationEventType, prefix, suffix string) error {
removeIndex := -1
for i, v := range b.QueueConfigs {
// if it matches events and filters, mark the index for deletion
if v.Queue == arn.String() && v.NotificationConfig.Equal(events, prefix, suffix) {
removeIndex = i
break // since we have at most one matching config
}
}
if removeIndex >= 0 {
b.QueueConfigs = append(b.QueueConfigs[:removeIndex], b.QueueConfigs[removeIndex+1:]...)
return nil
}
return ErrNoNotificationConfigMatch
}
// RemoveLambdaByArn removes all lambda configurations that match the exact specified ARN
func (b *BucketNotification) RemoveLambdaByArn(arn Arn) {
var lambdas []LambdaConfig
@@ -271,3 +366,20 @@ func (b *BucketNotification) RemoveLambdaByArn(arn Arn) {
}
b.LambdaConfigs = lambdas
}
// RemoveLambdaByArnEventsPrefixSuffix removes a topic configuration that match the exact specified ARN, events, prefix and suffix
func (b *BucketNotification) RemoveLambdaByArnEventsPrefixSuffix(arn Arn, events []NotificationEventType, prefix, suffix string) error {
removeIndex := -1
for i, v := range b.LambdaConfigs {
// if it matches events and filters, mark the index for deletion
if v.Lambda == arn.String() && v.NotificationConfig.Equal(events, prefix, suffix) {
removeIndex = i
break // since we have at most one matching config
}
}
if removeIndex >= 0 {
b.LambdaConfigs = append(b.LambdaConfigs[:removeIndex], b.LambdaConfigs[removeIndex+1:]...)
return nil
}
return ErrNoNotificationConfigMatch
}

25
vendor/github.com/minio/minio-go/v6/constants.go сгенерированный поставляемый
Просмотреть файл

@@ -55,13 +55,22 @@ const (
iso8601DateFormat = "20060102T150405Z"
)
// Storage class header constant.
const amzStorageClass = "X-Amz-Storage-Class"
const (
// Storage class header.
amzStorageClass = "X-Amz-Storage-Class"
// Website redirect location header constant
const amzWebsiteRedirectLocation = "X-Amz-Website-Redirect-Location"
// Website redirect location header
amzWebsiteRedirectLocation = "X-Amz-Website-Redirect-Location"
// Tagging header constants
const amzTaggingHeader = "X-Amz-Tagging"
const amzTaggingHeaderDirective = "X-Amz-Tagging-Directive"
const amzTaggingHeaderCount = "X-Amz-Tagging-Count"
// Object Tagging headers
amzTaggingHeader = "X-Amz-Tagging"
amzTaggingHeaderDirective = "X-Amz-Tagging-Directive"
// Object legal hold header
amzLegalHoldHeader = "X-Amz-Object-Lock-Legal-Hold"
// Object retention header
amzLockMode = "X-Amz-Object-Lock-Mode"
amzLockRetainUntil = "X-Amz-Object-Lock-Retain-Until-Date"
amzBypassGovernance = "X-Amz-Bypass-Governance-Retention"
)

2
vendor/github.com/minio/minio-go/v6/go.mod сгенерированный поставляемый
Просмотреть файл

@@ -6,7 +6,7 @@ require (
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/minio/sha256-simd v0.1.1
github.com/mitchellh/go-homedir v1.1.0
github.com/sirupsen/logrus v1.4.2 // indirect
github.com/sirupsen/logrus v1.5.0 // indirect
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect
golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f
golang.org/x/net v0.0.0-20190522155817-f3200d17e092

8
vendor/github.com/minio/minio-go/v6/go.sum сгенерированный поставляемый
Просмотреть файл

@@ -1,4 +1,3 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@@ -11,16 +10,13 @@ github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKU
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q=
github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo=

214
vendor/github.com/minio/minio-go/v6/pkg/credentials/assume_role.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,214 @@
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2020 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package credentials
import (
"encoding/hex"
"encoding/xml"
"errors"
"io"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/minio/minio-go/v6/pkg/signer"
sha256 "github.com/minio/sha256-simd"
)
// AssumeRoleResponse contains the result of successful AssumeRole request.
type AssumeRoleResponse struct {
XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleResponse" json:"-"`
Result AssumeRoleResult `xml:"AssumeRoleResult"`
ResponseMetadata struct {
RequestID string `xml:"RequestId,omitempty"`
} `xml:"ResponseMetadata,omitempty"`
}
// AssumeRoleResult - Contains the response to a successful AssumeRole
// request, including temporary credentials that can be used to make
// MinIO API requests.
type AssumeRoleResult struct {
// The identifiers for the temporary security credentials that the operation
// returns.
AssumedRoleUser AssumedRoleUser `xml:",omitempty"`
// The temporary security credentials, which include an access key ID, a secret
// access key, and a security (or session) token.
//
// Note: The size of the security token that STS APIs return is not fixed. We
// strongly recommend that you make no assumptions about the maximum size. As
// of this writing, the typical size is less than 4096 bytes, but that can vary.
// Also, future updates to AWS might require larger sizes.
Credentials struct {
AccessKey string `xml:"AccessKeyId" json:"accessKey,omitempty"`
SecretKey string `xml:"SecretAccessKey" json:"secretKey,omitempty"`
Expiration time.Time `xml:"Expiration" json:"expiration,omitempty"`
SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty"`
} `xml:",omitempty"`
// A percentage value that indicates the size of the policy in packed form.
// The service rejects any policy with a packed size greater than 100 percent,
// which means the policy exceeded the allowed space.
PackedPolicySize int `xml:",omitempty"`
}
// A STSAssumeRole retrieves credentials from MinIO service, and keeps track if
// those credentials are expired.
type STSAssumeRole struct {
Expiry
// Required http Client to use when connecting to MinIO STS service.
Client *http.Client
// STS endpoint to fetch STS credentials.
STSEndpoint string
// various options for this request.
Options STSAssumeRoleOptions
}
// STSAssumeRoleOptions collection of various input options
// to obtain AssumeRole credentials.
type STSAssumeRoleOptions struct {
// Mandatory inputs.
AccessKey string
SecretKey string
Location string // Optional commonly needed with AWS STS.
DurationSeconds int // Optional defaults to 1 hour.
// Optional only valid if using with AWS STS
RoleARN string
RoleSessionName string
}
// NewSTSAssumeRole returns a pointer to a new
// Credentials object wrapping the STSAssumeRole.
func NewSTSAssumeRole(stsEndpoint string, opts STSAssumeRoleOptions) (*Credentials, error) {
if stsEndpoint == "" {
return nil, errors.New("STS endpoint cannot be empty")
}
if opts.AccessKey == "" || opts.SecretKey == "" {
return nil, errors.New("AssumeRole credentials access/secretkey is mandatory")
}
return New(&STSAssumeRole{
Client: &http.Client{
Transport: http.DefaultTransport,
},
STSEndpoint: stsEndpoint,
Options: opts,
}), nil
}
const defaultDurationSeconds = 3600
// closeResponse close non nil response with any response Body.
// convenient wrapper to drain any remaining data on response body.
//
// Subsequently this allows golang http RoundTripper
// to re-use the same connection for future requests.
func closeResponse(resp *http.Response) {
// Callers should close resp.Body when done reading from it.
// If resp.Body is not closed, the Client's underlying RoundTripper
// (typically Transport) may not be able to re-use a persistent TCP
// connection to the server for a subsequent "keep-alive" request.
if resp != nil && resp.Body != nil {
// Drain any remaining Body and then close the connection.
// Without this closing connection would disallow re-using
// the same connection for future uses.
// - http://stackoverflow.com/a/17961593/4465767
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
}
}
func getAssumeRoleCredentials(clnt *http.Client, endpoint string, opts STSAssumeRoleOptions) (AssumeRoleResponse, error) {
v := url.Values{}
v.Set("Action", "AssumeRole")
v.Set("Version", "2011-06-15")
if opts.RoleARN != "" {
v.Set("RoleArn", opts.RoleARN)
}
if opts.RoleSessionName != "" {
v.Set("RoleSessionName", opts.RoleSessionName)
}
if opts.DurationSeconds > defaultDurationSeconds {
v.Set("DurationSeconds", strconv.Itoa(opts.DurationSeconds))
} else {
v.Set("DurationSeconds", strconv.Itoa(defaultDurationSeconds))
}
u, err := url.Parse(endpoint)
if err != nil {
return AssumeRoleResponse{}, err
}
u.Path = "/"
postBody := strings.NewReader(v.Encode())
hash := sha256.New()
if _, err = io.Copy(hash, postBody); err != nil {
return AssumeRoleResponse{}, err
}
postBody.Seek(0, 0)
req, err := http.NewRequest("POST", u.String(), postBody)
if err != nil {
return AssumeRoleResponse{}, err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(hash.Sum(nil)))
req = signer.SignV4STS(*req, opts.AccessKey, opts.SecretKey, opts.Location)
resp, err := clnt.Do(req)
if err != nil {
return AssumeRoleResponse{}, err
}
defer closeResponse(resp)
if resp.StatusCode != http.StatusOK {
return AssumeRoleResponse{}, errors.New(resp.Status)
}
a := AssumeRoleResponse{}
if err = xml.NewDecoder(resp.Body).Decode(&a); err != nil {
return AssumeRoleResponse{}, err
}
return a, nil
}
// Retrieve retrieves credentials from the MinIO service.
// Error will be returned if the request fails.
func (m *STSAssumeRole) Retrieve() (Value, error) {
a, err := getAssumeRoleCredentials(m.Client, m.STSEndpoint, m.Options)
if err != nil {
return Value{}, err
}
// Expiry window is set to 10secs.
m.SetExpiration(a.Result.Credentials.Expiration, DefaultExpiryWindow)
return Value{
AccessKeyID: a.Result.Credentials.AccessKey,
SecretAccessKey: a.Result.Credentials.SecretKey,
SessionToken: a.Result.Credentials.SessionToken,
SignerType: SignatureV4,
}, nil
}

26
vendor/github.com/minio/minio-go/v6/pkg/credentials/file_aws_credentials.go сгенерированный поставляемый
Просмотреть файл

@@ -36,12 +36,12 @@ type FileAWSCredentials struct {
// env value is empty will default to current user's home directory.
// Linux/OSX: "$HOME/.aws/credentials"
// Windows: "%USERPROFILE%\.aws\credentials"
filename string
Filename string
// AWS Profile to extract credentials from the shared credentials file. If empty
// will default to environment variable "AWS_PROFILE" or "default" if
// environment variable is also not set.
profile string
Profile string
// retrieved states if the credentials have been successfully retrieved.
retrieved bool
@@ -51,34 +51,34 @@ type FileAWSCredentials struct {
// wrapping the Profile file provider.
func NewFileAWSCredentials(filename string, profile string) *Credentials {
return New(&FileAWSCredentials{
filename: filename,
profile: profile,
Filename: filename,
Profile: profile,
})
}
// Retrieve reads and extracts the shared credentials from the current
// users home directory.
func (p *FileAWSCredentials) Retrieve() (Value, error) {
if p.filename == "" {
p.filename = os.Getenv("AWS_SHARED_CREDENTIALS_FILE")
if p.filename == "" {
if p.Filename == "" {
p.Filename = os.Getenv("AWS_SHARED_CREDENTIALS_FILE")
if p.Filename == "" {
homeDir, err := homedir.Dir()
if err != nil {
return Value{}, err
}
p.filename = filepath.Join(homeDir, ".aws", "credentials")
p.Filename = filepath.Join(homeDir, ".aws", "credentials")
}
}
if p.profile == "" {
p.profile = os.Getenv("AWS_PROFILE")
if p.profile == "" {
p.profile = "default"
if p.Profile == "" {
p.Profile = os.Getenv("AWS_PROFILE")
if p.Profile == "" {
p.Profile = "default"
}
}
p.retrieved = false
iniProfile, err := loadProfile(p.filename, p.profile)
iniProfile, err := loadProfile(p.Filename, p.Profile)
if err != nil {
return Value{}, err
}

26
vendor/github.com/minio/minio-go/v6/pkg/credentials/file_minio_client.go сгенерированный поставляемый
Просмотреть файл

@@ -38,12 +38,12 @@ type FileMinioClient struct {
// env value is empty will default to current user's home directory.
// Linux/OSX: "$HOME/.mc/config.json"
// Windows: "%USERALIAS%\mc\config.json"
filename string
Filename string
// MinIO Alias to extract credentials from the shared credentials file. If empty
// will default to environment variable "MINIO_ALIAS" or "default" if
// environment variable is also not set.
alias string
Alias string
// retrieved states if the credentials have been successfully retrieved.
retrieved bool
@@ -53,39 +53,39 @@ type FileMinioClient struct {
// wrapping the Alias file provider.
func NewFileMinioClient(filename string, alias string) *Credentials {
return New(&FileMinioClient{
filename: filename,
alias: alias,
Filename: filename,
Alias: alias,
})
}
// Retrieve reads and extracts the shared credentials from the current
// users home directory.
func (p *FileMinioClient) Retrieve() (Value, error) {
if p.filename == "" {
if p.Filename == "" {
if value, ok := os.LookupEnv("MINIO_SHARED_CREDENTIALS_FILE"); ok {
p.filename = value
p.Filename = value
} else {
homeDir, err := homedir.Dir()
if err != nil {
return Value{}, err
}
p.filename = filepath.Join(homeDir, ".mc", "config.json")
p.Filename = filepath.Join(homeDir, ".mc", "config.json")
if runtime.GOOS == "windows" {
p.filename = filepath.Join(homeDir, "mc", "config.json")
p.Filename = filepath.Join(homeDir, "mc", "config.json")
}
}
}
if p.alias == "" {
p.alias = os.Getenv("MINIO_ALIAS")
if p.alias == "" {
p.alias = "s3"
if p.Alias == "" {
p.Alias = os.Getenv("MINIO_ALIAS")
if p.Alias == "" {
p.Alias = "s3"
}
}
p.retrieved = false
hostCfg, err := loadAlias(p.filename, p.alias)
hostCfg, err := loadAlias(p.Filename, p.Alias)
if err != nil {
return Value{}, err
}

6
vendor/github.com/minio/minio-go/v6/pkg/credentials/iam_aws.go сгенерированный поставляемый
Просмотреть файл

@@ -82,7 +82,7 @@ func (m *IAM) Retrieve() (Value, error) {
case len(os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE")) > 0:
if len(endpoint) == 0 {
if len(os.Getenv("AWS_REGION")) > 0 {
endpoint = "sts." + os.Getenv("AWS_REGION") + ".amazonaws.com"
endpoint = "https://sts." + os.Getenv("AWS_REGION") + ".amazonaws.com"
} else {
endpoint = defaultSTSRoleEndpoint
}
@@ -168,6 +168,10 @@ type ec2RoleCredRespBody struct {
// be sent to fetch the rolling access credentials.
// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
func getIAMRoleURL(endpoint string) (*url.URL, error) {
if endpoint == "" {
endpoint = defaultIAMRoleEndpoint
}
u, err := url.Parse(endpoint)
if err != nil {
return nil, err

79
vendor/github.com/minio/minio-go/v6/pkg/s3utils/utils.go сгенерированный поставляемый
Просмотреть файл

@@ -1,6 +1,6 @@
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2015-2017 MinIO, Inc.
* Copyright 2015-2020 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -78,22 +78,28 @@ func IsVirtualHostSupported(endpointURL url.URL, bucketName string) bool {
return false
}
// Return true for all other cases
return IsAmazonEndpoint(endpointURL) || IsGoogleEndpoint(endpointURL)
return IsAmazonEndpoint(endpointURL) || IsGoogleEndpoint(endpointURL) || IsAliyunOSSEndpoint(endpointURL)
}
// Refer for region styles - https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
// amazonS3HostHyphen - regular expression used to determine if an arg is s3 host in hyphenated style.
var amazonS3HostHyphen = regexp.MustCompile(`^s3-(.*?)\.amazonaws\.com$`)
var amazonS3HostHyphen = regexp.MustCompile(`^s3-(.*?).amazonaws.com$`)
// amazonS3HostDualStack - regular expression used to determine if an arg is s3 host dualstack.
var amazonS3HostDualStack = regexp.MustCompile(`^s3\.dualstack\.(.*?)\.amazonaws\.com$`)
var amazonS3HostDualStack = regexp.MustCompile(`^s3.dualstack.(.*?).amazonaws.com$`)
// amazonS3HostDot - regular expression used to determine if an arg is s3 host in . style.
var amazonS3HostDot = regexp.MustCompile(`^s3\.(.*?)\.amazonaws\.com$`)
var amazonS3HostDot = regexp.MustCompile(`^s3.(.*?).amazonaws.com$`)
// amazonS3ChinaHost - regular expression used to determine if the arg is s3 china host.
var amazonS3ChinaHost = regexp.MustCompile(`^s3\.(cn.*?)\.amazonaws\.com\.cn$`)
var amazonS3ChinaHost = regexp.MustCompile(`^s3.(cn.*?).amazonaws.com.cn$`)
// Regular expression used to determine if the arg is elb host.
var elbAmazonRegex = regexp.MustCompile(`elb(.*?).amazonaws.com$`)
// Regular expression used to determine if the arg is elb host in china.
var elbAmazonCnRegex = regexp.MustCompile(`elb(.*?).amazonaws.com.cn$`)
// GetRegionFromURL - returns a region from url host.
func GetRegionFromURL(endpointURL url.URL) string {
@@ -106,6 +112,10 @@ func GetRegionFromURL(endpointURL url.URL) string {
if IsAmazonGovCloudEndpoint(endpointURL) {
return "us-gov-west-1"
}
// if elb's are used we cannot calculate which region it may be, just return empty.
if elbAmazonRegex.MatchString(endpointURL.Host) || elbAmazonCnRegex.MatchString(endpointURL.Host) {
return ""
}
parts := amazonS3HostDualStack.FindStringSubmatch(endpointURL.Host)
if len(parts) > 1 {
return parts[1]
@@ -125,6 +135,11 @@ func GetRegionFromURL(endpointURL url.URL) string {
return ""
}
// IsAliyunOSSEndpoint - Match if it is exactly Aliyun OSS endpoint.
func IsAliyunOSSEndpoint(endpointURL url.URL) bool {
return strings.HasSuffix(endpointURL.Host, "aliyuncs.com")
}
// IsAmazonEndpoint - Match if it is exactly Amazon S3 endpoint.
func IsAmazonEndpoint(endpointURL url.URL) bool {
if endpointURL.Host == "s3-external-1.amazonaws.com" || endpointURL.Host == "s3.amazonaws.com" {
@@ -219,29 +234,39 @@ func QueryEncode(v url.Values) string {
return buf.String()
}
// TagDecode - decodes canonical tag into map of key and value.
func TagDecode(ctag string) map[string]string {
if ctag == "" {
return map[string]string{}
}
tags := strings.Split(ctag, "&")
tagMap := make(map[string]string, len(tags))
var err error
for _, tag := range tags {
kvs := strings.SplitN(tag, "=", 2)
if len(kvs) == 0 {
return map[string]string{}
}
if len(kvs) == 1 {
return map[string]string{}
}
tagMap[kvs[0]], err = url.PathUnescape(kvs[1])
if err != nil {
continue
}
}
return tagMap
}
// TagEncode - encodes tag values in their URL encoded form. In
// addition to the percent encoding performed by urlEncodePath() used
// here, it also percent encodes '/' (forward slash)
func TagEncode(tags map[string]string) string {
if tags == nil {
return ""
values := url.Values{}
for k, v := range tags {
values[k] = []string{v}
}
var buf bytes.Buffer
keys := make([]string, 0, len(tags))
for k := range tags {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
v := tags[k]
prefix := percentEncodeSlash(EncodePath(k)) + "="
if buf.Len() > 0 {
buf.WriteByte('&')
}
buf.WriteString(prefix)
buf.WriteString(percentEncodeSlash(EncodePath(v)))
}
return buf.String()
return QueryEncode(values)
}
// if object matches reserved string, no need to encode them
@@ -299,10 +324,10 @@ func checkBucketNameCommon(bucketName string, strict bool) (err error) {
return errors.New("Bucket name cannot be empty")
}
if len(bucketName) < 3 {
return errors.New("Bucket name cannot be smaller than 3 characters")
return errors.New("Bucket name cannot be shorter than 3 characters")
}
if len(bucketName) > 63 {
return errors.New("Bucket name cannot be greater than 63 characters")
return errors.New("Bucket name cannot be longer than 63 characters")
}
if ipAddress.MatchString(bucketName) {
return errors.New("Bucket name cannot be an ip address")
@@ -338,7 +363,7 @@ func CheckValidBucketNameStrict(bucketName string) (err error) {
// - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
func CheckValidObjectNamePrefix(objectName string) error {
if len(objectName) > 1024 {
return errors.New("Object name cannot be greater than 1024 characters")
return errors.New("Object name cannot be longer than 1024 characters")
}
if !utf8.ValidString(objectName) {
return errors.New("Object name with non UTF-8 strings are not supported")

Просмотреть файл

@@ -15,7 +15,7 @@
* limitations under the License.
*/
package s3signer
package signer
import (
"bytes"
@@ -82,7 +82,7 @@ func buildChunkStringToSign(t time.Time, region, previousSig string, chunkData [
stringToSignParts := []string{
streamingPayloadHdr,
t.Format(iso8601DateFormat),
getScope(region, t),
getScope(region, t, ServiceTypeS3),
previousSig,
emptySHA256,
hex.EncodeToString(sum256(chunkData)),
@@ -118,19 +118,19 @@ func buildChunkSignature(chunkData []byte, reqTime time.Time, region,
chunkStringToSign := buildChunkStringToSign(reqTime, region,
previousSignature, chunkData)
signingKey := getSigningKey(secretAccessKey, region, reqTime)
signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3)
return getSignature(signingKey, chunkStringToSign)
}
// getSeedSignature - returns the seed signature for a given request.
func (s *StreamingReader) setSeedSignature(req *http.Request) {
// Get canonical request
canonicalRequest := getCanonicalRequest(*req, ignoredStreamingHeaders)
canonicalRequest := getCanonicalRequest(*req, ignoredStreamingHeaders, getHashedPayload(*req))
// Get string to sign from canonical request.
stringToSign := getStringToSignV4(s.reqTime, s.region, canonicalRequest)
stringToSign := getStringToSignV4(s.reqTime, s.region, canonicalRequest, ServiceTypeS3)
signingKey := getSigningKey(s.secretAccessKey, s.region, s.reqTime)
signingKey := getSigningKey(s.secretAccessKey, s.region, s.reqTime, ServiceTypeS3)
// Calculate signature.
s.seedSignature = getSignature(signingKey, stringToSign)
@@ -185,7 +185,7 @@ func (s *StreamingReader) signChunk(chunkLen int) {
// setStreamingAuthHeader - builds and sets authorization header value
// for streaming signature.
func (s *StreamingReader) setStreamingAuthHeader(req *http.Request) {
credential := GetCredential(s.accessKeyID, s.region, s.reqTime)
credential := GetCredential(s.accessKeyID, s.region, s.reqTime, ServiceTypeS3)
authParts := []string{
signV4Algorithm + " Credential=" + credential,
"SignedHeaders=" + getSignedHeaders(*req, ignoredStreamingHeaders),

Просмотреть файл

@@ -15,7 +15,7 @@
* limitations under the License.
*/
package s3signer
package signer
import (
"bytes"

Просмотреть файл

@@ -15,7 +15,7 @@
* limitations under the License.
*/
package s3signer
package signer
import (
"bytes"
@@ -36,6 +36,12 @@ const (
yyyymmdd = "20060102"
)
// Different service types
const (
ServiceTypeS3 = "s3"
ServiceTypeSTS = "sts"
)
///
/// Excerpts from @lsegal -
/// https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258.
@@ -47,42 +53,21 @@ const (
/// by other agents) or when customers pass requests through
/// proxies, which may modify the user-agent.
///
/// Content-Length:
///
/// This is ignored from signing because generating a pre-signed
/// URL should not provide a content-length constraint,
/// specifically when vending a S3 pre-signed PUT URL. The
/// corollary to this is that when sending regular requests
/// (non-pre-signed), the signature contains a checksum of the
/// body, which implicitly validates the payload length (since
/// changing the number of bytes would change the checksum)
/// and therefore this header is not valuable in the signature.
///
/// Content-Type:
///
/// Signing this header causes quite a number of problems in
/// browser environments, where browsers like to modify and
/// normalize the content-type header in different ways. There is
/// more information on this in https://goo.gl/2E9gyy. Avoiding
/// this field simplifies logic and reduces the possibility of
/// future bugs.
///
/// Authorization:
///
/// Is skipped for obvious reasons
///
var v4IgnoredHeaders = map[string]bool{
"Authorization": true,
"Content-Type": true,
"Content-Length": true,
"User-Agent": true,
"Authorization": true,
"User-Agent": true,
}
// getSigningKey hmac seed to calculate final signature.
func getSigningKey(secret, loc string, t time.Time) []byte {
func getSigningKey(secret, loc string, t time.Time, serviceType string) []byte {
date := sumHMAC([]byte("AWS4"+secret), []byte(t.Format(yyyymmdd)))
location := sumHMAC(date, []byte(loc))
service := sumHMAC(location, []byte("s3"))
service := sumHMAC(location, []byte(serviceType))
signingKey := sumHMAC(service, []byte("aws4_request"))
return signingKey
}
@@ -94,19 +79,19 @@ func getSignature(signingKey []byte, stringToSign string) string {
// getScope generate a string of a specific date, an AWS region, and a
// service.
func getScope(location string, t time.Time) string {
func getScope(location string, t time.Time, serviceType string) string {
scope := strings.Join([]string{
t.Format(yyyymmdd),
location,
"s3",
serviceType,
"aws4_request",
}, "/")
return scope
}
// GetCredential generate a credential string.
func GetCredential(accessKeyID, location string, t time.Time) string {
scope := getScope(location, t)
func GetCredential(accessKeyID, location string, t time.Time, serviceType string) string {
scope := getScope(location, t, serviceType)
return accessKeyID + "/" + scope
}
@@ -184,7 +169,7 @@ func getSignedHeaders(req http.Request, ignoredHeaders map[string]bool) string {
// <CanonicalHeaders>\n
// <SignedHeaders>\n
// <HashedPayload>
func getCanonicalRequest(req http.Request, ignoredHeaders map[string]bool) string {
func getCanonicalRequest(req http.Request, ignoredHeaders map[string]bool, hashedPayload string) string {
req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1)
canonicalRequest := strings.Join([]string{
req.Method,
@@ -192,15 +177,15 @@ func getCanonicalRequest(req http.Request, ignoredHeaders map[string]bool) strin
req.URL.RawQuery,
getCanonicalHeaders(req, ignoredHeaders),
getSignedHeaders(req, ignoredHeaders),
getHashedPayload(req),
hashedPayload,
}, "\n")
return canonicalRequest
}
// getStringToSign a string based on selected query values.
func getStringToSignV4(t time.Time, location, canonicalRequest string) string {
func getStringToSignV4(t time.Time, location, canonicalRequest, serviceType string) string {
stringToSign := signV4Algorithm + "\n" + t.Format(iso8601DateFormat) + "\n"
stringToSign = stringToSign + getScope(location, t) + "\n"
stringToSign = stringToSign + getScope(location, t, serviceType) + "\n"
stringToSign = stringToSign + hex.EncodeToString(sum256([]byte(canonicalRequest)))
return stringToSign
}
@@ -217,7 +202,7 @@ func PreSignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, loc
t := time.Now().UTC()
// Get credential string.
credential := GetCredential(accessKeyID, location, t)
credential := GetCredential(accessKeyID, location, t, ServiceTypeS3)
// Get all signed headers.
signedHeaders := getSignedHeaders(req, v4IgnoredHeaders)
@@ -236,13 +221,13 @@ func PreSignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, loc
req.URL.RawQuery = query.Encode()
// Get canonical request.
canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders)
canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders, getHashedPayload(req))
// Get string to sign from canonical request.
stringToSign := getStringToSignV4(t, location, canonicalRequest)
stringToSign := getStringToSignV4(t, location, canonicalRequest, ServiceTypeS3)
// Gext hmac signing key.
signingKey := getSigningKey(secretAccessKey, location, t)
signingKey := getSigningKey(secretAccessKey, location, t, ServiceTypeS3)
// Calculate signature.
signature := getSignature(signingKey, stringToSign)
@@ -257,15 +242,19 @@ func PreSignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, loc
// requests.
func PostPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string {
// Get signining key.
signingkey := getSigningKey(secretAccessKey, location, t)
signingkey := getSigningKey(secretAccessKey, location, t, ServiceTypeS3)
// Calculate signature.
signature := getSignature(signingkey, policyBase64)
return signature
}
// SignV4 sign the request before Do(), in accordance with
// http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html.
func SignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string) *http.Request {
// SignV4STS - signature v4 for STS request.
func SignV4STS(req http.Request, accessKeyID, secretAccessKey, location string) *http.Request {
return signV4(req, accessKeyID, secretAccessKey, "", location, ServiceTypeSTS)
}
// Internal function called for different service types.
func signV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location, serviceType string) *http.Request {
// Signature calculation is not needed for anonymous credentials.
if accessKeyID == "" || secretAccessKey == "" {
return &req
@@ -282,17 +271,25 @@ func SignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, locati
req.Header.Set("X-Amz-Security-Token", sessionToken)
}
hashedPayload := getHashedPayload(req)
if serviceType == ServiceTypeSTS {
// Content sha256 header is not sent with the request
// but it is expected to have sha256 of payload for signature
// in STS service type request.
req.Header.Del("X-Amz-Content-Sha256")
}
// Get canonical request.
canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders)
canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders, hashedPayload)
// Get string to sign from canonical request.
stringToSign := getStringToSignV4(t, location, canonicalRequest)
stringToSign := getStringToSignV4(t, location, canonicalRequest, serviceType)
// Get hmac signing key.
signingKey := getSigningKey(secretAccessKey, location, t)
signingKey := getSigningKey(secretAccessKey, location, t, serviceType)
// Get credential string.
credential := GetCredential(accessKeyID, location, t)
credential := GetCredential(accessKeyID, location, t, serviceType)
// Get all signed headers.
signedHeaders := getSignedHeaders(req, v4IgnoredHeaders)
@@ -313,3 +310,9 @@ func SignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, locati
return &req
}
// SignV4 sign the request before Do(), in accordance with
// http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html.
func SignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string) *http.Request {
return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3)
}

Просмотреть файл

@@ -15,7 +15,7 @@
* limitations under the License.
*/
package s3signer
package signer
import (
"crypto/hmac"

2
vendor/github.com/minio/minio-go/v6/post-policy.go сгенерированный поставляемый
Просмотреть файл

@@ -258,7 +258,7 @@ func (p *PostPolicy) addNewPolicy(policyCond policyCondition) error {
return nil
}
// Stringer interface for printing policy in json formatted string.
// String function for printing policy in json formatted string.
func (p PostPolicy) String() string {
return string(p.marshalJSON())
}

14
vendor/github.com/minio/minio-go/v6/retry.go сгенерированный поставляемый
Просмотреть файл

@@ -18,6 +18,7 @@
package minio
import (
"context"
"net/http"
"time"
)
@@ -41,7 +42,7 @@ const DefaultRetryCap = time.Second * 30
// newRetryTimer creates a timer with exponentially increasing
// delays until the maximum retry attempts are reached.
func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duration, jitter float64, doneCh chan struct{}) <-chan int {
func (c Client) newRetryTimer(ctx context.Context, maxRetry int, unit time.Duration, cap time.Duration, jitter float64) <-chan int {
attemptCh := make(chan int)
// computes the exponential backoff duration according to
@@ -70,13 +71,16 @@ func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duratio
defer close(attemptCh)
for i := 0; i < maxRetry; i++ {
select {
// Attempts start from 1.
case attemptCh <- i + 1:
case <-doneCh:
// Stop the routine.
case <-ctx.Done():
return
}
select {
case <-time.After(exponentialBackoffWait(i)):
case <-ctx.Done():
return
}
time.Sleep(exponentialBackoffWait(i))
}
}()
return attemptCh

2
vendor/github.com/minio/minio-go/v6/s3-endpoints.go сгенерированный поставляемый
Просмотреть файл

@@ -35,6 +35,8 @@ var awsS3EndpointMap = map[string]string{
"ap-southeast-2": "s3.dualstack.ap-southeast-2.amazonaws.com",
"ap-northeast-1": "s3.dualstack.ap-northeast-1.amazonaws.com",
"ap-northeast-2": "s3.dualstack.ap-northeast-2.amazonaws.com",
"ap-northeast-3": "s3.dualstack.ap-northeast-3.amazonaws.com",
"me-south-1": "s3.dualstack.me-south-1.amazonaws.com",
"sa-east-1": "s3.dualstack.sa-east-1.amazonaws.com",
"us-gov-west-1": "s3.dualstack.us-gov-west-1.amazonaws.com",
"us-gov-east-1": "s3.dualstack.us-gov-east-1.amazonaws.com",

3
vendor/github.com/minio/minio-go/v6/transport.go сгенерированный поставляемый
Просмотреть файл

@@ -38,7 +38,8 @@ var DefaultTransport = func(secure bool) (http.RoundTripper, error) {
}).DialContext,
MaxIdleConns: 1024,
MaxIdleConnsPerHost: 1024,
IdleConnTimeout: 90 * time.Second,
ResponseHeaderTimeout: 60 * time.Second,
IdleConnTimeout: 60 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
// Set this value so that the underlying transport round-tripper

130
vendor/github.com/minio/minio-go/v6/utils.go сгенерированный поставляемый
Просмотреть файл

@@ -28,6 +28,7 @@ import (
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"time"
@@ -159,29 +160,122 @@ func isValidExpiry(expires time.Duration) error {
return nil
}
// make a copy of http.Header
func cloneHeader(h http.Header) http.Header {
h2 := make(http.Header, len(h))
for k, vv := range h {
vv2 := make([]string, len(vv))
copy(vv2, vv)
h2[k] = vv2
// Extract only necessary metadata header key/values by
// filtering them out with a list of custom header keys.
func extractObjMetadata(header http.Header) http.Header {
preserveKeys := []string{
"Content-Type",
"Cache-Control",
"Content-Encoding",
"Content-Language",
"Content-Disposition",
"X-Amz-Storage-Class",
"X-Amz-Object-Lock-Mode",
"X-Amz-Object-Lock-Retain-Until-Date",
"X-Amz-Object-Lock-Legal-Hold",
"X-Amz-Website-Redirect-Location",
"X-Amz-Server-Side-Encryption",
"X-Amz-Meta-",
// Add new headers to be preserved.
// if you add new headers here, please extend
// PutObjectOptions{} to preserve them
// upon upload as well.
}
return h2
}
// Filter relevant response headers from
// the HEAD, GET http response. The function takes
// a list of headers which are filtered out and
// returned as a new http header.
func filterHeader(header http.Header, filterKeys []string) (filteredHeader http.Header) {
filteredHeader = cloneHeader(header)
for _, key := range filterKeys {
filteredHeader.Del(key)
filteredHeader := make(http.Header)
for k, v := range header {
var found bool
for _, prefix := range preserveKeys {
if !strings.HasPrefix(k, prefix) {
continue
}
found = true
break
}
if found {
filteredHeader[k] = v
}
}
return filteredHeader
}
// ToObjectInfo converts http header values into ObjectInfo type,
// extracts metadata and fills in all the necessary fields in ObjectInfo.
func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectInfo, error) {
var err error
// Trim off the odd double quotes from ETag in the beginning and end.
etag := trimEtag(h.Get("ETag"))
// Parse content length is exists
var size int64 = -1
contentLengthStr := h.Get("Content-Length")
if contentLengthStr != "" {
size, err = strconv.ParseInt(contentLengthStr, 10, 64)
if err != nil {
// Content-Length is not valid
return ObjectInfo{}, ErrorResponse{
Code: "InternalError",
Message: "Content-Length is invalid. " + reportIssue,
BucketName: bucketName,
Key: objectName,
RequestID: h.Get("x-amz-request-id"),
HostID: h.Get("x-amz-id-2"),
Region: h.Get("x-amz-bucket-region"),
}
}
}
// Parse Last-Modified has http time format.
date, err := time.Parse(http.TimeFormat, h.Get("Last-Modified"))
if err != nil {
return ObjectInfo{}, ErrorResponse{
Code: "InternalError",
Message: "Last-Modified time format is invalid. " + reportIssue,
BucketName: bucketName,
Key: objectName,
RequestID: h.Get("x-amz-request-id"),
HostID: h.Get("x-amz-id-2"),
Region: h.Get("x-amz-bucket-region"),
}
}
// Fetch content type if any present.
contentType := strings.TrimSpace(h.Get("Content-Type"))
if contentType == "" {
contentType = "application/octet-stream"
}
expiryStr := h.Get("Expires")
var expTime time.Time
if t, err := time.Parse(http.TimeFormat, expiryStr); err == nil {
expTime = t.UTC()
}
metadata := extractObjMetadata(h)
userMetadata := make(map[string]string)
for k, v := range metadata {
if strings.HasPrefix(k, "X-Amz-Meta-") {
userMetadata[strings.TrimPrefix(k, "X-Amz-Meta-")] = v[0]
}
}
userTags := s3utils.TagDecode(h.Get(amzTaggingHeader))
// Save object metadata info.
return ObjectInfo{
ETag: etag,
Key: objectName,
Size: size,
LastModified: date,
ContentType: contentType,
Expires: expTime,
// Extract only the relevant header keys describing the object.
// following function filters out a list of standard set of keys
// which are not part of object metadata.
Metadata: metadata,
UserMetadata: userMetadata,
UserTags: userTags,
}, nil
}
// regCred matches credential string in HTTP header
var regCred = regexp.MustCompile("Credential=([A-Z0-9]+)/")