From c89c56ab2efcd6b08879695083682c58ea13a232 Mon Sep 17 00:00:00 2001 From: Claudio Costa Date: Wed, 12 Aug 2020 19:53:47 +0200 Subject: [PATCH] [MM-27328] Upgrade minio-go to v7 (#15177) * Upgrade minio-go to v7 * Update to v7.0.3 Co-authored-by: Mattermod --- api4/apitestlib.go | 19 +- go.mod | 11 +- go.sum | 25 +- services/filesstore/filesstore_test.go | 11 +- services/filesstore/s3store.go | 95 +-- vendor/github.com/klauspost/cpuid/README.md | 6 +- .../klauspost/cpuid/detect_arm64.go | 4 + .../klauspost/cpuid/detect_intel.go | 2 +- vendor/github.com/minio/minio-go/v6/NOTICE | 2 - .../minio/minio-go/v6/api-compose-object.go | 624 ------------------ .../minio-go/v6/api-get-bucket-encryption.go | 71 -- .../minio-go/v6/api-get-bucket-versioning.go | 78 --- .../minio/minio-go/v6/api-get-lifecycle.go | 77 --- .../minio/minio-go/v6/api-get-object-acl.go | 27 - .../minio-go/v6/api-get-object-context.go | 26 - .../minio/minio-go/v6/api-get-policy.go | 78 --- .../minio/minio-go/v6/api-put-bucket.go | 537 --------------- .../minio-go/v6/api-put-object-context.go | 33 - .../minio/minio-go/v6/api-put-object-copy.go | 99 --- .../minio/minio-go/v6/api-put-object-file.go | 27 - vendor/github.com/minio/minio-go/v6/core.go | 185 ------ vendor/github.com/minio/minio-go/v6/go.mod | 16 - .../minio/minio-go/{v6 => v7}/.gitignore | 0 .../minio/minio-go/{v6 => v7}/.golangci.yml | 0 vendor/github.com/minio/minio-go/v7/CNAME | 1 + .../minio/minio-go/{v6 => v7}/CONTRIBUTING.md | 0 .../minio/minio-go/{v6 => v7}/LICENSE | 0 .../minio/minio-go/{v6 => v7}/MAINTAINERS.md | 0 .../minio/minio-go/{v6 => v7}/Makefile | 13 +- vendor/github.com/minio/minio-go/v7/NOTICE | 9 + .../minio/minio-go/{v6 => v7}/README.md | 43 +- .../minio/minio-go/{v6 => v7}/README_zh_CN.md | 24 +- .../minio-go/v7/api-bucket-encryption.go | 134 ++++ .../minio/minio-go/v7/api-bucket-lifecycle.go | 147 +++++ .../api-bucket-notification.go} | 188 +++--- .../minio/minio-go/v7/api-bucket-policy.go | 142 ++++ .../minio-go/v7/api-bucket-replication.go | 149 +++++ .../minio-go/{v6 => v7}/api-bucket-tagging.go | 34 +- .../minio-go/v7/api-bucket-versioning.go | 120 ++++ .../minio/minio-go/v7/api-compose-object.go | 552 ++++++++++++++++ .../minio-go/{v6 => v7}/api-datatypes.go | 50 +- .../minio-go/{v6 => v7}/api-error-response.go | 53 +- .../api-get-object-acl.go} | 6 +- .../{v6 => v7}/api-get-object-file.go | 20 +- .../minio-go/{v6 => v7}/api-get-object.go | 48 +- .../minio-go/{v6 => v7}/api-get-options.go | 19 +- .../minio/minio-go/{v6 => v7}/api-list.go | 469 +++++++------ .../{v6 => v7}/api-object-legal-hold.go | 10 +- .../minio-go/{v6 => v7}/api-object-lock.go | 18 +- .../{v6 => v7}/api-object-retention.go | 43 +- .../minio-go/{v6 => v7}/api-object-tagging.go | 75 ++- .../minio-go/{v6 => v7}/api-presigned.go | 39 +- .../minio/minio-go/v7/api-put-bucket.go | 123 ++++ .../{v6 => v7}/api-put-object-common.go | 12 +- .../minio/minio-go/v7/api-put-object-copy.go | 77 +++ .../{v6 => v7}/api-put-object-file-context.go | 16 +- .../{v6 => v7}/api-put-object-multipart.go | 94 +-- .../{v6 => v7}/api-put-object-streaming.go | 161 +++-- .../minio-go/{v6 => v7}/api-put-object.go | 105 ++- .../minio/minio-go/{v6 => v7}/api-remove.go | 105 +-- .../minio-go/{v6 => v7}/api-s3-datatypes.go | 102 ++- .../minio/minio-go/{v6 => v7}/api-select.go | 12 +- .../minio/minio-go/{v6 => v7}/api-stat.go | 32 +- .../minio/minio-go/{v6 => v7}/api.go | 144 +--- .../minio/minio-go/{v6 => v7}/bucket-cache.go | 15 +- .../minio/minio-go/v7/code_of_conduct.md | 80 +++ .../minio/minio-go/{v6 => v7}/constants.go | 10 + vendor/github.com/minio/minio-go/v7/core.go | 132 ++++ vendor/github.com/minio/minio-go/v7/go.mod | 25 + .../minio/minio-go/{v6 => v7}/go.sum | 57 +- .../minio/minio-go/{v6 => v7}/hook-reader.go | 0 .../{v6 => v7}/pkg/credentials/assume_role.go | 4 +- .../{v6 => v7}/pkg/credentials/chain.go | 0 .../pkg/credentials/config.json.sample | 0 .../{v6 => v7}/pkg/credentials/credentials.go | 0 .../pkg/credentials/credentials.sample | 0 .../{v6 => v7}/pkg/credentials/doc.go | 0 .../{v6 => v7}/pkg/credentials/env_aws.go | 0 .../{v6 => v7}/pkg/credentials/env_minio.go | 0 .../pkg/credentials/file_aws_credentials.go | 0 .../pkg/credentials/file_minio_client.go | 0 .../{v6 => v7}/pkg/credentials/iam_aws.go | 12 +- .../pkg/credentials/signature-type.go | 0 .../{v6 => v7}/pkg/credentials/static.go | 0 .../pkg/credentials/sts_client_grants.go | 2 +- .../pkg/credentials/sts_ldap_identity.go | 2 +- .../pkg/credentials/sts_web_identity.go | 7 +- .../{v6 => v7}/pkg/encrypt/server-side.go | 0 .../minio-go/v7/pkg/lifecycle/lifecycle.go | 282 ++++++++ .../minio-go/v7/pkg/notification/info.go | 78 +++ .../pkg/notification/notification.go} | 118 ++-- .../v7/pkg/replication/replication.go | 360 ++++++++++ .../minio-go/{v6 => v7}/pkg/s3utils/utils.go | 3 + .../minio-go/{v6 => v7}/pkg/set/stringset.go | 0 .../pkg/signer/request-signature-streaming.go | 0 .../pkg/signer/request-signature-v2.go | 3 +- .../pkg/signer/request-signature-v4.go | 2 +- .../minio-go/{v6 => v7}/pkg/signer/utils.go | 0 .../minio/minio-go/v7/pkg/sse/sse.go | 66 ++ .../minio-go/{v6 => v7}/pkg/tags/tags.go | 12 + .../minio/minio-go/{v6 => v7}/post-policy.go | 50 +- .../minio-go/{v6 => v7}/retry-continous.go | 0 .../minio/minio-go/{v6 => v7}/retry.go | 0 .../minio/minio-go/{v6 => v7}/s3-endpoints.go | 0 .../minio/minio-go/{v6 => v7}/s3-error.go | 0 .../minio-go/{v6 => v7}/staticcheck.conf | 0 .../minio/minio-go/{v6 => v7}/transport.go | 2 +- .../minio/minio-go/{v6 => v7}/utils.go | 114 +++- vendor/github.com/rs/xid/.appveyor.yml | 27 + vendor/github.com/rs/xid/.travis.yml | 8 + vendor/github.com/rs/xid/LICENSE | 19 + vendor/github.com/rs/xid/README.md | 112 ++++ vendor/github.com/rs/xid/go.mod | 1 + vendor/github.com/rs/xid/hostid_darwin.go | 9 + vendor/github.com/rs/xid/hostid_fallback.go | 9 + vendor/github.com/rs/xid/hostid_freebsd.go | 9 + vendor/github.com/rs/xid/hostid_linux.go | 10 + vendor/github.com/rs/xid/hostid_windows.go | 38 ++ vendor/github.com/rs/xid/id.go | 365 ++++++++++ vendor/golang.org/x/sys/cpu/cpu.go | 96 +++ vendor/golang.org/x/sys/cpu/cpu_aix.go | 4 +- vendor/golang.org/x/sys/cpu/cpu_arm.go | 33 + vendor/golang.org/x/sys/cpu/cpu_arm64.go | 31 +- vendor/golang.org/x/sys/cpu/cpu_linux.go | 2 +- .../golang.org/x/sys/cpu/cpu_linux_ppc64x.go | 2 - .../golang.org/x/sys/cpu/cpu_linux_s390x.go | 2 - vendor/golang.org/x/sys/cpu/cpu_mips64x.go | 6 + vendor/golang.org/x/sys/cpu/cpu_mipsx.go | 2 + vendor/golang.org/x/sys/cpu/cpu_other_arm.go | 9 + vendor/golang.org/x/sys/cpu/cpu_ppc64x.go | 16 + vendor/golang.org/x/sys/cpu/cpu_riscv64.go | 2 + vendor/golang.org/x/sys/cpu/cpu_s390x.go | 30 + vendor/golang.org/x/sys/cpu/cpu_wasm.go | 4 + vendor/golang.org/x/sys/cpu/cpu_x86.go | 31 +- vendor/golang.org/x/sys/unix/mkerrors.sh | 5 + vendor/golang.org/x/sys/unix/syscall_bsd.go | 17 + vendor/golang.org/x/sys/unix/syscall_linux.go | 25 +- .../x/sys/unix/zerrors_freebsd_386.go | 6 + .../x/sys/unix/zerrors_freebsd_amd64.go | 6 + .../x/sys/unix/zerrors_freebsd_arm.go | 6 + .../x/sys/unix/zerrors_freebsd_arm64.go | 6 + vendor/golang.org/x/sys/unix/zerrors_linux.go | 18 +- .../x/sys/unix/zerrors_netbsd_386.go | 6 + .../x/sys/unix/zerrors_netbsd_amd64.go | 6 + .../x/sys/unix/zerrors_netbsd_arm.go | 6 + .../x/sys/unix/zerrors_netbsd_arm64.go | 6 + .../x/sys/unix/zerrors_openbsd_386.go | 7 + .../x/sys/unix/zerrors_openbsd_amd64.go | 7 + .../x/sys/unix/zerrors_openbsd_arm.go | 7 + .../x/sys/unix/zerrors_openbsd_arm64.go | 7 + .../golang.org/x/sys/unix/zsyscall_linux.go | 61 ++ .../x/sys/unix/zsysnum_linux_386.go | 1 + .../x/sys/unix/zsysnum_linux_amd64.go | 1 + .../x/sys/unix/zsysnum_linux_arm.go | 1 + .../x/sys/unix/zsysnum_linux_arm64.go | 1 + .../x/sys/unix/zsysnum_linux_mips.go | 1 + .../x/sys/unix/zsysnum_linux_mips64.go | 1 + .../x/sys/unix/zsysnum_linux_mips64le.go | 1 + .../x/sys/unix/zsysnum_linux_mipsle.go | 1 + .../x/sys/unix/zsysnum_linux_ppc64.go | 1 + .../x/sys/unix/zsysnum_linux_ppc64le.go | 1 + .../x/sys/unix/zsysnum_linux_riscv64.go | 1 + .../x/sys/unix/zsysnum_linux_s390x.go | 1 + .../x/sys/unix/zsysnum_linux_sparc64.go | 1 + vendor/golang.org/x/sys/unix/ztypes_linux.go | 4 +- vendor/modules.txt | 33 +- 166 files changed, 5005 insertions(+), 3002 deletions(-) delete mode 100644 vendor/github.com/minio/minio-go/v6/NOTICE delete mode 100644 vendor/github.com/minio/minio-go/v6/api-compose-object.go delete mode 100644 vendor/github.com/minio/minio-go/v6/api-get-bucket-encryption.go delete mode 100644 vendor/github.com/minio/minio-go/v6/api-get-bucket-versioning.go delete mode 100644 vendor/github.com/minio/minio-go/v6/api-get-lifecycle.go delete mode 100644 vendor/github.com/minio/minio-go/v6/api-get-object-acl.go delete mode 100644 vendor/github.com/minio/minio-go/v6/api-get-object-context.go delete mode 100644 vendor/github.com/minio/minio-go/v6/api-get-policy.go delete mode 100644 vendor/github.com/minio/minio-go/v6/api-put-bucket.go delete mode 100644 vendor/github.com/minio/minio-go/v6/api-put-object-context.go delete mode 100644 vendor/github.com/minio/minio-go/v6/api-put-object-copy.go delete mode 100644 vendor/github.com/minio/minio-go/v6/api-put-object-file.go delete mode 100644 vendor/github.com/minio/minio-go/v6/core.go delete mode 100644 vendor/github.com/minio/minio-go/v6/go.mod rename vendor/github.com/minio/minio-go/{v6 => v7}/.gitignore (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/.golangci.yml (100%) create mode 100644 vendor/github.com/minio/minio-go/v7/CNAME rename vendor/github.com/minio/minio-go/{v6 => v7}/CONTRIBUTING.md (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/LICENSE (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/MAINTAINERS.md (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/Makefile (57%) create mode 100644 vendor/github.com/minio/minio-go/v7/NOTICE rename vendor/github.com/minio/minio-go/{v6 => v7}/README.md (88%) rename vendor/github.com/minio/minio-go/{v6 => v7}/README_zh_CN.md (92%) create mode 100644 vendor/github.com/minio/minio-go/v7/api-bucket-encryption.go create mode 100644 vendor/github.com/minio/minio-go/v7/api-bucket-lifecycle.go rename vendor/github.com/minio/minio-go/{v6/api-notification.go => v7/api-bucket-notification.go} (51%) create mode 100644 vendor/github.com/minio/minio-go/v7/api-bucket-policy.go create mode 100644 vendor/github.com/minio/minio-go/v7/api-bucket-replication.go rename vendor/github.com/minio/minio-go/{v6 => v7}/api-bucket-tagging.go (69%) create mode 100644 vendor/github.com/minio/minio-go/v7/api-bucket-versioning.go create mode 100644 vendor/github.com/minio/minio-go/v7/api-compose-object.go rename vendor/github.com/minio/minio-go/{v6 => v7}/api-datatypes.go (77%) rename vendor/github.com/minio/minio-go/{v6 => v7}/api-error-response.go (82%) rename vendor/github.com/minio/minio-go/{v6/api-get-object-acl-context.go => v7/api-get-object-acl.go} (94%) rename vendor/github.com/minio/minio-go/{v6 => v7}/api-get-object-file.go (78%) rename vendor/github.com/minio/minio-go/{v6 => v7}/api-get-object.go (93%) rename vendor/github.com/minio/minio-go/{v6 => v7}/api-get-options.go (88%) rename vendor/github.com/minio/minio-go/{v6 => v7}/api-list.go (65%) rename vendor/github.com/minio/minio-go/{v6 => v7}/api-object-legal-hold.go (90%) rename vendor/github.com/minio/minio-go/{v6 => v7}/api-object-lock.go (86%) rename vendor/github.com/minio/minio-go/{v6 => v7}/api-object-retention.go (77%) rename vendor/github.com/minio/minio-go/{v6 => v7}/api-object-tagging.go (56%) rename vendor/github.com/minio/minio-go/{v6 => v7}/api-presigned.go (75%) create mode 100644 vendor/github.com/minio/minio-go/v7/api-put-bucket.go rename vendor/github.com/minio/minio-go/{v6 => v7}/api-put-object-common.go (91%) create mode 100644 vendor/github.com/minio/minio-go/v7/api-put-object-copy.go rename vendor/github.com/minio/minio-go/{v6 => v7}/api-put-object-file-context.go (75%) rename vendor/github.com/minio/minio-go/{v6 => v7}/api-put-object-multipart.go (81%) rename vendor/github.com/minio/minio-go/{v6 => v7}/api-put-object-streaming.go (75%) rename vendor/github.com/minio/minio-go/{v6 => v7}/api-put-object.go (74%) rename vendor/github.com/minio/minio-go/{v6 => v7}/api-remove.go (70%) rename vendor/github.com/minio/minio-go/{v6 => v7}/api-s3-datatypes.go (73%) rename vendor/github.com/minio/minio-go/{v6 => v7}/api-select.go (98%) rename vendor/github.com/minio/minio-go/{v6 => v7}/api-stat.go (70%) rename vendor/github.com/minio/minio-go/{v6 => v7}/api.go (85%) rename vendor/github.com/minio/minio-go/{v6 => v7}/bucket-cache.go (93%) create mode 100644 vendor/github.com/minio/minio-go/v7/code_of_conduct.md rename vendor/github.com/minio/minio-go/{v6 => v7}/constants.go (87%) create mode 100644 vendor/github.com/minio/minio-go/v7/core.go create mode 100644 vendor/github.com/minio/minio-go/v7/go.mod rename vendor/github.com/minio/minio-go/{v6 => v7}/go.sum (52%) rename vendor/github.com/minio/minio-go/{v6 => v7}/hook-reader.go (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/credentials/assume_role.go (98%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/credentials/chain.go (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/credentials/config.json.sample (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/credentials/credentials.go (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/credentials/credentials.sample (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/credentials/doc.go (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/credentials/env_aws.go (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/credentials/env_minio.go (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/credentials/file_aws_credentials.go (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/credentials/file_minio_client.go (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/credentials/iam_aws.go (96%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/credentials/signature-type.go (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/credentials/static.go (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/credentials/sts_client_grants.go (98%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/credentials/sts_ldap_identity.go (98%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/credentials/sts_web_identity.go (96%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/encrypt/server-side.go (100%) create mode 100644 vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go create mode 100644 vendor/github.com/minio/minio-go/v7/pkg/notification/info.go rename vendor/github.com/minio/minio-go/{v6/bucket-notification.go => v7/pkg/notification/notification.go} (68%) create mode 100644 vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/s3utils/utils.go (99%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/set/stringset.go (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/signer/request-signature-streaming.go (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/signer/request-signature-v2.go (99%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/signer/request-signature-v4.go (99%) rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/signer/utils.go (100%) create mode 100644 vendor/github.com/minio/minio-go/v7/pkg/sse/sse.go rename vendor/github.com/minio/minio-go/{v6 => v7}/pkg/tags/tags.go (95%) rename vendor/github.com/minio/minio-go/{v6 => v7}/post-policy.go (84%) rename vendor/github.com/minio/minio-go/{v6 => v7}/retry-continous.go (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/retry.go (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/s3-endpoints.go (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/s3-error.go (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/staticcheck.conf (100%) rename vendor/github.com/minio/minio-go/{v6 => v7}/transport.go (97%) rename vendor/github.com/minio/minio-go/{v6 => v7}/utils.go (77%) create mode 100644 vendor/github.com/rs/xid/.appveyor.yml create mode 100644 vendor/github.com/rs/xid/.travis.yml create mode 100644 vendor/github.com/rs/xid/LICENSE create mode 100644 vendor/github.com/rs/xid/README.md create mode 100644 vendor/github.com/rs/xid/go.mod create mode 100644 vendor/github.com/rs/xid/hostid_darwin.go create mode 100644 vendor/github.com/rs/xid/hostid_fallback.go create mode 100644 vendor/github.com/rs/xid/hostid_freebsd.go create mode 100644 vendor/github.com/rs/xid/hostid_linux.go create mode 100644 vendor/github.com/rs/xid/hostid_windows.go create mode 100644 vendor/github.com/rs/xid/id.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_other_arm.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_ppc64x.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_s390x.go diff --git a/api4/apitestlib.go b/api4/apitestlib.go index 8798050c5f..2475349c66 100644 --- a/api4/apitestlib.go +++ b/api4/apitestlib.go @@ -4,6 +4,7 @@ package api4 import ( + "context" "fmt" "io/ioutil" "math/rand" @@ -29,8 +30,8 @@ import ( "github.com/mattermost/mattermost-server/v5/web" "github.com/mattermost/mattermost-server/v5/wsapi" - s3 "github.com/minio/minio-go/v6" - "github.com/minio/minio-go/v6/pkg/credentials" + s3 "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/stretchr/testify/require" ) @@ -890,7 +891,13 @@ func s3New(endpoint, accessKey, secretKey string, secure bool, signV2 bool, regi } else { creds = credentials.NewStatic(accessKey, secretKey, "", credentials.SignatureV4) } - return s3.NewWithCredentials(endpoint, creds, secure, region) + + opts := s3.Options{ + Creds: creds, + Secure: secure, + Region: region, + } + return s3.New(endpoint, &opts) } func (me *TestHelper) cleanupTestFile(info *model.FileInfo) error { @@ -907,18 +914,18 @@ func (me *TestHelper) cleanupTestFile(info *model.FileInfo) error { return err } bucket := *cfg.FileSettings.AmazonS3Bucket - if err := s3Clnt.RemoveObject(bucket, info.Path); err != nil { + if err := s3Clnt.RemoveObject(context.Background(), bucket, info.Path, s3.RemoveObjectOptions{}); err != nil { return err } if info.ThumbnailPath != "" { - if err := s3Clnt.RemoveObject(bucket, info.ThumbnailPath); err != nil { + if err := s3Clnt.RemoveObject(context.Background(), bucket, info.ThumbnailPath, s3.RemoveObjectOptions{}); err != nil { return err } } if info.PreviewPath != "" { - if err := s3Clnt.RemoveObject(bucket, info.PreviewPath); err != nil { + if err := s3Clnt.RemoveObject(context.Background(), bucket, info.PreviewPath, s3.RemoveObjectOptions{}); err != nil { return err } } diff --git a/go.mod b/go.mod index 067fe1ec56..1cb713571a 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,6 @@ require ( github.com/go-gorp/gorp v2.2.0+incompatible // indirect github.com/go-sql-driver/mysql v1.5.0 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 - github.com/google/uuid v1.1.1 // indirect github.com/gorilla/handlers v1.4.2 github.com/gorilla/mux v1.7.4 github.com/gorilla/schema v1.1.0 @@ -55,7 +54,6 @@ require ( github.com/jmhodges/levigo v1.0.0 // indirect github.com/jmoiron/sqlx v1.2.0 github.com/jonboulle/clockwork v0.1.0 // indirect - github.com/klauspost/cpuid v1.3.0 // indirect github.com/lib/pq v1.7.0 github.com/magiconair/properties v1.8.1 // indirect github.com/mailru/easyjson v0.7.1 // indirect @@ -70,7 +68,7 @@ require ( github.com/mattn/go-runewidth v0.0.9 // indirect github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/miekg/dns v1.1.29 // indirect - github.com/minio/minio-go/v6 v6.0.57 + github.com/minio/minio-go/v7 v7.0.3 github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/mapstructure v1.3.2 // indirect github.com/muesli/smartcrop v0.3.0 // indirect @@ -116,12 +114,12 @@ require ( github.com/ziutek/mymysql v1.5.4 // indirect go.etcd.io/bbolt v1.3.5 // indirect go.uber.org/zap v1.15.0 - golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de golang.org/x/image v0.0.0-20200618115811-c13761719519 golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect golang.org/x/mod v0.3.0 // indirect - golang.org/x/net v0.0.0-20200625001655-4c5254603344 - golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect + golang.org/x/net v0.0.0-20200707034311-ab3426394381 + golang.org/x/sys v0.0.0-20200805065543-0cf7623e9dbd // indirect golang.org/x/text v0.3.3 golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f google.golang.org/appengine v1.6.6 // indirect @@ -129,7 +127,6 @@ require ( google.golang.org/grpc v1.30.0 // indirect google.golang.org/protobuf v1.25.0 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect - gopkg.in/ini.v1 v1.57.0 // indirect gopkg.in/mail.v2 v2.3.1 gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/olivere/elastic.v6 v6.2.33 diff --git a/go.sum b/go.sum index cc4c561fbd..4346809274 100644 --- a/go.sum +++ b/go.sum @@ -385,8 +385,8 @@ github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid v1.3.0 h1:2JqaNE1hGdABW2YbA3TenkO7RiPFRvSWnEnGqWh9sHE= -github.com/klauspost/cpuid v1.3.0/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= +github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= +github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= github.com/kljensen/snowball v0.6.0/go.mod h1:27N7E8fVU5H68RlUmnWwZCfxgt4POBJfENGMvNRhldw= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -466,8 +466,8 @@ github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= -github.com/minio/minio-go/v6 v6.0.57 h1:ixPkbKkyD7IhnluRgQpGSpHdpvNVaW6OD5R9IAO/9Tw= -github.com/minio/minio-go/v6 v6.0.57/go.mod h1:5+R/nM9Pwrh0vqF+HbYYDQ84wdUFPyXHkrdT4AIkifM= +github.com/minio/minio-go/v7 v7.0.3 h1:a2VHaXDlKBcB3J5XJhKVfWBRi1+ZmMWFXABQ8TLlWbA= +github.com/minio/minio-go/v7 v7.0.3/go.mod h1:TA0CQCjJZHM5SJj9IjqR0NmpmQJ6bCbXifAJ3mUU6Hw= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -613,6 +613,8 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rudderlabs/analytics-go v3.2.1+incompatible h1:XDocL6elYIi8WhLXLklDahq+Ws3FAYVOvJSsMuYWaKk= github.com/rudderlabs/analytics-go v3.2.1+incompatible/go.mod h1:LF8/ty9kUX4PTY3l5c97K3nZZaX5Hwsvt+NBaRL/f30= github.com/russellhaering/goxmldsig v0.0.0-20180430223755-7acd5e4a6ef7 h1:J4AOUcOh/t1XbQcJfkEqhzgvMJ2tDxdCVvmHxW5QXao= @@ -654,8 +656,6 @@ github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5k github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= -github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -796,13 +796,16 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90Pveol golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de h1:ikNHVSjEfnvz6sxdSPCaPt572qowuyMDMJLLm3Db3ig= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190321063152-3fc05d484e9f h1:FO4MZ3N56GnxbqxGKqh+YTzUWQ2sDwtFQEZgLOxh9Jc= @@ -851,8 +854,8 @@ golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -903,6 +906,8 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200805065543-0cf7623e9dbd h1:wefLe/3g5tC0FcXw3NneLA5tHgbyouyZlfcSjNfOdgk= +golang.org/x/sys v0.0.0-20200805065543-0cf7623e9dbd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To= @@ -1014,7 +1019,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mail.v2 v2.3.1 h1:WYFn/oANrAGP2C0dcV6/pbkPzv8yGzqTjPmTeO7qoXk= @@ -1033,6 +1037,7 @@ gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= diff --git a/services/filesstore/filesstore_test.go b/services/filesstore/filesstore_test.go index ff7654fa88..46074cbc7d 100644 --- a/services/filesstore/filesstore_test.go +++ b/services/filesstore/filesstore_test.go @@ -158,11 +158,14 @@ func (s *FileBackendTestSuite) TestCopyFile() { s.Nil(err) defer s.backend.RemoveFile(path2) - _, err = s.backend.ReadFile(path1) + data1, err := s.backend.ReadFile(path1) s.Nil(err) - _, err = s.backend.ReadFile(path2) + data2, err := s.backend.ReadFile(path2) s.Nil(err) + + s.Equal(b, data1) + s.Equal(b, data2) } func (s *FileBackendTestSuite) TestCopyFileToDirectoryThatDoesntExist() { @@ -202,8 +205,10 @@ func (s *FileBackendTestSuite) TestMoveFile() { _, err = s.backend.ReadFile(path1) s.Error(err) - _, err = s.backend.ReadFile(path2) + data, err := s.backend.ReadFile(path2) s.Nil(err) + + s.Equal(b, data) } func (s *FileBackendTestSuite) TestRemoveFile() { diff --git a/services/filesstore/s3store.go b/services/filesstore/s3store.go index 5e1cd6e8ae..42f7308127 100644 --- a/services/filesstore/s3store.go +++ b/services/filesstore/s3store.go @@ -5,6 +5,7 @@ package filesstore import ( "bytes" + "context" "io" "io/ioutil" "net/http" @@ -12,9 +13,9 @@ import ( "path/filepath" "strings" - s3 "github.com/minio/minio-go/v6" - "github.com/minio/minio-go/v6/pkg/credentials" - "github.com/minio/minio-go/v6/pkg/encrypt" + s3 "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/mattermost/mattermost-server/v5/mlog" "github.com/mattermost/mattermost-server/v5/model" @@ -49,7 +50,12 @@ func (b *S3FileBackend) s3New() (*s3.Client, error) { creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV4) } - s3Clnt, err := s3.NewWithCredentials(b.endpoint, creds, b.secure, b.region) + opts := s3.Options{ + Creds: creds, + Secure: b.secure, + Region: b.region, + } + s3Clnt, err := s3.New(b.endpoint, &opts) if err != nil { return nil, err } @@ -67,14 +73,14 @@ func (b *S3FileBackend) TestConnection() *model.AppError { return model.NewAppError("TestFileConnection", "api.file.test_connection.s3.connection.app_error", nil, err.Error(), http.StatusInternalServerError) } - exists, err := s3Clnt.BucketExists(b.bucket) + exists, err := s3Clnt.BucketExists(context.Background(), b.bucket) if err != nil { return model.NewAppError("TestFileConnection", "api.file.test_connection.s3.bucket_exists.app_error", nil, err.Error(), http.StatusInternalServerError) } if !exists { mlog.Warn("Bucket specified does not exist. Attempting to create...") - err := s3Clnt.MakeBucket(b.bucket, b.region) + err := s3Clnt.MakeBucket(context.Background(), b.bucket, s3.MakeBucketOptions{Region: b.region}) if err != nil { mlog.Error("Unable to create bucket.") return model.NewAppError("TestFileConnection", "api.file.test_connection.s3.bucked_create.app_error", nil, err.Error(), http.StatusInternalServerError) @@ -92,7 +98,7 @@ func (b *S3FileBackend) Reader(path string) (ReadCloseSeeker, *model.AppError) { } path = filepath.Join(b.pathPrefix, path) - minioObject, err := s3Clnt.GetObject(b.bucket, path, s3.GetObjectOptions{}) + minioObject, err := s3Clnt.GetObject(context.Background(), b.bucket, path, s3.GetObjectOptions{}) if err != nil { return nil, model.NewAppError("Reader", "api.file.reader.s3.app_error", nil, err.Error(), http.StatusInternalServerError) } @@ -107,7 +113,7 @@ func (b *S3FileBackend) ReadFile(path string) ([]byte, *model.AppError) { } path = filepath.Join(b.pathPrefix, path) - minioObject, err := s3Clnt.GetObject(b.bucket, path, s3.GetObjectOptions{}) + minioObject, err := s3Clnt.GetObject(context.Background(), b.bucket, path, s3.GetObjectOptions{}) if err != nil { return nil, model.NewAppError("ReadFile", "api.file.read_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) } @@ -128,7 +134,7 @@ func (b *S3FileBackend) FileExists(path string) (bool, *model.AppError) { } path = filepath.Join(b.pathPrefix, path) - _, err = s3Clnt.StatObject(b.bucket, path, s3.StatObjectOptions{}) + _, err = s3Clnt.StatObject(context.Background(), b.bucket, path, s3.StatObjectOptions{}) if err == nil { return true, nil @@ -149,14 +155,17 @@ func (b *S3FileBackend) CopyFile(oldPath, newPath string) *model.AppError { oldPath = filepath.Join(b.pathPrefix, oldPath) newPath = filepath.Join(b.pathPrefix, newPath) - - source := s3.NewSourceInfo(b.bucket, oldPath, nil) - destination, err := s3.NewDestinationInfo(b.bucket, newPath, encrypt.NewSSE(), nil) - if err != nil { - return model.NewAppError("copyFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + srcOpts := s3.CopySrcOptions{ + Bucket: b.bucket, + Object: oldPath, + Encryption: encrypt.NewSSE(), } - - if err = s3Clnt.CopyObject(destination, source); err != nil { + dstOpts := s3.CopyDestOptions{ + Bucket: b.bucket, + Object: newPath, + Encryption: encrypt.NewSSE(), + } + if _, err = s3Clnt.CopyObject(context.Background(), dstOpts, srcOpts); err != nil { return model.NewAppError("copyFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError) } return nil @@ -170,18 +179,22 @@ func (b *S3FileBackend) MoveFile(oldPath, newPath string) *model.AppError { oldPath = filepath.Join(b.pathPrefix, oldPath) newPath = filepath.Join(b.pathPrefix, newPath) - - source := s3.NewSourceInfo(b.bucket, oldPath, nil) - destination, err := s3.NewDestinationInfo(b.bucket, newPath, encrypt.NewSSE(), nil) - if err != nil { - return model.NewAppError("moveFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + srcOpts := s3.CopySrcOptions{ + Bucket: b.bucket, + Object: oldPath, + Encryption: encrypt.NewSSE(), + } + dstOpts := s3.CopyDestOptions{ + Bucket: b.bucket, + Object: newPath, + Encryption: encrypt.NewSSE(), } - if err = s3Clnt.CopyObject(destination, source); err != nil { + if _, err = s3Clnt.CopyObject(context.Background(), dstOpts, srcOpts); err != nil { return model.NewAppError("moveFile", "api.file.move_file.copy_within_s3.app_error", nil, err.Error(), http.StatusInternalServerError) } - if err = s3Clnt.RemoveObject(b.bucket, oldPath); err != nil { + if err = s3Clnt.RemoveObject(context.Background(), b.bucket, oldPath, s3.RemoveObjectOptions{}); err != nil { return model.NewAppError("moveFile", "api.file.move_file.delete_from_s3.app_error", nil, err.Error(), http.StatusInternalServerError) } @@ -208,12 +221,12 @@ func (b *S3FileBackend) WriteFile(fr io.Reader, path string) (int64, *model.AppE if err != nil { return 0, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) } - written, err := s3Clnt.PutObject(b.bucket, path, &buf, int64(buf.Len()), options) + info, err := s3Clnt.PutObject(context.Background(), b.bucket, path, &buf, int64(buf.Len()), options) if err != nil { - return written, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) + return info.Size, model.NewAppError("WriteFile", "api.file.write_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) } - return written, nil + return info.Size, nil } func (b *S3FileBackend) RemoveFile(path string) *model.AppError { @@ -223,15 +236,15 @@ func (b *S3FileBackend) RemoveFile(path string) *model.AppError { } path = filepath.Join(b.pathPrefix, path) - if err := s3Clnt.RemoveObject(b.bucket, path); err != nil { + if err := s3Clnt.RemoveObject(context.Background(), b.bucket, path, s3.RemoveObjectOptions{}); err != nil { return model.NewAppError("RemoveFile", "utils.file.remove_file.s3.app_error", nil, err.Error(), http.StatusInternalServerError) } return nil } -func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan string { - out := make(chan string, 1) +func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan s3.ObjectInfo { + out := make(chan s3.ObjectInfo, 1) go func() { defer close(out) @@ -243,7 +256,7 @@ func getPathsFromObjectInfos(in <-chan s3.ObjectInfo) <-chan string { break } - out <- info.Key + out <- info } }() @@ -258,9 +271,6 @@ func (b *S3FileBackend) ListDirectory(path string) (*[]string, *model.AppError) return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) } - doneCh := make(chan struct{}) - defer close(doneCh) - path = filepath.Join(b.pathPrefix, path) if !strings.HasSuffix(path, "/") && len(path) > 0 { // s3Clnt returns only the path itself when "/" is not present @@ -268,7 +278,10 @@ func (b *S3FileBackend) ListDirectory(path string) (*[]string, *model.AppError) path = path + "/" } - for object := range s3Clnt.ListObjects(b.bucket, path, false, doneCh) { + opts := s3.ListObjectsOptions{ + Prefix: path, + } + for object := range s3Clnt.ListObjects(context.Background(), b.bucket, opts) { if object.Err != nil { return nil, model.NewAppError("ListDirectory", "utils.file.list_directory.s3.app_error", nil, object.Err.Error(), http.StatusInternalServerError) } @@ -283,18 +296,18 @@ func (b *S3FileBackend) RemoveDirectory(path string) *model.AppError { if err != nil { return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Error(), http.StatusInternalServerError) } - - doneCh := make(chan struct{}) - - path = filepath.Join(b.pathPrefix, path) - for err := range s3Clnt.RemoveObjects(b.bucket, getPathsFromObjectInfos(s3Clnt.ListObjects(b.bucket, path, true, doneCh))) { + opts := s3.ListObjectsOptions{ + Prefix: filepath.Join(b.pathPrefix, path), + Recursive: true, + } + list := s3Clnt.ListObjects(context.Background(), b.bucket, opts) + objectsCh := s3Clnt.RemoveObjects(context.Background(), b.bucket, getPathsFromObjectInfos(list), s3.RemoveObjectsOptions{}) + for err := range objectsCh { if err.Err != nil { - doneCh <- struct{}{} return model.NewAppError("RemoveDirectory", "utils.file.remove_directory.s3.app_error", nil, err.Err.Error(), http.StatusInternalServerError) } } - close(doneCh) return nil } diff --git a/vendor/github.com/klauspost/cpuid/README.md b/vendor/github.com/klauspost/cpuid/README.md index 6e3c6c9dec..38d4a8b936 100644 --- a/vendor/github.com/klauspost/cpuid/README.md +++ b/vendor/github.com/klauspost/cpuid/README.md @@ -2,7 +2,7 @@ Package cpuid provides information about the CPU running the current program. CPU features are detected on startup, and kept for fast access through the life of the application. -Currently x86 / x64 (AMD64) is supported, and no external C (cgo) code is used, which should make the library very easy to use. +Currently x86 / x64 (AMD64/i386) and ARM (ARM64) is supported, and no external C (cgo) code is used, which should make the library very easy to use. You can access the CPU information by accessing the shared CPU variable of the cpuid library. @@ -86,6 +86,10 @@ Package home: https://github.com/klauspost/cpuid ## ARM CPU features +# ARM FEATURE DETECTION DISABLED! + +See [#52](https://github.com/klauspost/cpuid/issues/52). + Currently only `arm64` platforms are implemented. * **FP** Single-precision and double-precision floating point diff --git a/vendor/github.com/klauspost/cpuid/detect_arm64.go b/vendor/github.com/klauspost/cpuid/detect_arm64.go index 0d9380ffe4..923a826183 100644 --- a/vendor/github.com/klauspost/cpuid/detect_arm64.go +++ b/vendor/github.com/klauspost/cpuid/detect_arm64.go @@ -16,6 +16,10 @@ func initCPU() { } func addInfo(c *CPUInfo) { + // ARM64 disabled for now. + if true { + return + } // midr := getMidr() // MIDR_EL1 - Main ID Register diff --git a/vendor/github.com/klauspost/cpuid/detect_intel.go b/vendor/github.com/klauspost/cpuid/detect_intel.go index 09a4614654..363951b3b2 100644 --- a/vendor/github.com/klauspost/cpuid/detect_intel.go +++ b/vendor/github.com/klauspost/cpuid/detect_intel.go @@ -1,6 +1,6 @@ // Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. -//+build 386,!gccgo amd64,!gccgo,!noasm,!appengine +//+build 386,!gccgo,!noasm amd64,!gccgo,!noasm,!appengine package cpuid diff --git a/vendor/github.com/minio/minio-go/v6/NOTICE b/vendor/github.com/minio/minio-go/v6/NOTICE deleted file mode 100644 index fc6c8808e9..0000000000 --- a/vendor/github.com/minio/minio-go/v6/NOTICE +++ /dev/null @@ -1,2 +0,0 @@ -minio-go -Copyright 2015-2017 MinIO, Inc. \ No newline at end of file diff --git a/vendor/github.com/minio/minio-go/v6/api-compose-object.go b/vendor/github.com/minio/minio-go/v6/api-compose-object.go deleted file mode 100644 index 15e3b6f38a..0000000000 --- a/vendor/github.com/minio/minio-go/v6/api-compose-object.go +++ /dev/null @@ -1,624 +0,0 @@ -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017, 2018 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "context" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - "strconv" - "strings" - "time" - - "github.com/minio/minio-go/v6/pkg/encrypt" - "github.com/minio/minio-go/v6/pkg/s3utils" -) - -// DestinationInfo - type with information about the object to be -// created via server-side copy requests, using the Compose API. -type DestinationInfo struct { - bucket, object string - opts DestInfoOptions -} - -// DestInfoOptions represents options specified by user for NewDestinationInfo call -type DestInfoOptions struct { - // `Encryption` is the key info for server-side-encryption with customer - // provided key. If it is nil, no encryption is performed. - Encryption encrypt.ServerSide - - // `userMeta` is the user-metadata key-value pairs to be set on the - // destination. The keys are automatically prefixed with `x-amz-meta-` - // if needed. If nil is passed, and if only a single source (of any - // size) is provided in the ComposeObject call, then metadata from the - // source is copied to the destination. - // if no user-metadata is provided, it is copied from source - // (when there is only once source object in the compose - // request) - UserMeta map[string]string - - // `userTags` is the user defined object tags to be set on destination. - // This will be set only if the `replaceTags` field is set to true. - // Otherwise this field is ignored - UserTags map[string]string - ReplaceTags bool - - // Specifies whether you want to apply a Legal Hold to the copied object. - LegalHold LegalHoldStatus - - // Object Retention related fields - Mode RetentionMode - RetainUntilDate time.Time -} - -// Process custom-metadata to remove a `x-amz-meta-` prefix if -// present and validate that keys are distinct (after this -// prefix removal). -func filterCustomMeta(userMeta map[string]string) (map[string]string, error) { - m := make(map[string]string) - for k, v := range userMeta { - if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") { - k = k[len("x-amz-meta-"):] - } - if _, ok := m[k]; ok { - return nil, ErrInvalidArgument(fmt.Sprintf("Cannot add both %s and x-amz-meta-%s keys as custom metadata", k, k)) - } - m[k] = v - } - return m, nil -} - -// NewDestinationInfo - creates a compose-object/copy-source -// destination info object. -// -// `sse` is the key info for server-side-encryption with customer -// provided key. If it is nil, no encryption is performed. -// -// `userMeta` is the user-metadata key-value pairs to be set on the -// destination. The keys are automatically prefixed with `x-amz-meta-` -// if needed. If nil is passed, and if only a single source (of any -// size) is provided in the ComposeObject call, then metadata from the -// source is copied to the destination. -func NewDestinationInfo(bucket, object string, sse encrypt.ServerSide, userMeta map[string]string) (d DestinationInfo, err error) { - // Input validation. - if err = s3utils.CheckValidBucketName(bucket); err != nil { - return d, err - } - if err = s3utils.CheckValidObjectName(object); err != nil { - return d, err - } - m, err := filterCustomMeta(userMeta) - if err != nil { - return d, err - } - opts := DestInfoOptions{ - Encryption: sse, - UserMeta: m, - UserTags: nil, - ReplaceTags: false, - LegalHold: LegalHoldStatus(""), - Mode: RetentionMode(""), - } - return DestinationInfo{ - bucket: bucket, - object: object, - opts: opts, - }, nil -} - -// NewDestinationInfoWithOptions - creates a compose-object/copy-source -// destination info object. -func NewDestinationInfoWithOptions(bucket, object string, destOpts DestInfoOptions) (d DestinationInfo, err error) { - // Input validation. - if err = s3utils.CheckValidBucketName(bucket); err != nil { - return d, err - } - if err = s3utils.CheckValidObjectName(object); err != nil { - return d, err - } - destOpts.UserMeta, err = filterCustomMeta(destOpts.UserMeta) - if err != nil { - return d, err - } - return DestinationInfo{ - bucket: bucket, - object: object, - opts: destOpts, - }, nil -} - -// getUserMetaHeadersMap - construct appropriate key-value pairs to send -// as headers from metadata map to pass into copy-object request. For -// single part copy-object (i.e. non-multipart object), enable the -// withCopyDirectiveHeader to set the `x-amz-metadata-directive` to -// `REPLACE`, so that metadata headers from the source are not copied -// over. -func (d *DestinationInfo) getUserMetaHeadersMap(withCopyDirectiveHeader bool) map[string]string { - if len(d.opts.UserMeta) == 0 { - return nil - } - r := make(map[string]string) - if withCopyDirectiveHeader { - r["x-amz-metadata-directive"] = "REPLACE" - } - for k, v := range d.opts.UserMeta { - if isAmzHeader(k) || isStandardHeader(k) || isStorageClassHeader(k) { - r[k] = v - } else { - r["x-amz-meta-"+k] = v - } - } - return r -} - -// SourceInfo - represents a source object to be copied, using -// server-side copying APIs. -type SourceInfo struct { - bucket, object string - start, end int64 - encryption encrypt.ServerSide - // Headers to send with the upload-part-copy request involving - // this source object. - Headers http.Header -} - -// NewSourceInfo - create a compose-object/copy-object source info -// object. -// -// `decryptSSEC` is the decryption key using server-side-encryption -// with customer provided key. It may be nil if the source is not -// encrypted. -func NewSourceInfo(bucket, object string, sse encrypt.ServerSide) SourceInfo { - r := SourceInfo{ - bucket: bucket, - object: object, - start: -1, // range is unspecified by default - encryption: sse, - Headers: make(http.Header), - } - - // Set the source header - r.Headers.Set("x-amz-copy-source", s3utils.EncodePath(bucket+"/"+object)) - return r -} - -// SetRange - Set the start and end offset of the source object to be -// copied. If this method is not called, the whole source object is -// copied. -func (s *SourceInfo) SetRange(start, end int64) error { - if start > end || start < 0 { - return ErrInvalidArgument("start must be non-negative, and start must be at most end.") - } - // Note that 0 <= start <= end - s.start, s.end = start, end - return nil -} - -// SetMatchETagCond - Set ETag match condition. The object is copied -// only if the etag of the source matches the value given here. -func (s *SourceInfo) SetMatchETagCond(etag string) error { - if etag == "" { - return ErrInvalidArgument("ETag cannot be empty.") - } - s.Headers.Set("x-amz-copy-source-if-match", etag) - return nil -} - -// SetMatchETagExceptCond - Set the ETag match exception -// condition. The object is copied only if the etag of the source is -// not the value given here. -func (s *SourceInfo) SetMatchETagExceptCond(etag string) error { - if etag == "" { - return ErrInvalidArgument("ETag cannot be empty.") - } - s.Headers.Set("x-amz-copy-source-if-none-match", etag) - return nil -} - -// SetModifiedSinceCond - Set the modified since condition. -func (s *SourceInfo) SetModifiedSinceCond(modTime time.Time) error { - if modTime.IsZero() { - return ErrInvalidArgument("Input time cannot be 0.") - } - s.Headers.Set("x-amz-copy-source-if-modified-since", modTime.Format(http.TimeFormat)) - return nil -} - -// SetUnmodifiedSinceCond - Set the unmodified since condition. -func (s *SourceInfo) SetUnmodifiedSinceCond(modTime time.Time) error { - if modTime.IsZero() { - return ErrInvalidArgument("Input time cannot be 0.") - } - s.Headers.Set("x-amz-copy-source-if-unmodified-since", modTime.Format(http.TimeFormat)) - return nil -} - -// Helper to fetch size and etag of an object using a StatObject call. -func (s *SourceInfo) getProps(c Client) (size int64, etag string, userMeta map[string]string, err error) { - // Get object info - need size and etag here. Also, decryption - // headers are added to the stat request if given. - var objInfo ObjectInfo - opts := StatObjectOptions{GetObjectOptions{ServerSideEncryption: encrypt.SSE(s.encryption)}} - objInfo, err = c.statObject(context.Background(), s.bucket, s.object, opts) - if err != nil { - err = ErrInvalidArgument(fmt.Sprintf("Could not stat object - %s/%s: %v", s.bucket, s.object, err)) - } else { - size = objInfo.Size - etag = objInfo.ETag - userMeta = make(map[string]string) - for k, v := range objInfo.Metadata { - if strings.HasPrefix(k, "x-amz-meta-") { - if len(v) > 0 { - userMeta[k] = v[0] - } - } - } - } - return -} - -// Low level implementation of CopyObject API, supports only upto 5GiB worth of copy. -func (c Client) copyObjectDo(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, - metadata map[string]string) (ObjectInfo, error) { - - // Build headers. - headers := make(http.Header) - - // Set all the metadata headers. - for k, v := range metadata { - headers.Set(k, v) - } - - // Set the source header - headers.Set("x-amz-copy-source", s3utils.EncodePath(srcBucket+"/"+srcObject)) - - // Send upload-part-copy request - resp, err := c.executeMethod(ctx, "PUT", requestMetadata{ - bucketName: destBucket, - objectName: destObject, - customHeader: headers, - }) - defer closeResponse(resp) - if err != nil { - return ObjectInfo{}, err - } - - // Check if we got an error response. - if resp.StatusCode != http.StatusOK { - return ObjectInfo{}, httpRespToErrorResponse(resp, srcBucket, srcObject) - } - - cpObjRes := copyObjectResult{} - err = xmlDecoder(resp.Body, &cpObjRes) - if err != nil { - return ObjectInfo{}, err - } - - objInfo := ObjectInfo{ - Key: destObject, - ETag: strings.Trim(cpObjRes.ETag, "\""), - LastModified: cpObjRes.LastModified, - } - return objInfo, nil -} - -func (c Client) copyObjectPartDo(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, - partID int, startOffset int64, length int64, metadata map[string]string) (p CompletePart, err error) { - - headers := make(http.Header) - - // Set source - headers.Set("x-amz-copy-source", s3utils.EncodePath(srcBucket+"/"+srcObject)) - - if startOffset < 0 { - return p, ErrInvalidArgument("startOffset must be non-negative") - } - - if length >= 0 { - headers.Set("x-amz-copy-source-range", fmt.Sprintf("bytes=%d-%d", startOffset, startOffset+length-1)) - } - - for k, v := range metadata { - headers.Set(k, v) - } - - queryValues := make(url.Values) - queryValues.Set("partNumber", strconv.Itoa(partID)) - queryValues.Set("uploadId", uploadID) - - resp, err := c.executeMethod(ctx, "PUT", requestMetadata{ - bucketName: destBucket, - objectName: destObject, - customHeader: headers, - queryValues: queryValues, - }) - defer closeResponse(resp) - if err != nil { - return - } - - // Check if we got an error response. - if resp.StatusCode != http.StatusOK { - return p, httpRespToErrorResponse(resp, destBucket, destObject) - } - - // Decode copy-part response on success. - cpObjRes := copyObjectResult{} - err = xmlDecoder(resp.Body, &cpObjRes) - if err != nil { - return p, err - } - p.PartNumber, p.ETag = partID, cpObjRes.ETag - return p, nil -} - -// uploadPartCopy - helper function to create a part in a multipart -// upload via an upload-part-copy request -// https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPartCopy.html -func (c Client) uploadPartCopy(ctx context.Context, bucket, object, uploadID string, partNumber int, - headers http.Header) (p CompletePart, err error) { - - // Build query parameters - urlValues := make(url.Values) - urlValues.Set("partNumber", strconv.Itoa(partNumber)) - urlValues.Set("uploadId", uploadID) - - // Send upload-part-copy request - resp, err := c.executeMethod(ctx, "PUT", requestMetadata{ - bucketName: bucket, - objectName: object, - customHeader: headers, - queryValues: urlValues, - }) - defer closeResponse(resp) - if err != nil { - return p, err - } - - // Check if we got an error response. - if resp.StatusCode != http.StatusOK { - return p, httpRespToErrorResponse(resp, bucket, object) - } - - // Decode copy-part response on success. - cpObjRes := copyObjectResult{} - err = xmlDecoder(resp.Body, &cpObjRes) - if err != nil { - return p, err - } - p.PartNumber, p.ETag = partNumber, cpObjRes.ETag - return p, nil -} - -// ComposeObjectWithProgress - creates an object using server-side copying of -// existing objects. It takes a list of source objects (with optional -// offsets) and concatenates them into a new object using only -// server-side copying operations. Optionally takes progress reader hook -// for applications to look at current progress. -func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo, progress io.Reader) error { - if len(srcs) < 1 || len(srcs) > maxPartsCount { - return ErrInvalidArgument("There must be as least one and up to 10000 source objects.") - } - ctx := context.Background() - srcSizes := make([]int64, len(srcs)) - var totalSize, size, totalParts int64 - var srcUserMeta map[string]string - etags := make([]string, len(srcs)) - var err error - for i, src := range srcs { - size, etags[i], srcUserMeta, err = src.getProps(c) - if err != nil { - return err - } - - // Error out if client side encryption is used in this source object when - // more than one source objects are given. - if len(srcs) > 1 && src.Headers.Get("x-amz-meta-x-amz-key") != "" { - return ErrInvalidArgument( - fmt.Sprintf("Client side encryption is used in source object %s/%s", src.bucket, src.object)) - } - - // Check if a segment is specified, and if so, is the - // segment within object bounds? - if src.start != -1 { - // Since range is specified, - // 0 <= src.start <= src.end - // so only invalid case to check is: - if src.end >= size { - return ErrInvalidArgument( - fmt.Sprintf("SourceInfo %d has invalid segment-to-copy [%d, %d] (size is %d)", - i, src.start, src.end, size)) - } - size = src.end - src.start + 1 - } - - // Only the last source may be less than `absMinPartSize` - if size < absMinPartSize && i < len(srcs)-1 { - return ErrInvalidArgument( - fmt.Sprintf("SourceInfo %d is too small (%d) and it is not the last part", i, size)) - } - - // Is data to copy too large? - totalSize += size - if totalSize > maxMultipartPutObjectSize { - return ErrInvalidArgument(fmt.Sprintf("Cannot compose an object of size %d (> 5TiB)", totalSize)) - } - - // record source size - srcSizes[i] = size - - // calculate parts needed for current source - totalParts += partsRequired(size) - // Do we need more parts than we are allowed? - if totalParts > maxPartsCount { - return ErrInvalidArgument(fmt.Sprintf( - "Your proposed compose object requires more than %d parts", maxPartsCount)) - } - } - - // Single source object case (i.e. when only one source is - // involved, it is being copied wholly and at most 5GiB in - // size, emptyfiles are also supported). - if (totalParts == 1 && srcs[0].start == -1 && totalSize <= maxPartSize) || (totalSize == 0) { - return c.CopyObjectWithProgress(dst, srcs[0], progress) - } - - // Now, handle multipart-copy cases. - - // 1. Ensure that the object has not been changed while - // we are copying data. - for i, src := range srcs { - if src.Headers.Get("x-amz-copy-source-if-match") == "" { - src.SetMatchETagCond(etags[i]) - } - } - - // 2. Initiate a new multipart upload. - - // Set user-metadata on the destination object. If no - // user-metadata is specified, and there is only one source, - // (only) then metadata from source is copied. - userMeta := dst.getUserMetaHeadersMap(false) - metaMap := userMeta - if len(userMeta) == 0 && len(srcs) == 1 { - metaMap = srcUserMeta - } - metaHeaders := make(map[string]string) - for k, v := range metaMap { - metaHeaders[k] = v - } - - uploadID, err := c.newUploadID(ctx, dst.bucket, dst.object, PutObjectOptions{ServerSideEncryption: dst.opts.Encryption, UserMetadata: metaHeaders}) - if err != nil { - return err - } - - // 3. Perform copy part uploads - objParts := []CompletePart{} - partIndex := 1 - for i, src := range srcs { - h := src.Headers - if src.encryption != nil { - encrypt.SSECopy(src.encryption).Marshal(h) - } - // Add destination encryption headers - if dst.opts.Encryption != nil { - dst.opts.Encryption.Marshal(h) - } - - // calculate start/end indices of parts after - // splitting. - startIdx, endIdx := calculateEvenSplits(srcSizes[i], src) - for j, start := range startIdx { - end := endIdx[j] - - // Add (or reset) source range header for - // upload part copy request. - h.Set("x-amz-copy-source-range", - fmt.Sprintf("bytes=%d-%d", start, end)) - - // make upload-part-copy request - complPart, err := c.uploadPartCopy(ctx, dst.bucket, - dst.object, uploadID, partIndex, h) - if err != nil { - return err - } - if progress != nil { - io.CopyN(ioutil.Discard, progress, end-start+1) - } - objParts = append(objParts, complPart) - partIndex++ - } - } - - // 4. Make final complete-multipart request. - _, err = c.completeMultipartUpload(ctx, dst.bucket, dst.object, uploadID, - completeMultipartUpload{Parts: objParts}) - if err != nil { - return err - } - return nil -} - -// ComposeObject - creates an object using server-side copying of -// existing objects. It takes a list of source objects (with optional -// offsets) and concatenates them into a new object using only -// server-side copying operations. -func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error { - return c.ComposeObjectWithProgress(dst, srcs, nil) -} - -// partsRequired is maximum parts possible with -// max part size of ceiling(maxMultipartPutObjectSize / (maxPartsCount - 1)) -func partsRequired(size int64) int64 { - maxPartSize := maxMultipartPutObjectSize / (maxPartsCount - 1) - r := size / int64(maxPartSize) - if size%int64(maxPartSize) > 0 { - r++ - } - return r -} - -// calculateEvenSplits - computes splits for a source and returns -// start and end index slices. Splits happen evenly to be sure that no -// part is less than 5MiB, as that could fail the multipart request if -// it is not the last part. -func calculateEvenSplits(size int64, src SourceInfo) (startIndex, endIndex []int64) { - if size == 0 { - return - } - - reqParts := partsRequired(size) - startIndex = make([]int64, reqParts) - endIndex = make([]int64, reqParts) - // Compute number of required parts `k`, as: - // - // k = ceiling(size / copyPartSize) - // - // Now, distribute the `size` bytes in the source into - // k parts as evenly as possible: - // - // r parts sized (q+1) bytes, and - // (k - r) parts sized q bytes, where - // - // size = q * k + r (by simple division of size by k, - // so that 0 <= r < k) - // - start := src.start - if start == -1 { - start = 0 - } - quot, rem := size/reqParts, size%reqParts - nextStart := start - for j := int64(0); j < reqParts; j++ { - curPartSize := quot - if j < rem { - curPartSize++ - } - - cStart := nextStart - cEnd := cStart + curPartSize - 1 - nextStart = cEnd + 1 - - startIndex[j], endIndex[j] = cStart, cEnd - } - return -} diff --git a/vendor/github.com/minio/minio-go/v6/api-get-bucket-encryption.go b/vendor/github.com/minio/minio-go/v6/api-get-bucket-encryption.go deleted file mode 100644 index 213abb8fc2..0000000000 --- a/vendor/github.com/minio/minio-go/v6/api-get-bucket-encryption.go +++ /dev/null @@ -1,71 +0,0 @@ -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2020 MinIO, Inc. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "context" - "encoding/xml" - "io/ioutil" - "net/http" - "net/url" - - "github.com/minio/minio-go/v6/pkg/s3utils" -) - -// GetBucketEncryption - get default encryption configuration for a bucket. -func (c Client) GetBucketEncryption(bucketName string) (ServerSideEncryptionConfiguration, error) { - return c.GetBucketEncryptionWithContext(context.Background(), bucketName) -} - -// GetBucketEncryptionWithContext gets the default encryption configuration on an existing bucket with a context to control cancellations and timeouts. -func (c Client) GetBucketEncryptionWithContext(ctx context.Context, bucketName string) (ServerSideEncryptionConfiguration, error) { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return ServerSideEncryptionConfiguration{}, err - } - - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("encryption", "") - - // Execute GET on bucket to get the default encryption configuration. - resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - }) - - defer closeResponse(resp) - if err != nil { - return ServerSideEncryptionConfiguration{}, err - } - - if resp.StatusCode != http.StatusOK { - return ServerSideEncryptionConfiguration{}, httpRespToErrorResponse(resp, bucketName, "") - } - - bucketEncryptionBuf, err := ioutil.ReadAll(resp.Body) - if err != nil { - return ServerSideEncryptionConfiguration{}, err - } - - encryptionConfig := ServerSideEncryptionConfiguration{} - if err := xml.Unmarshal(bucketEncryptionBuf, &encryptionConfig); err != nil { - return ServerSideEncryptionConfiguration{}, err - } - return encryptionConfig, nil -} diff --git a/vendor/github.com/minio/minio-go/v6/api-get-bucket-versioning.go b/vendor/github.com/minio/minio-go/v6/api-get-bucket-versioning.go deleted file mode 100644 index 9b13d6730e..0000000000 --- a/vendor/github.com/minio/minio-go/v6/api-get-bucket-versioning.go +++ /dev/null @@ -1,78 +0,0 @@ -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2020 MinIO, Inc. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "context" - "encoding/xml" - "io/ioutil" - "net/http" - "net/url" - - "github.com/minio/minio-go/v6/pkg/s3utils" -) - -// BucketVersioningConfiguration is the versioning configuration structure -type BucketVersioningConfiguration struct { - XMLName xml.Name `xml:"VersioningConfiguration"` - Status string `xml:"Status"` - MfaDelete string `xml:"MfaDelete,omitempty"` -} - -// GetBucketVersioning - get versioning configuration for a bucket. -func (c Client) GetBucketVersioning(bucketName string) (BucketVersioningConfiguration, error) { - return c.GetBucketVersioningWithContext(context.Background(), bucketName) -} - -// GetBucketVersioningWithContext gets the versioning configuration on an existing bucket with a context to control cancellations and timeouts. -func (c Client) GetBucketVersioningWithContext(ctx context.Context, bucketName string) (BucketVersioningConfiguration, error) { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return BucketVersioningConfiguration{}, err - } - - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("versioning", "") - - // Execute GET on bucket to get the versioning configuration. - resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - }) - - defer closeResponse(resp) - if err != nil { - return BucketVersioningConfiguration{}, err - } - - if resp.StatusCode != http.StatusOK { - return BucketVersioningConfiguration{}, httpRespToErrorResponse(resp, bucketName, "") - } - - bucketVersioningBuf, err := ioutil.ReadAll(resp.Body) - if err != nil { - return BucketVersioningConfiguration{}, err - } - - versioningConfig := BucketVersioningConfiguration{} - if err := xml.Unmarshal(bucketVersioningBuf, &versioningConfig); err != nil { - return BucketVersioningConfiguration{}, err - } - return versioningConfig, nil -} diff --git a/vendor/github.com/minio/minio-go/v6/api-get-lifecycle.go b/vendor/github.com/minio/minio-go/v6/api-get-lifecycle.go deleted file mode 100644 index a24d03e370..0000000000 --- a/vendor/github.com/minio/minio-go/v6/api-get-lifecycle.go +++ /dev/null @@ -1,77 +0,0 @@ -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "context" - "io/ioutil" - "net/http" - "net/url" - - "github.com/minio/minio-go/v6/pkg/s3utils" -) - -// GetBucketLifecycle - get bucket lifecycle. -func (c Client) GetBucketLifecycle(bucketName string) (string, error) { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return "", err - } - bucketLifecycle, err := c.getBucketLifecycle(bucketName) - if err != nil { - errResponse := ToErrorResponse(err) - if errResponse.Code == "NoSuchLifecycleConfiguration" { - return "", nil - } - return "", err - } - return bucketLifecycle, nil -} - -// Request server for current bucket lifecycle. -func (c Client) getBucketLifecycle(bucketName string) (string, error) { - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("lifecycle", "") - - // Execute GET on bucket to get lifecycle. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - }) - - defer closeResponse(resp) - if err != nil { - return "", err - } - - if resp != nil { - if resp.StatusCode != http.StatusOK { - return "", httpRespToErrorResponse(resp, bucketName, "") - } - } - - bucketLifecycleBuf, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", err - } - - lifecycle := string(bucketLifecycleBuf) - return lifecycle, err -} diff --git a/vendor/github.com/minio/minio-go/v6/api-get-object-acl.go b/vendor/github.com/minio/minio-go/v6/api-get-object-acl.go deleted file mode 100644 index 9a664f189e..0000000000 --- a/vendor/github.com/minio/minio-go/v6/api-get-object-acl.go +++ /dev/null @@ -1,27 +0,0 @@ -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2018 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "context" -) - -//GetObjectACL get object ACLs -func (c Client) GetObjectACL(bucketName, objectName string) (*ObjectInfo, error) { - return c.GetObjectACLWithContext(context.Background(), bucketName, objectName) -} diff --git a/vendor/github.com/minio/minio-go/v6/api-get-object-context.go b/vendor/github.com/minio/minio-go/v6/api-get-object-context.go deleted file mode 100644 index 3c3afd0590..0000000000 --- a/vendor/github.com/minio/minio-go/v6/api-get-object-context.go +++ /dev/null @@ -1,26 +0,0 @@ -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import "context" - -// GetObjectWithContext - returns an seekable, readable object. -// The options can be used to specify the GET request further. -func (c Client) GetObjectWithContext(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (*Object, error) { - return c.getObjectWithContext(ctx, bucketName, objectName, opts) -} diff --git a/vendor/github.com/minio/minio-go/v6/api-get-policy.go b/vendor/github.com/minio/minio-go/v6/api-get-policy.go deleted file mode 100644 index bc1d10530a..0000000000 --- a/vendor/github.com/minio/minio-go/v6/api-get-policy.go +++ /dev/null @@ -1,78 +0,0 @@ -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "context" - "io/ioutil" - "net/http" - "net/url" - - "github.com/minio/minio-go/v6/pkg/s3utils" -) - -// GetBucketPolicy - get bucket policy at a given path. -func (c Client) GetBucketPolicy(bucketName string) (string, error) { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return "", err - } - bucketPolicy, err := c.getBucketPolicy(bucketName) - if err != nil { - errResponse := ToErrorResponse(err) - if errResponse.Code == "NoSuchBucketPolicy" { - return "", nil - } - return "", err - } - return bucketPolicy, nil -} - -// Request server for current bucket policy. -func (c Client) getBucketPolicy(bucketName string) (string, error) { - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("policy", "") - - // Execute GET on bucket to list objects. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentSHA256Hex: emptySHA256Hex, - }) - - defer closeResponse(resp) - if err != nil { - return "", err - } - - if resp != nil { - if resp.StatusCode != http.StatusOK { - return "", httpRespToErrorResponse(resp, bucketName, "") - } - } - - bucketPolicyBuf, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", err - } - - policy := string(bucketPolicyBuf) - return policy, err -} diff --git a/vendor/github.com/minio/minio-go/v6/api-put-bucket.go b/vendor/github.com/minio/minio-go/v6/api-put-bucket.go deleted file mode 100644 index 9d2486ad0c..0000000000 --- a/vendor/github.com/minio/minio-go/v6/api-put-bucket.go +++ /dev/null @@ -1,537 +0,0 @@ -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2020 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "bytes" - "context" - "encoding/xml" - "io/ioutil" - "net/http" - "net/url" - "strings" - - "github.com/minio/minio-go/v6/pkg/s3utils" -) - -// ApplyServerSideEncryptionByDefault defines default encryption configuration, KMS or SSE. To activate -// KMS, SSEAlgoritm needs to be set to "aws:kms" -// Minio currently does not support Kms. -type ApplyServerSideEncryptionByDefault struct { - KmsMasterKeyID string `xml:"KMSMasterKeyID,omitempty"` - SSEAlgorithm string `xml:"SSEAlgorithm"` -} - -// Rule layer encapsulates default encryption configuration -type Rule struct { - Apply ApplyServerSideEncryptionByDefault `xml:"ApplyServerSideEncryptionByDefault"` -} - -// ServerSideEncryptionConfiguration is the default encryption configuration structure -type ServerSideEncryptionConfiguration struct { - XMLName xml.Name `xml:"ServerSideEncryptionConfiguration"` - Rules []Rule `xml:"Rule"` -} - -/// Bucket operations - -func (c Client) makeBucket(ctx context.Context, bucketName string, location string, objectLockEnabled bool) (err error) { - // Validate the input arguments. - if err := s3utils.CheckValidBucketNameStrict(bucketName); err != nil { - return err - } - - err = c.doMakeBucket(ctx, bucketName, location, objectLockEnabled) - if err != nil && (location == "" || location == "us-east-1") { - if resp, ok := err.(ErrorResponse); ok && resp.Code == "AuthorizationHeaderMalformed" && resp.Region != "" { - err = c.doMakeBucket(ctx, bucketName, resp.Region, objectLockEnabled) - } - } - return err -} - -func (c Client) doMakeBucket(ctx context.Context, bucketName string, location string, objectLockEnabled bool) (err error) { - defer func() { - // Save the location into cache on a successful makeBucket response. - if err == nil { - c.bucketLocCache.Set(bucketName, location) - } - }() - - // If location is empty, treat is a default region 'us-east-1'. - if location == "" { - location = "us-east-1" - // For custom region clients, default - // to custom region instead not 'us-east-1'. - if c.region != "" { - location = c.region - } - } - // PUT bucket request metadata. - reqMetadata := requestMetadata{ - bucketName: bucketName, - bucketLocation: location, - } - - if objectLockEnabled { - headers := make(http.Header) - headers.Add("x-amz-bucket-object-lock-enabled", "true") - reqMetadata.customHeader = headers - } - - // If location is not 'us-east-1' create bucket location config. - if location != "us-east-1" && location != "" { - createBucketConfig := createBucketConfiguration{} - createBucketConfig.Location = location - var createBucketConfigBytes []byte - createBucketConfigBytes, err = xml.Marshal(createBucketConfig) - if err != nil { - return err - } - reqMetadata.contentMD5Base64 = sumMD5Base64(createBucketConfigBytes) - reqMetadata.contentSHA256Hex = sum256Hex(createBucketConfigBytes) - reqMetadata.contentBody = bytes.NewReader(createBucketConfigBytes) - reqMetadata.contentLength = int64(len(createBucketConfigBytes)) - } - - // Execute PUT to create a new bucket. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) - defer closeResponse(resp) - if err != nil { - return err - } - - if resp != nil { - if resp.StatusCode != http.StatusOK { - return httpRespToErrorResponse(resp, bucketName, "") - } - } - - // Success. - return nil -} - -// MakeBucket creates a new bucket with bucketName. -// -// Location is an optional argument, by default all buckets are -// created in US Standard Region. -// -// For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html -// For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations -func (c Client) MakeBucket(bucketName string, location string) (err error) { - return c.MakeBucketWithContext(context.Background(), bucketName, location) -} - -// MakeBucketWithContext creates a new bucket with bucketName with a context to control cancellations and timeouts. -// -// Location is an optional argument, by default all buckets are -// created in US Standard Region. -// -// For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html -// For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations -func (c Client) MakeBucketWithContext(ctx context.Context, bucketName string, location string) (err error) { - return c.makeBucket(ctx, bucketName, location, false) -} - -// MakeBucketWithObjectLock creates a object lock enabled new bucket with bucketName. -// -// Location is an optional argument, by default all buckets are -// created in US Standard Region. -// -// For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html -// For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations -func (c Client) MakeBucketWithObjectLock(bucketName string, location string) (err error) { - return c.MakeBucketWithObjectLockWithContext(context.Background(), bucketName, location) -} - -// MakeBucketWithObjectLockWithContext creates a object lock enabled new bucket with bucketName with a context to -// control cancellations and timeouts. -// -// Location is an optional argument, by default all buckets are -// created in US Standard Region. -// -// For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html -// For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations -func (c Client) MakeBucketWithObjectLockWithContext(ctx context.Context, bucketName string, location string) (err error) { - return c.makeBucket(ctx, bucketName, location, true) -} - -// SetBucketPolicy set the access permissions on an existing bucket. -func (c Client) SetBucketPolicy(bucketName, policy string) error { - return c.SetBucketPolicyWithContext(context.Background(), bucketName, policy) -} - -// SetBucketPolicyWithContext set the access permissions on an existing bucket. -func (c Client) SetBucketPolicyWithContext(ctx context.Context, bucketName, policy string) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - - // If policy is empty then delete the bucket policy. - if policy == "" { - return c.removeBucketPolicy(ctx, bucketName) - } - - // Save the updated policies. - return c.putBucketPolicy(ctx, bucketName, policy) -} - -// Saves a new bucket policy. -func (c Client) putBucketPolicy(ctx context.Context, bucketName, policy string) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("policy", "") - - // Content-length is mandatory for put policy request - policyReader := strings.NewReader(policy) - b, err := ioutil.ReadAll(policyReader) - if err != nil { - return err - } - - reqMetadata := requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentBody: policyReader, - contentLength: int64(len(b)), - } - - // Execute PUT to upload a new bucket policy. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) - defer closeResponse(resp) - if err != nil { - return err - } - if resp != nil { - if resp.StatusCode != http.StatusNoContent { - return httpRespToErrorResponse(resp, bucketName, "") - } - } - return nil -} - -// Removes all policies on a bucket. -func (c Client) removeBucketPolicy(ctx context.Context, bucketName string) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("policy", "") - - // Execute DELETE on objectName. - resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentSHA256Hex: emptySHA256Hex, - }) - defer closeResponse(resp) - if err != nil { - return err - } - return nil -} - -// SetBucketLifecycle set the lifecycle on an existing bucket. -func (c Client) SetBucketLifecycle(bucketName, lifecycle string) error { - return c.SetBucketLifecycleWithContext(context.Background(), bucketName, lifecycle) -} - -// SetBucketLifecycleWithContext set the lifecycle on an existing bucket with a context to control cancellations and timeouts. -func (c Client) SetBucketLifecycleWithContext(ctx context.Context, bucketName, lifecycle string) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - - // If lifecycle is empty then delete it. - if lifecycle == "" { - return c.removeBucketLifecycle(ctx, bucketName) - } - - // Save the updated lifecycle. - return c.putBucketLifecycle(ctx, bucketName, lifecycle) -} - -// Saves a new bucket lifecycle. -func (c Client) putBucketLifecycle(ctx context.Context, bucketName, lifecycle string) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("lifecycle", "") - - // Content-length is mandatory for put lifecycle request - lifecycleReader := strings.NewReader(lifecycle) - b, err := ioutil.ReadAll(lifecycleReader) - if err != nil { - return err - } - - reqMetadata := requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentBody: lifecycleReader, - contentLength: int64(len(b)), - contentMD5Base64: sumMD5Base64(b), - } - - // Execute PUT to upload a new bucket lifecycle. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) - defer closeResponse(resp) - if err != nil { - return err - } - if resp != nil { - if resp.StatusCode != http.StatusOK { - return httpRespToErrorResponse(resp, bucketName, "") - } - } - return nil -} - -// Remove lifecycle from a bucket. -func (c Client) removeBucketLifecycle(ctx context.Context, bucketName string) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("lifecycle", "") - - // Execute DELETE on objectName. - resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentSHA256Hex: emptySHA256Hex, - }) - defer closeResponse(resp) - if err != nil { - return err - } - return nil -} - -// SetBucketEncryption sets the default encryption configuration on an existing bucket. -func (c Client) SetBucketEncryption(bucketName string, configuration ServerSideEncryptionConfiguration) error { - return c.SetBucketEncryptionWithContext(context.Background(), bucketName, configuration) -} - -// SetBucketEncryptionWithContext sets the default encryption configuration on an existing bucket with a context to control cancellations and timeouts. -func (c Client) SetBucketEncryptionWithContext(ctx context.Context, bucketName string, configuration ServerSideEncryptionConfiguration) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - - buf, err := xml.Marshal(&configuration) - if err != nil { - return err - } - - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("encryption", "") - - // Content-length is mandatory to set a default encryption configuration - reqMetadata := requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentBody: bytes.NewReader(buf), - contentLength: int64(len(buf)), - contentMD5Base64: sumMD5Base64(buf), - } - - // Execute PUT to upload a new bucket default encryption configuration. - resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) - defer closeResponse(resp) - if err != nil { - return err - } - if resp.StatusCode != http.StatusOK { - return httpRespToErrorResponse(resp, bucketName, "") - } - return nil -} - -// DeleteBucketEncryption removes the default encryption configuration on a bucket. -func (c Client) DeleteBucketEncryption(bucketName string) error { - return c.DeleteBucketEncryptionWithContext(context.Background(), bucketName) -} - -// DeleteBucketEncryptionWithContext removes the default encryption configuration on a bucket with a context to control cancellations and timeouts. -func (c Client) DeleteBucketEncryptionWithContext(ctx context.Context, bucketName string) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("encryption", "") - - // DELETE default encryption configuration on a bucket. - resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentSHA256Hex: emptySHA256Hex, - }) - defer closeResponse(resp) - if err != nil { - return err - } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - return httpRespToErrorResponse(resp, bucketName, "") - } - return nil -} - -// SetBucketNotification saves a new bucket notification. -func (c Client) SetBucketNotification(bucketName string, bucketNotification BucketNotification) error { - return c.SetBucketNotificationWithContext(context.Background(), bucketName, bucketNotification) -} - -// SetBucketNotificationWithContext saves a new bucket notification with a context to control cancellations -// and timeouts. -func (c Client) SetBucketNotificationWithContext(ctx context.Context, bucketName string, bucketNotification BucketNotification) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("notification", "") - - notifBytes, err := xml.Marshal(bucketNotification) - if err != nil { - return err - } - - notifBuffer := bytes.NewReader(notifBytes) - reqMetadata := requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentBody: notifBuffer, - contentLength: int64(len(notifBytes)), - contentMD5Base64: sumMD5Base64(notifBytes), - contentSHA256Hex: sum256Hex(notifBytes), - } - - // Execute PUT to upload a new bucket notification. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) - defer closeResponse(resp) - if err != nil { - return err - } - if resp != nil { - if resp.StatusCode != http.StatusOK { - return httpRespToErrorResponse(resp, bucketName, "") - } - } - return nil -} - -// RemoveAllBucketNotification - Remove bucket notification clears all previously specified config -func (c Client) RemoveAllBucketNotification(bucketName string) error { - return c.SetBucketNotification(bucketName, BucketNotification{}) -} - -var ( - versionEnableConfig = []byte("Enabled") - versionEnableConfigLen = int64(len(versionEnableConfig)) - versionEnableConfigMD5Sum = sumMD5Base64(versionEnableConfig) - versionEnableConfigSHA256 = sum256Hex(versionEnableConfig) - - versionDisableConfig = []byte("Suspended") - versionDisableConfigLen = int64(len(versionDisableConfig)) - versionDisableConfigMD5Sum = sumMD5Base64(versionDisableConfig) - versionDisableConfigSHA256 = sum256Hex(versionDisableConfig) -) - -func (c Client) setVersioning(ctx context.Context, bucketName string, config []byte, length int64, md5sum, sha256sum string) error { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return err - } - - // Get resources properly escaped and lined up before - // using them in http request. - urlValues := make(url.Values) - urlValues.Set("versioning", "") - - reqMetadata := requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, - contentBody: bytes.NewReader(config), - contentLength: length, - contentMD5Base64: md5sum, - contentSHA256Hex: sha256sum, - } - - // Execute PUT to set a bucket versioning. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) - defer closeResponse(resp) - if err != nil { - return err - } - if resp != nil { - if resp.StatusCode != http.StatusOK { - return httpRespToErrorResponse(resp, bucketName, "") - } - } - return nil -} - -// EnableVersioning - Enable object versioning in given bucket. -func (c Client) EnableVersioning(bucketName string) error { - return c.EnableVersioningWithContext(context.Background(), bucketName) -} - -// EnableVersioningWithContext - Enable object versioning in given bucket with a context to control cancellations and timeouts. -func (c Client) EnableVersioningWithContext(ctx context.Context, bucketName string) error { - return c.setVersioning(ctx, bucketName, versionEnableConfig, versionEnableConfigLen, versionEnableConfigMD5Sum, versionEnableConfigSHA256) -} - -// DisableVersioning - Disable object versioning in given bucket. -func (c Client) DisableVersioning(bucketName string) error { - return c.DisableVersioningWithContext(context.Background(), bucketName) -} - -// DisableVersioningWithContext - Disable object versioning in given bucket with a context to control cancellations and timeouts. -func (c Client) DisableVersioningWithContext(ctx context.Context, bucketName string) error { - return c.setVersioning(ctx, bucketName, versionDisableConfig, versionDisableConfigLen, versionDisableConfigMD5Sum, versionDisableConfigSHA256) -} diff --git a/vendor/github.com/minio/minio-go/v6/api-put-object-context.go b/vendor/github.com/minio/minio-go/v6/api-put-object-context.go deleted file mode 100644 index 415a7878fd..0000000000 --- a/vendor/github.com/minio/minio-go/v6/api-put-object-context.go +++ /dev/null @@ -1,33 +0,0 @@ -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "context" - "io" -) - -// PutObjectWithContext - Identical to PutObject call, but accepts context to facilitate request cancellation. -func (c Client) PutObjectWithContext(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64, - opts PutObjectOptions) (n int64, err error) { - err = opts.validate() - if err != nil { - return 0, err - } - return c.putObjectCommon(ctx, bucketName, objectName, reader, objectSize, opts) -} diff --git a/vendor/github.com/minio/minio-go/v6/api-put-object-copy.go b/vendor/github.com/minio/minio-go/v6/api-put-object-copy.go deleted file mode 100644 index aa5a36c4de..0000000000 --- a/vendor/github.com/minio/minio-go/v6/api-put-object-copy.go +++ /dev/null @@ -1,99 +0,0 @@ -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017, 2018 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "context" - "io" - "io/ioutil" - "net/http" - "time" - - "github.com/minio/minio-go/v6/pkg/encrypt" - "github.com/minio/minio-go/v6/pkg/s3utils" -) - -// CopyObject - copy a source object into a new object -func (c Client) CopyObject(dst DestinationInfo, src SourceInfo) error { - return c.CopyObjectWithProgress(dst, src, nil) -} - -// CopyObjectWithProgress - copy a source object into a new object, optionally takes -// progress bar input to notify current progress. -func (c Client) CopyObjectWithProgress(dst DestinationInfo, src SourceInfo, progress io.Reader) error { - header := make(http.Header) - for k, v := range src.Headers { - header[k] = v - } - - if dst.opts.ReplaceTags && len(dst.opts.UserTags) != 0 { - header.Set(amzTaggingHeaderDirective, "REPLACE") - header.Set(amzTaggingHeader, s3utils.TagEncode(dst.opts.UserTags)) - } - - if dst.opts.LegalHold != LegalHoldStatus("") { - header.Set(amzLegalHoldHeader, dst.opts.LegalHold.String()) - } - - if dst.opts.Mode != RetentionMode("") && !dst.opts.RetainUntilDate.IsZero() { - header.Set(amzLockMode, dst.opts.Mode.String()) - header.Set(amzLockRetainUntil, dst.opts.RetainUntilDate.Format(time.RFC3339)) - } - - var err error - var size int64 - // If progress bar is specified, size should be requested as well initiate a StatObject request. - if progress != nil { - size, _, _, err = src.getProps(c) - if err != nil { - return err - } - } - - if src.encryption != nil { - encrypt.SSECopy(src.encryption).Marshal(header) - } - - if dst.opts.Encryption != nil { - dst.opts.Encryption.Marshal(header) - } - for k, v := range dst.getUserMetaHeadersMap(true) { - header.Set(k, v) - } - - resp, err := c.executeMethod(context.Background(), "PUT", requestMetadata{ - bucketName: dst.bucket, - objectName: dst.object, - customHeader: header, - }) - if err != nil { - return err - } - defer closeResponse(resp) - - if resp.StatusCode != http.StatusOK { - return httpRespToErrorResponse(resp, dst.bucket, dst.object) - } - - // Update the progress properly after successful copy. - if progress != nil { - io.CopyN(ioutil.Discard, progress, size) - } - - return nil -} diff --git a/vendor/github.com/minio/minio-go/v6/api-put-object-file.go b/vendor/github.com/minio/minio-go/v6/api-put-object-file.go deleted file mode 100644 index 23df6cd1c9..0000000000 --- a/vendor/github.com/minio/minio-go/v6/api-put-object-file.go +++ /dev/null @@ -1,27 +0,0 @@ -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "context" -) - -// FPutObject - Create an object in a bucket, with contents from file at filePath -func (c Client) FPutObject(bucketName, objectName, filePath string, opts PutObjectOptions) (n int64, err error) { - return c.FPutObjectWithContext(context.Background(), bucketName, objectName, filePath, opts) -} diff --git a/vendor/github.com/minio/minio-go/v6/core.go b/vendor/github.com/minio/minio-go/v6/core.go deleted file mode 100644 index 6d1fddb2e1..0000000000 --- a/vendor/github.com/minio/minio-go/v6/core.go +++ /dev/null @@ -1,185 +0,0 @@ -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "context" - "io" - "net/http" - - "github.com/minio/minio-go/v6/pkg/encrypt" -) - -// Core - Inherits Client and adds new methods to expose the low level S3 APIs. -type Core struct { - *Client -} - -// NewCore - Returns new initialized a Core client, this CoreClient should be -// only used under special conditions such as need to access lower primitives -// and being able to use them to write your own wrappers. -func NewCore(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Core, error) { - var s3Client Core - client, err := NewV4(endpoint, accessKeyID, secretAccessKey, secure) - if err != nil { - return nil, err - } - s3Client.Client = client - return &s3Client, nil -} - -// ListObjects - List all the objects at a prefix, optionally with marker and delimiter -// you can further filter the results. -func (c Core) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListBucketResult, err error) { - return c.listObjectsQuery(bucket, prefix, marker, delimiter, maxKeys) -} - -// ListObjectsV2 - Lists all the objects at a prefix, similar to ListObjects() but uses -// continuationToken instead of marker to support iteration over the results. -func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) { - return c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, false, delimiter, maxkeys, startAfter) -} - -// CopyObjectWithContext - copies an object from source object to destination object on server side. -func (c Core) CopyObjectWithContext(ctx context.Context, sourceBucket, sourceObject, destBucket, destObject string, metadata map[string]string) (ObjectInfo, error) { - return c.copyObjectDo(ctx, sourceBucket, sourceObject, destBucket, destObject, metadata) -} - -// CopyObject - copies an object from source object to destination object on server side. -func (c Core) CopyObject(sourceBucket, sourceObject, destBucket, destObject string, metadata map[string]string) (ObjectInfo, error) { - return c.CopyObjectWithContext(context.Background(), sourceBucket, sourceObject, destBucket, destObject, metadata) -} - -// CopyObjectPartWithContext - creates a part in a multipart upload by copying (a -// part of) an existing object. -func (c Core) CopyObjectPartWithContext(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, - partID int, startOffset, length int64, metadata map[string]string) (p CompletePart, err error) { - - return c.copyObjectPartDo(ctx, srcBucket, srcObject, destBucket, destObject, uploadID, - partID, startOffset, length, metadata) -} - -// CopyObjectPart - creates a part in a multipart upload by copying (a -// part of) an existing object. -func (c Core) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string, uploadID string, - partID int, startOffset, length int64, metadata map[string]string) (p CompletePart, err error) { - - return c.CopyObjectPartWithContext(context.Background(), srcBucket, srcObject, destBucket, destObject, uploadID, - partID, startOffset, length, metadata) -} - -// PutObjectWithContext - Upload object. Uploads using single PUT call. -func (c Core) PutObjectWithContext(ctx context.Context, bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, opts PutObjectOptions) (ObjectInfo, error) { - return c.putObjectDo(ctx, bucket, object, data, md5Base64, sha256Hex, size, opts) -} - -// PutObject - Upload object. Uploads using single PUT call. -func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, opts PutObjectOptions) (ObjectInfo, error) { - return c.PutObjectWithContext(context.Background(), bucket, object, data, size, md5Base64, sha256Hex, opts) -} - -// NewMultipartUpload - Initiates new multipart upload and returns the new uploadID. -func (c Core) NewMultipartUpload(bucket, object string, opts PutObjectOptions) (uploadID string, err error) { - result, err := c.initiateMultipartUpload(context.Background(), bucket, object, opts) - return result.UploadID, err -} - -// ListMultipartUploads - List incomplete uploads. -func (c Core) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartUploadsResult, err error) { - return c.listMultipartUploadsQuery(bucket, keyMarker, uploadIDMarker, prefix, delimiter, maxUploads) -} - -// PutObjectPartWithContext - Upload an object part. -func (c Core) PutObjectPartWithContext(ctx context.Context, bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string, sse encrypt.ServerSide) (ObjectPart, error) { - return c.uploadPart(ctx, bucket, object, uploadID, data, partID, md5Base64, sha256Hex, size, sse) -} - -// PutObjectPart - Upload an object part. -func (c Core) PutObjectPart(bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string, sse encrypt.ServerSide) (ObjectPart, error) { - return c.PutObjectPartWithContext(context.Background(), bucket, object, uploadID, partID, data, size, md5Base64, sha256Hex, sse) -} - -// ListObjectParts - List uploaded parts of an incomplete upload.x -func (c Core) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListObjectPartsResult, err error) { - return c.listObjectPartsQuery(bucket, object, uploadID, partNumberMarker, maxParts) -} - -// CompleteMultipartUploadWithContext - Concatenate uploaded parts and commit to an object. -func (c Core) CompleteMultipartUploadWithContext(ctx context.Context, bucket, object, uploadID string, parts []CompletePart) (string, error) { - res, err := c.completeMultipartUpload(ctx, bucket, object, uploadID, completeMultipartUpload{ - Parts: parts, - }) - return res.ETag, err -} - -// CompleteMultipartUpload - Concatenate uploaded parts and commit to an object. -func (c Core) CompleteMultipartUpload(bucket, object, uploadID string, parts []CompletePart) (string, error) { - return c.CompleteMultipartUploadWithContext(context.Background(), bucket, object, uploadID, parts) -} - -// AbortMultipartUploadWithContext - Abort an incomplete upload. -func (c Core) AbortMultipartUploadWithContext(ctx context.Context, bucket, object, uploadID string) error { - return c.abortMultipartUpload(ctx, bucket, object, uploadID) -} - -// AbortMultipartUpload - Abort an incomplete upload. -func (c Core) AbortMultipartUpload(bucket, object, uploadID string) error { - return c.AbortMultipartUploadWithContext(context.Background(), bucket, object, uploadID) -} - -// GetBucketPolicy - fetches bucket access policy for a given bucket. -func (c Core) GetBucketPolicy(bucket string) (string, error) { - return c.getBucketPolicy(bucket) -} - -// PutBucketPolicy - applies a new bucket access policy for a given bucket. -func (c Core) PutBucketPolicy(bucket, bucketPolicy string) error { - return c.PutBucketPolicyWithContext(context.Background(), bucket, bucketPolicy) -} - -// PutBucketPolicyWithContext - applies a new bucket access policy for a given bucket with a context to control -// cancellations and timeouts. -func (c Core) PutBucketPolicyWithContext(ctx context.Context, bucket, bucketPolicy string) error { - return c.putBucketPolicy(ctx, bucket, bucketPolicy) -} - -// GetObjectWithContext is a lower level API implemented to support reading -// partial objects and also downloading objects with special conditions -// matching etag, modtime etc. -func (c Core) GetObjectWithContext(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, http.Header, error) { - return c.getObject(ctx, bucketName, objectName, opts) -} - -// GetObject is a lower level API implemented to support reading -// partial objects and also downloading objects with special conditions -// matching etag, modtime etc. -func (c Core) GetObject(bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, http.Header, error) { - return c.GetObjectWithContext(context.Background(), bucketName, objectName, opts) -} - -// StatObjectWithContext is a lower level API implemented to support special -// conditions matching etag, modtime on a request. -func (c Core) StatObjectWithContext(ctx context.Context, bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { - return c.statObject(ctx, bucketName, objectName, opts) -} - -// StatObject is a lower level API implemented to support special -// conditions matching etag, modtime on a request. -func (c Core) StatObject(bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { - return c.StatObjectWithContext(context.Background(), bucketName, objectName, opts) -} diff --git a/vendor/github.com/minio/minio-go/v6/go.mod b/vendor/github.com/minio/minio-go/v6/go.mod deleted file mode 100644 index 03497bccf4..0000000000 --- a/vendor/github.com/minio/minio-go/v6/go.mod +++ /dev/null @@ -1,16 +0,0 @@ -module github.com/minio/minio-go/v6 - -go 1.12 - -require ( - github.com/dustin/go-humanize v1.0.0 // indirect - github.com/json-iterator/go v1.1.9 - github.com/minio/md5-simd v1.1.0 - github.com/minio/sha256-simd v0.1.1 - github.com/mitchellh/go-homedir v1.1.0 - github.com/sirupsen/logrus v1.5.0 // indirect - github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect - golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f - golang.org/x/net v0.0.0-20190522155817-f3200d17e092 - gopkg.in/ini.v1 v1.42.0 -) diff --git a/vendor/github.com/minio/minio-go/v6/.gitignore b/vendor/github.com/minio/minio-go/v7/.gitignore similarity index 100% rename from vendor/github.com/minio/minio-go/v6/.gitignore rename to vendor/github.com/minio/minio-go/v7/.gitignore diff --git a/vendor/github.com/minio/minio-go/v6/.golangci.yml b/vendor/github.com/minio/minio-go/v7/.golangci.yml similarity index 100% rename from vendor/github.com/minio/minio-go/v6/.golangci.yml rename to vendor/github.com/minio/minio-go/v7/.golangci.yml diff --git a/vendor/github.com/minio/minio-go/v7/CNAME b/vendor/github.com/minio/minio-go/v7/CNAME new file mode 100644 index 0000000000..d365a7bb29 --- /dev/null +++ b/vendor/github.com/minio/minio-go/v7/CNAME @@ -0,0 +1 @@ +minio-go.min.io \ No newline at end of file diff --git a/vendor/github.com/minio/minio-go/v6/CONTRIBUTING.md b/vendor/github.com/minio/minio-go/v7/CONTRIBUTING.md similarity index 100% rename from vendor/github.com/minio/minio-go/v6/CONTRIBUTING.md rename to vendor/github.com/minio/minio-go/v7/CONTRIBUTING.md diff --git a/vendor/github.com/minio/minio-go/v6/LICENSE b/vendor/github.com/minio/minio-go/v7/LICENSE similarity index 100% rename from vendor/github.com/minio/minio-go/v6/LICENSE rename to vendor/github.com/minio/minio-go/v7/LICENSE diff --git a/vendor/github.com/minio/minio-go/v6/MAINTAINERS.md b/vendor/github.com/minio/minio-go/v7/MAINTAINERS.md similarity index 100% rename from vendor/github.com/minio/minio-go/v6/MAINTAINERS.md rename to vendor/github.com/minio/minio-go/v7/MAINTAINERS.md diff --git a/vendor/github.com/minio/minio-go/v6/Makefile b/vendor/github.com/minio/minio-go/v7/Makefile similarity index 57% rename from vendor/github.com/minio/minio-go/v6/Makefile rename to vendor/github.com/minio/minio-go/v7/Makefile index afe8d978fb..d9b433443f 100644 --- a/vendor/github.com/minio/minio-go/v6/Makefile +++ b/vendor/github.com/minio/minio-go/v7/Makefile @@ -1,12 +1,17 @@ +GOPATH := $(shell go env GOPATH) + all: checks .PHONY: examples docs -checks: diff vet test examples functional-test +checks: lint vet test examples functional-test -diff: - @curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b . v1.21.0 - @./golangci-lint run --timeout=5m --config ./.golangci.yml +lint: + @mkdir -p ${GOPATH}/bin + @which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0) + @echo "Running $@ check" + @GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean + @GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=5m --config ./.golangci.yml vet: @GO111MODULE=on go vet ./... diff --git a/vendor/github.com/minio/minio-go/v7/NOTICE b/vendor/github.com/minio/minio-go/v7/NOTICE new file mode 100644 index 0000000000..1e8fd3b92a --- /dev/null +++ b/vendor/github.com/minio/minio-go/v7/NOTICE @@ -0,0 +1,9 @@ +MinIO Cloud Storage, (C) 2014-2020 MinIO, Inc. + +This product includes software developed at MinIO, Inc. +(https://min.io/). + +The MinIO project contains unmodified/modified subcomponents too with +separate copyright notices and license terms. Your use of the source +code for these subcomponents is subject to the terms and conditions +of Apache License Version 2.0 diff --git a/vendor/github.com/minio/minio-go/v6/README.md b/vendor/github.com/minio/minio-go/v7/README.md similarity index 88% rename from vendor/github.com/minio/minio-go/v6/README.md rename to vendor/github.com/minio/minio-go/v7/README.md index 33eb6c44de..0c83e9efb1 100644 --- a/vendor/github.com/minio/minio-go/v6/README.md +++ b/vendor/github.com/minio/minio-go/v7/README.md @@ -8,7 +8,7 @@ This document assumes that you have a working [Go development environment](https ## Download from Github ```sh -GO111MODULE=on go get github.com/minio/minio-go/v6 +GO111MODULE=on go get github.com/minio/minio-go/v7 ``` ## Initialize MinIO Client @@ -17,17 +17,17 @@ MinIO client requires the following four parameters specified to connect to an A | Parameter | Description| | :--- | :--- | | endpoint | URL to object storage service. | -| accessKeyID | Access key is the user ID that uniquely identifies your account. | -| secretAccessKey | Secret key is the password to your account. | -| secure | Set this value to 'true' to enable secure (HTTPS) access. | +| _minio.Options_ | All the options such as credentials, custom transport etc. | ```go package main import ( - "github.com/minio/minio-go/v6" "log" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -37,7 +37,10 @@ func main() { useSSL := true // Initialize minio client object. - minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL) + minioClient, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: useSSL, + }) if err != nil { log.Fatalln(err) } @@ -56,8 +59,11 @@ We will use the MinIO server running at [https://play.min.io](https://play.min.i package main import ( - "github.com/minio/minio-go/v6" + "context" "log" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -67,7 +73,10 @@ func main() { useSSL := true // Initialize minio client object. - minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL) + minioClient, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: useSSL, + }) if err != nil { log.Fatalln(err) } @@ -76,7 +85,7 @@ func main() { bucketName := "mymusic" location := "us-east-1" - err = minioClient.MakeBucket(bucketName, location) + err = minioClient.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: location}) if err != nil { // Check to see if we already own this bucket (which happens if you run this twice) exists, errBucketExists := minioClient.BucketExists(bucketName) @@ -95,7 +104,7 @@ func main() { contentType := "application/zip" // Upload the zip file with FPutObject - n, err := minioClient.FPutObject(bucketName, objectName, filePath, minio.PutObjectOptions{ContentType:contentType}) + n, err := minioClient.FPutObject(context.Background(), bucketName, objectName, filePath, minio.PutObjectOptions{ContentType: contentType}) if err != nil { log.Fatalln(err) } @@ -137,18 +146,15 @@ The full API Reference is available here. * [`GetBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#GetBucketNotification) * [`RemoveAllBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#RemoveAllBucketNotification) * [`ListenBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#ListenBucketNotification) (MinIO Extension) +* [`ListenNotification`](https://docs.min.io/docs/golang-client-api-reference#ListenNotification) (MinIO Extension) ### API Reference : File Object Operations * [`FPutObject`](https://docs.min.io/docs/golang-client-api-reference#FPutObject) * [`FGetObject`](https://docs.min.io/docs/golang-client-api-reference#FGetObject) -* [`FPutObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#FPutObjectWithContext) -* [`FGetObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#FGetObjectWithContext) ### API Reference : Object Operations * [`GetObject`](https://docs.min.io/docs/golang-client-api-reference#GetObject) * [`PutObject`](https://docs.min.io/docs/golang-client-api-reference#PutObject) -* [`GetObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#GetObjectWithContext) -* [`PutObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#PutObjectWithContext) * [`PutObjectStreaming`](https://docs.min.io/docs/golang-client-api-reference#PutObjectStreaming) * [`StatObject`](https://docs.min.io/docs/golang-client-api-reference#StatObject) * [`CopyObject`](https://docs.min.io/docs/golang-client-api-reference#CopyObject) @@ -195,11 +201,17 @@ The full API Reference is available here. * [getbucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketencryption.go) * [deletebucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/deletebucketencryption.go) +### Full Examples : Bucket replication Operations +* [setbucketreplication.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketreplication.go) +* [getbucketreplication.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketreplication.go) +* [removebucketreplication.go](https://github.com/minio/minio-go/blob/master/examples/s3/removebucketreplication.go) + ### Full Examples : Bucket notification Operations * [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go) * [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go) * [removeallbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeallbucketnotification.go) * [listenbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listenbucketnotification.go) (MinIO Extension) +* [listennotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listen-notification.go) (MinIO Extension) ### Full Examples : File Object Operations * [fputobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputobject.go) @@ -236,8 +248,5 @@ The full API Reference is available here. ## Contribute [Contributors Guide](https://github.com/minio/minio-go/blob/master/CONTRIBUTING.md) -[![Build Status](https://travis-ci.org/minio/minio-go.svg)](https://travis-ci.org/minio/minio-go) -[![Build status](https://ci.appveyor.com/api/projects/status/1d05e6nvxcelmrak?svg=true)](https://ci.appveyor.com/project/harshavardhana/minio-go) - ## License This SDK is distributed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0), see [LICENSE](./LICENSE) and [NOTICE](./NOTICE) for more information. diff --git a/vendor/github.com/minio/minio-go/v6/README_zh_CN.md b/vendor/github.com/minio/minio-go/v7/README_zh_CN.md similarity index 92% rename from vendor/github.com/minio/minio-go/v6/README_zh_CN.md rename to vendor/github.com/minio/minio-go/v7/README_zh_CN.md index b5ba0c1d02..0911b0905a 100644 --- a/vendor/github.com/minio/minio-go/v6/README_zh_CN.md +++ b/vendor/github.com/minio/minio-go/v7/README_zh_CN.md @@ -2,7 +2,7 @@ MinIO Go Client SDK提供了简单的API来访问任何与Amazon S3兼容的对象存储服务。 -**支持的云存储:** +**支持的云存储:** - AWS Signature Version 4 - Amazon S3 @@ -26,10 +26,10 @@ go get -u github.com/minio/minio-go ## 初始化MinIO Client MinIO client需要以下4个参数来连接与Amazon S3兼容的对象存储。 -| 参数 | 描述| +| 参数 | 描述| | :--- | :--- | -| endpoint | 对象存储服务的URL | -| accessKeyID | Access key是唯一标识你的账户的用户ID。 | +| endpoint | 对象存储服务的URL | +| accessKeyID | Access key是唯一标识你的账户的用户ID。 | | secretAccessKey | Secret key是你账户的密码。 | | secure | true代表使用HTTPS | @@ -38,7 +38,7 @@ MinIO client需要以下4个参数来连接与Amazon S3兼容的对象存储。 package main import ( - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" "log" ) @@ -68,7 +68,7 @@ func main() { package main import ( - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" "log" ) @@ -118,7 +118,7 @@ func main() { ### 运行FileUploader ```sh go run file-uploader.go -2016/08/13 17:03:28 Successfully created mymusic +2016/08/13 17:03:28 Successfully created mymusic 2016/08/13 17:03:40 Successfully uploaded golden-oldies.zip of size 16253413 mc ls play/mymusic/ @@ -151,14 +151,10 @@ mc ls play/mymusic/ ### API文档 : 操作文件对象 * [`FPutObject`](https://docs.min.io/docs/golang-client-api-reference#FPutObject) * [`FGetObject`](https://docs.min.io/docs/golang-client-api-reference#FPutObject) -* [`FPutObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#FPutObjectWithContext) -* [`FGetObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#FGetObjectWithContext) ### API文档 : 操作对象 * [`GetObject`](https://docs.min.io/docs/golang-client-api-reference#GetObject) * [`PutObject`](https://docs.min.io/docs/golang-client-api-reference#PutObject) -* [`GetObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#GetObjectWithContext) -* [`PutObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#PutObjectWithContext) * [`PutObjectStreaming`](https://docs.min.io/docs/golang-client-api-reference#PutObjectStreaming) * [`StatObject`](https://docs.min.io/docs/golang-client-api-reference#StatObject) * [`CopyObject`](https://docs.min.io/docs/golang-client-api-reference#CopyObject) @@ -197,7 +193,7 @@ mc ls play/mymusic/ * [setbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketpolicy.go) * [getbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketpolicy.go) * [listbucketpolicies.go](https://github.com/minio/minio-go/blob/master/examples/s3/listbucketpolicies.go) - + ### 完整示例 : 存储桶通知 * [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go) * [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go) @@ -238,7 +234,3 @@ mc ls play/mymusic/ ## 贡献 [贡献指南](https://github.com/minio/minio-go/blob/master/docs/zh_CN/CONTRIBUTING.md) - -[![Build Status](https://travis-ci.org/minio/minio-go.svg)](https://travis-ci.org/minio/minio-go) -[![Build status](https://ci.appveyor.com/api/projects/status/1d05e6nvxcelmrak?svg=true)](https://ci.appveyor.com/project/harshavardhana/minio-go) - diff --git a/vendor/github.com/minio/minio-go/v7/api-bucket-encryption.go b/vendor/github.com/minio/minio-go/v7/api-bucket-encryption.go new file mode 100644 index 0000000000..e02ab84afa --- /dev/null +++ b/vendor/github.com/minio/minio-go/v7/api-bucket-encryption.go @@ -0,0 +1,134 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/xml" + "net/http" + "net/url" + + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/sse" +) + +// SetBucketEncryption sets the default encryption configuration on an existing bucket. +func (c Client) SetBucketEncryption(ctx context.Context, bucketName string, config *sse.Configuration) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + if config == nil { + return errInvalidArgument("configuration cannot be empty") + } + + buf, err := xml.Marshal(config) + if err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("encryption", "") + + // Content-length is mandatory to set a default encryption configuration + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: bytes.NewReader(buf), + contentLength: int64(len(buf)), + contentMD5Base64: sumMD5Base64(buf), + } + + // Execute PUT to upload a new bucket default encryption configuration. + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + return nil +} + +// RemoveBucketEncryption removes the default encryption configuration on a bucket with a context to control cancellations and timeouts. +func (c Client) RemoveBucketEncryption(ctx context.Context, bucketName string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("encryption", "") + + // DELETE default encryption configuration on a bucket. + resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return httpRespToErrorResponse(resp, bucketName, "") + } + return nil +} + +// GetBucketEncryption gets the default encryption configuration +// on an existing bucket with a context to control cancellations and timeouts. +func (c Client) GetBucketEncryption(ctx context.Context, bucketName string) (*sse.Configuration, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return nil, err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("encryption", "") + + // Execute GET on bucket to get the default encryption configuration. + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + }) + + defer closeResponse(resp) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, httpRespToErrorResponse(resp, bucketName, "") + } + + encryptionConfig := &sse.Configuration{} + if err = xmlDecoder(resp.Body, encryptionConfig); err != nil { + return nil, err + } + + return encryptionConfig, nil +} diff --git a/vendor/github.com/minio/minio-go/v7/api-bucket-lifecycle.go b/vendor/github.com/minio/minio-go/v7/api-bucket-lifecycle.go new file mode 100644 index 0000000000..e1fac813cc --- /dev/null +++ b/vendor/github.com/minio/minio-go/v7/api-bucket-lifecycle.go @@ -0,0 +1,147 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/xml" + "io/ioutil" + "net/http" + "net/url" + + "github.com/minio/minio-go/v7/pkg/lifecycle" + "github.com/minio/minio-go/v7/pkg/s3utils" +) + +// SetBucketLifecycle set the lifecycle on an existing bucket. +func (c Client) SetBucketLifecycle(ctx context.Context, bucketName string, config *lifecycle.Configuration) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // If lifecycle is empty then delete it. + if config.Empty() { + return c.removeBucketLifecycle(ctx, bucketName) + } + + buf, err := xml.Marshal(config) + if err != nil { + return err + } + + // Save the updated lifecycle. + return c.putBucketLifecycle(ctx, bucketName, buf) +} + +// Saves a new bucket lifecycle. +func (c Client) putBucketLifecycle(ctx context.Context, bucketName string, buf []byte) error { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("lifecycle", "") + + // Content-length is mandatory for put lifecycle request + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: bytes.NewReader(buf), + contentLength: int64(len(buf)), + contentMD5Base64: sumMD5Base64(buf), + } + + // Execute PUT to upload a new bucket lifecycle. + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + } + return nil +} + +// Remove lifecycle from a bucket. +func (c Client) removeBucketLifecycle(ctx context.Context, bucketName string) error { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("lifecycle", "") + + // Execute DELETE on objectName. + resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return err + } + return nil +} + +// GetBucketLifecycle fetch bucket lifecycle configuration +func (c Client) GetBucketLifecycle(ctx context.Context, bucketName string) (*lifecycle.Configuration, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return nil, err + } + + bucketLifecycle, err := c.getBucketLifecycle(ctx, bucketName) + if err != nil { + return nil, err + } + + config := lifecycle.NewConfiguration() + if err = xml.Unmarshal(bucketLifecycle, config); err != nil { + return nil, err + } + return config, nil +} + +// Request server for current bucket lifecycle. +func (c Client) getBucketLifecycle(ctx context.Context, bucketName string) ([]byte, error) { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("lifecycle", "") + + // Execute GET on bucket to get lifecycle. + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + }) + + defer closeResponse(resp) + if err != nil { + return nil, err + } + + if resp != nil { + if resp.StatusCode != http.StatusOK { + return nil, httpRespToErrorResponse(resp, bucketName, "") + } + } + + return ioutil.ReadAll(resp.Body) +} diff --git a/vendor/github.com/minio/minio-go/v6/api-notification.go b/vendor/github.com/minio/minio-go/v7/api-bucket-notification.go similarity index 51% rename from vendor/github.com/minio/minio-go/v6/api-notification.go rename to vendor/github.com/minio/minio-go/v7/api-bucket-notification.go index 24ee121e38..76787ecabc 100644 --- a/vendor/github.com/minio/minio-go/v6/api-notification.go +++ b/vendor/github.com/minio/minio-go/v7/api-bucket-notification.go @@ -19,35 +19,80 @@ package minio import ( "bufio" + "bytes" "context" + "encoding/xml" "net/http" "net/url" "time" jsoniter "github.com/json-iterator/go" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/notification" + "github.com/minio/minio-go/v7/pkg/s3utils" ) -// GetBucketNotification - get bucket notification at a given path. -func (c Client) GetBucketNotification(bucketName string) (bucketNotification BucketNotification, err error) { +// SetBucketNotification saves a new bucket notification with a context to control cancellations and timeouts. +func (c Client) SetBucketNotification(ctx context.Context, bucketName string, config notification.Configuration) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return BucketNotification{}, err + return err } - notification, err := c.getBucketNotification(bucketName) + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("notification", "") + + notifBytes, err := xml.Marshal(&config) if err != nil { - return BucketNotification{}, err + return err } - return notification, nil + + notifBuffer := bytes.NewReader(notifBytes) + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: notifBuffer, + contentLength: int64(len(notifBytes)), + contentMD5Base64: sumMD5Base64(notifBytes), + contentSHA256Hex: sum256Hex(notifBytes), + } + + // Execute PUT to upload a new bucket notification. + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + } + return nil +} + +// RemoveAllBucketNotification - Remove bucket notification clears all previously specified config +func (c Client) RemoveAllBucketNotification(ctx context.Context, bucketName string) error { + return c.SetBucketNotification(ctx, bucketName, notification.Configuration{}) +} + +// GetBucketNotification returns current bucket notification configuration +func (c Client) GetBucketNotification(ctx context.Context, bucketName string) (bucketNotification notification.Configuration, err error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return notification.Configuration{}, err + } + return c.getBucketNotification(ctx, bucketName) } // Request server for notification rules. -func (c Client) getBucketNotification(bucketName string) (BucketNotification, error) { +func (c Client) getBucketNotification(ctx context.Context, bucketName string) (notification.Configuration, error) { urlValues := make(url.Values) urlValues.Set("notification", "") // Execute GET on bucket to list objects. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, @@ -55,113 +100,60 @@ func (c Client) getBucketNotification(bucketName string) (BucketNotification, er defer closeResponse(resp) if err != nil { - return BucketNotification{}, err + return notification.Configuration{}, err } return processBucketNotificationResponse(bucketName, resp) } // processes the GetNotification http response from the server. -func processBucketNotificationResponse(bucketName string, resp *http.Response) (BucketNotification, error) { +func processBucketNotificationResponse(bucketName string, resp *http.Response) (notification.Configuration, error) { if resp.StatusCode != http.StatusOK { errResponse := httpRespToErrorResponse(resp, bucketName, "") - return BucketNotification{}, errResponse + return notification.Configuration{}, errResponse } - var bucketNotification BucketNotification + var bucketNotification notification.Configuration err := xmlDecoder(resp.Body, &bucketNotification) if err != nil { - return BucketNotification{}, err + return notification.Configuration{}, err } return bucketNotification, nil } -// Indentity represents the user id, this is a compliance field. -type identity struct { - PrincipalID string `json:"principalId"` +// ListenNotification listen for all events, this is a MinIO specific API +func (c Client) ListenNotification(ctx context.Context, prefix, suffix string, events []string) <-chan notification.Info { + return c.ListenBucketNotification(ctx, "", prefix, suffix, events) } -// Notification event bucket metadata. -type bucketMeta struct { - Name string `json:"name"` - OwnerIdentity identity `json:"ownerIdentity"` - ARN string `json:"arn"` -} - -// Notification event object metadata. -type objectMeta struct { - Key string `json:"key"` - Size int64 `json:"size,omitempty"` - ETag string `json:"eTag,omitempty"` - ContentType string `json:"contentType,omitempty"` - UserMetadata map[string]string `json:"userMetadata,omitempty"` - VersionID string `json:"versionId,omitempty"` - Sequencer string `json:"sequencer"` -} - -// Notification event server specific metadata. -type eventMeta struct { - SchemaVersion string `json:"s3SchemaVersion"` - ConfigurationID string `json:"configurationId"` - Bucket bucketMeta `json:"bucket"` - Object objectMeta `json:"object"` -} - -// sourceInfo represents information on the client that -// triggered the event notification. -type sourceInfo struct { - Host string `json:"host"` - Port string `json:"port"` - UserAgent string `json:"userAgent"` -} - -// NotificationEvent represents an Amazon an S3 bucket notification event. -type NotificationEvent struct { - EventVersion string `json:"eventVersion"` - EventSource string `json:"eventSource"` - AwsRegion string `json:"awsRegion"` - EventTime string `json:"eventTime"` - EventName string `json:"eventName"` - UserIdentity identity `json:"userIdentity"` - RequestParameters map[string]string `json:"requestParameters"` - ResponseElements map[string]string `json:"responseElements"` - S3 eventMeta `json:"s3"` - Source sourceInfo `json:"source"` -} - -// NotificationInfo - represents the collection of notification events, additionally -// also reports errors if any while listening on bucket notifications. -type NotificationInfo struct { - Records []NotificationEvent - Err error -} - -// ListenBucketNotification - listen on bucket notifications. -func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, events []string, doneCh <-chan struct{}) <-chan NotificationInfo { - notificationInfoCh := make(chan NotificationInfo, 1) - const notificationCapacity = 1024 * 1024 +// ListenBucketNotification listen for bucket events, this is a MinIO specific API +func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix, suffix string, events []string) <-chan notification.Info { + notificationInfoCh := make(chan notification.Info, 1) + const notificationCapacity = 4 * 1024 * 1024 notificationEventBuffer := make([]byte, notificationCapacity) // Only success, start a routine to start reading line by line. - go func(notificationInfoCh chan<- NotificationInfo) { + go func(notificationInfoCh chan<- notification.Info) { defer close(notificationInfoCh) // Validate the bucket name. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - select { - case notificationInfoCh <- NotificationInfo{ - Err: err, - }: - case <-doneCh: + if bucketName != "" { + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + select { + case notificationInfoCh <- notification.Info{ + Err: err, + }: + case <-ctx.Done(): + } + return } - return } // Check ARN partition to verify if listening bucket is supported if s3utils.IsAmazonEndpoint(*c.endpointURL) || s3utils.IsGoogleEndpoint(*c.endpointURL) { select { - case notificationInfoCh <- NotificationInfo{ - Err: ErrAPINotSupported("Listening for bucket notification is specific only to `minio` server endpoints"), + case notificationInfoCh <- notification.Info{ + Err: errAPINotSupported("Listening for bucket notification is specific only to `minio` server endpoints"), }: - case <-doneCh: + case <-ctx.Done(): } return } @@ -182,17 +174,17 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even // Wait on the jitter retry loop. for range c.newRetryTimerContinous(time.Second, time.Second*30, MaxJitter, retryDoneCh) { // Execute GET on bucket to list objects. - resp, err := c.executeMethod(context.Background(), http.MethodGet, requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, }) if err != nil { select { - case notificationInfoCh <- NotificationInfo{ + case notificationInfoCh <- notification.Info{ Err: err, }: - case <-doneCh: + case <-ctx.Done(): } return } @@ -201,10 +193,10 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even if resp.StatusCode != http.StatusOK { errResponse := httpRespToErrorResponse(resp, bucketName, "") select { - case notificationInfoCh <- NotificationInfo{ + case notificationInfoCh <- notification.Info{ Err: errResponse, }: - case <-doneCh: + case <-ctx.Done(): } return } @@ -219,15 +211,15 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even // Unmarshal each line, returns marshaled values. for bio.Scan() { - var notificationInfo NotificationInfo + var notificationInfo notification.Info if err = json.Unmarshal(bio.Bytes(), ¬ificationInfo); err != nil { // Unexpected error during json unmarshal, send // the error to caller for actionable as needed. select { - case notificationInfoCh <- NotificationInfo{ + case notificationInfoCh <- notification.Info{ Err: err, }: - case <-doneCh: + case <-ctx.Done(): return } closeResponse(resp) @@ -236,7 +228,7 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even // Send notificationInfo select { case notificationInfoCh <- notificationInfo: - case <-doneCh: + case <-ctx.Done(): closeResponse(resp) return } @@ -244,10 +236,10 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even if err = bio.Err(); err != nil { select { - case notificationInfoCh <- NotificationInfo{ + case notificationInfoCh <- notification.Info{ Err: err, }: - case <-doneCh: + case <-ctx.Done(): return } } diff --git a/vendor/github.com/minio/minio-go/v7/api-bucket-policy.go b/vendor/github.com/minio/minio-go/v7/api-bucket-policy.go new file mode 100644 index 0000000000..72676f3447 --- /dev/null +++ b/vendor/github.com/minio/minio-go/v7/api-bucket-policy.go @@ -0,0 +1,142 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "io/ioutil" + "net/http" + "net/url" + "strings" + + "github.com/minio/minio-go/v7/pkg/s3utils" +) + +// SetBucketPolicy sets the access permissions on an existing bucket. +func (c Client) SetBucketPolicy(ctx context.Context, bucketName, policy string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // If policy is empty then delete the bucket policy. + if policy == "" { + return c.removeBucketPolicy(ctx, bucketName) + } + + // Save the updated policies. + return c.putBucketPolicy(ctx, bucketName, policy) +} + +// Saves a new bucket policy. +func (c Client) putBucketPolicy(ctx context.Context, bucketName, policy string) error { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("policy", "") + + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: strings.NewReader(policy), + contentLength: int64(len(policy)), + } + + // Execute PUT to upload a new bucket policy. + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusNoContent { + return httpRespToErrorResponse(resp, bucketName, "") + } + } + return nil +} + +// Removes all policies on a bucket. +func (c Client) removeBucketPolicy(ctx context.Context, bucketName string) error { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("policy", "") + + // Execute DELETE on objectName. + resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return err + } + return nil +} + +// GetBucketPolicy returns the current policy +func (c Client) GetBucketPolicy(ctx context.Context, bucketName string) (string, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return "", err + } + bucketPolicy, err := c.getBucketPolicy(ctx, bucketName) + if err != nil { + errResponse := ToErrorResponse(err) + if errResponse.Code == "NoSuchBucketPolicy" { + return "", nil + } + return "", err + } + return bucketPolicy, nil +} + +// Request server for current bucket policy. +func (c Client) getBucketPolicy(ctx context.Context, bucketName string) (string, error) { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("policy", "") + + // Execute GET on bucket to list objects. + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + + defer closeResponse(resp) + if err != nil { + return "", err + } + + if resp != nil { + if resp.StatusCode != http.StatusOK { + return "", httpRespToErrorResponse(resp, bucketName, "") + } + } + + bucketPolicyBuf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + policy := string(bucketPolicyBuf) + return policy, err +} diff --git a/vendor/github.com/minio/minio-go/v7/api-bucket-replication.go b/vendor/github.com/minio/minio-go/v7/api-bucket-replication.go new file mode 100644 index 0000000000..bfd5ea4360 --- /dev/null +++ b/vendor/github.com/minio/minio-go/v7/api-bucket-replication.go @@ -0,0 +1,149 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/xml" + "net/http" + "net/url" + + "github.com/minio/minio-go/v7/pkg/replication" + "github.com/minio/minio-go/v7/pkg/s3utils" +) + +// RemoveBucketReplication removes a replication config on an existing bucket. +func (c Client) RemoveBucketReplication(ctx context.Context, bucketName string) error { + return c.removeBucketReplication(ctx, bucketName) +} + +// SetBucketReplication sets a replication config on an existing bucket. +func (c Client) SetBucketReplication(ctx context.Context, bucketName string, cfg replication.Config) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // If replication is empty then delete it. + if cfg.Empty() { + return c.removeBucketReplication(ctx, bucketName) + } + // Save the updated replication. + return c.putBucketReplication(ctx, bucketName, cfg) +} + +// Saves a new bucket replication. +func (c Client) putBucketReplication(ctx context.Context, bucketName string, cfg replication.Config) error { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("replication", "") + replication, err := xml.Marshal(cfg) + if err != nil { + return err + } + + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: bytes.NewReader(replication), + contentLength: int64(len(replication)), + contentMD5Base64: sumMD5Base64(replication), + } + + // Execute PUT to upload a new bucket replication config. + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + + return nil +} + +// Remove replication from a bucket. +func (c Client) removeBucketReplication(ctx context.Context, bucketName string) error { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("replication", "") + + // Execute DELETE on objectName. + resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return err + } + return nil +} + +// GetBucketReplication fetches bucket replication configuration.If config is not +// found, returns empty config with nil error. +func (c Client) GetBucketReplication(ctx context.Context, bucketName string) (cfg replication.Config, err error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return cfg, err + } + bucketReplicationCfg, err := c.getBucketReplication(ctx, bucketName) + if err != nil { + errResponse := ToErrorResponse(err) + if errResponse.Code == "ReplicationConfigurationNotFoundError" { + return cfg, nil + } + return cfg, err + } + return bucketReplicationCfg, nil +} + +// Request server for current bucket replication config. +func (c Client) getBucketReplication(ctx context.Context, bucketName string) (cfg replication.Config, err error) { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("replication", "") + + // Execute GET on bucket to get replication config. + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + }) + + defer closeResponse(resp) + if err != nil { + return cfg, err + } + + if resp.StatusCode != http.StatusOK { + return cfg, httpRespToErrorResponse(resp, bucketName, "") + } + + if err = xmlDecoder(resp.Body, &cfg); err != nil { + return cfg, err + } + + return cfg, nil +} diff --git a/vendor/github.com/minio/minio-go/v6/api-bucket-tagging.go b/vendor/github.com/minio/minio-go/v7/api-bucket-tagging.go similarity index 69% rename from vendor/github.com/minio/minio-go/v6/api-bucket-tagging.go rename to vendor/github.com/minio/minio-go/v7/api-bucket-tagging.go index ab1828b778..fcb966e636 100644 --- a/vendor/github.com/minio/minio-go/v6/api-bucket-tagging.go +++ b/vendor/github.com/minio/minio-go/v7/api-bucket-tagging.go @@ -26,17 +26,13 @@ import ( "net/http" "net/url" - "github.com/minio/minio-go/v6/pkg/s3utils" - "github.com/minio/minio-go/v6/pkg/tags" + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/tags" ) -// GetBucketTagging gets tagging configuration for a bucket. -func (c Client) GetBucketTagging(bucketName string) (*tags.Tags, error) { - return c.GetBucketTaggingWithContext(context.Background(), bucketName) -} - -// GetBucketTaggingWithContext gets tagging configuration for a bucket with a context to control cancellations and timeouts. -func (c Client) GetBucketTaggingWithContext(ctx context.Context, bucketName string) (*tags.Tags, error) { +// GetBucketTagging fetch tagging configuration for a bucket with a +// context to control cancellations and timeouts. +func (c Client) GetBucketTagging(ctx context.Context, bucketName string) (*tags.Tags, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, err @@ -66,13 +62,9 @@ func (c Client) GetBucketTaggingWithContext(ctx context.Context, bucketName stri return tags.ParseBucketXML(resp.Body) } -// SetBucketTagging sets tagging configuration for a bucket. -func (c Client) SetBucketTagging(bucketName string, tags *tags.Tags) error { - return c.SetBucketTaggingWithContext(context.Background(), bucketName, tags) -} - -// SetBucketTaggingWithContext sets tagging configuration for a bucket with a context to control cancellations and timeouts. -func (c Client) SetBucketTaggingWithContext(ctx context.Context, bucketName string, tags *tags.Tags) error { +// SetBucketTagging sets tagging configuration for a bucket +// with a context to control cancellations and timeouts. +func (c Client) SetBucketTagging(ctx context.Context, bucketName string, tags *tags.Tags) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -113,13 +105,9 @@ func (c Client) SetBucketTaggingWithContext(ctx context.Context, bucketName stri return nil } -// DeleteBucketTagging removes tagging configuration for a bucket. -func (c Client) DeleteBucketTagging(bucketName string) error { - return c.DeleteBucketTaggingWithContext(context.Background(), bucketName) -} - -// DeleteBucketTaggingWithContext removes tagging configuration for a bucket with a context to control cancellations and timeouts. -func (c Client) DeleteBucketTaggingWithContext(ctx context.Context, bucketName string) error { +// RemoveBucketTagging removes tagging configuration for a +// bucket with a context to control cancellations and timeouts. +func (c Client) RemoveBucketTagging(ctx context.Context, bucketName string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err diff --git a/vendor/github.com/minio/minio-go/v7/api-bucket-versioning.go b/vendor/github.com/minio/minio-go/v7/api-bucket-versioning.go new file mode 100644 index 0000000000..0889d43b0e --- /dev/null +++ b/vendor/github.com/minio/minio-go/v7/api-bucket-versioning.go @@ -0,0 +1,120 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/xml" + "net/http" + "net/url" + + "github.com/minio/minio-go/v7/pkg/s3utils" +) + +// SetBucketVersioning sets a bucket versioning configuration +func (c Client) SetBucketVersioning(ctx context.Context, bucketName string, config BucketVersioningConfiguration) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + buf, err := xml.Marshal(config) + if err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("versioning", "") + + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: bytes.NewReader(buf), + contentLength: int64(len(buf)), + contentMD5Base64: sumMD5Base64(buf), + contentSHA256Hex: sum256Hex(buf), + } + + // Execute PUT to set a bucket versioning. + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + } + return nil +} + +// EnableVersioning - enable object versioning in given bucket. +func (c Client) EnableVersioning(ctx context.Context, bucketName string) error { + return c.SetBucketVersioning(ctx, bucketName, BucketVersioningConfiguration{Status: "Enabled"}) +} + +// SuspendVersioning - suspend object versioning in given bucket. +func (c Client) SuspendVersioning(ctx context.Context, bucketName string) error { + return c.SetBucketVersioning(ctx, bucketName, BucketVersioningConfiguration{Status: "Suspended"}) +} + +// BucketVersioningConfiguration is the versioning configuration structure +type BucketVersioningConfiguration struct { + XMLName xml.Name `xml:"VersioningConfiguration"` + Status string `xml:"Status"` + MFADelete string `xml:"MfaDelete,omitempty"` +} + +// GetBucketVersioning gets the versioning configuration on +// an existing bucket with a context to control cancellations and timeouts. +func (c Client) GetBucketVersioning(ctx context.Context, bucketName string) (BucketVersioningConfiguration, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return BucketVersioningConfiguration{}, err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("versioning", "") + + // Execute GET on bucket to get the versioning configuration. + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + }) + + defer closeResponse(resp) + if err != nil { + return BucketVersioningConfiguration{}, err + } + + if resp.StatusCode != http.StatusOK { + return BucketVersioningConfiguration{}, httpRespToErrorResponse(resp, bucketName, "") + } + + versioningConfig := BucketVersioningConfiguration{} + if err = xmlDecoder(resp.Body, &versioningConfig); err != nil { + return versioningConfig, err + } + + return versioningConfig, nil +} diff --git a/vendor/github.com/minio/minio-go/v7/api-compose-object.go b/vendor/github.com/minio/minio-go/v7/api-compose-object.go new file mode 100644 index 0000000000..e107375583 --- /dev/null +++ b/vendor/github.com/minio/minio-go/v7/api-compose-object.go @@ -0,0 +1,552 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017, 2018 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/minio/minio-go/v7/pkg/encrypt" + "github.com/minio/minio-go/v7/pkg/s3utils" +) + +// CopyDestOptions represents options specified by user for CopyObject/ComposeObject APIs +type CopyDestOptions struct { + Bucket string // points to destination bucket + Object string // points to destination object + + // `Encryption` is the key info for server-side-encryption with customer + // provided key. If it is nil, no encryption is performed. + Encryption encrypt.ServerSide + + // `userMeta` is the user-metadata key-value pairs to be set on the + // destination. The keys are automatically prefixed with `x-amz-meta-` + // if needed. If nil is passed, and if only a single source (of any + // size) is provided in the ComposeObject call, then metadata from the + // source is copied to the destination. + // if no user-metadata is provided, it is copied from source + // (when there is only once source object in the compose + // request) + UserMetadata map[string]string + // UserMetadata is only set to destination if ReplaceMetadata is true + // other value is UserMetadata is ignored and we preserve src.UserMetadata + // NOTE: if you set this value to true and now metadata is present + // in UserMetadata your destination object will not have any metadata + // set. + ReplaceMetadata bool + + // `userTags` is the user defined object tags to be set on destination. + // This will be set only if the `replaceTags` field is set to true. + // Otherwise this field is ignored + UserTags map[string]string + ReplaceTags bool + + // Specifies whether you want to apply a Legal Hold to the copied object. + LegalHold LegalHoldStatus + + // Object Retention related fields + Mode RetentionMode + RetainUntilDate time.Time + + Size int64 // Needs to be specified if progress bar is specified. + // Progress of the entire copy operation will be sent here. + Progress io.Reader +} + +// Process custom-metadata to remove a `x-amz-meta-` prefix if +// present and validate that keys are distinct (after this +// prefix removal). +func filterCustomMeta(userMeta map[string]string) map[string]string { + m := make(map[string]string) + for k, v := range userMeta { + if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") { + k = k[len("x-amz-meta-"):] + } + if _, ok := m[k]; ok { + continue + } + m[k] = v + } + return m +} + +// Marshal converts all the CopyDestOptions into their +// equivalent HTTP header representation +func (opts CopyDestOptions) Marshal(header http.Header) { + const replaceDirective = "REPLACE" + if opts.ReplaceTags { + header.Set(amzTaggingHeaderDirective, replaceDirective) + if tags := s3utils.TagEncode(opts.UserTags); tags != "" { + header.Set(amzTaggingHeader, tags) + } + } + + if opts.LegalHold != LegalHoldStatus("") { + header.Set(amzLegalHoldHeader, opts.LegalHold.String()) + } + + if opts.Mode != RetentionMode("") && !opts.RetainUntilDate.IsZero() { + header.Set(amzLockMode, opts.Mode.String()) + header.Set(amzLockRetainUntil, opts.RetainUntilDate.Format(time.RFC3339)) + } + + if opts.Encryption != nil { + opts.Encryption.Marshal(header) + } + + if opts.ReplaceMetadata { + header.Set("x-amz-metadata-directive", replaceDirective) + for k, v := range filterCustomMeta(opts.UserMetadata) { + if isAmzHeader(k) || isStandardHeader(k) || isStorageClassHeader(k) { + header.Set(k, v) + } else { + header.Set("x-amz-meta-"+k, v) + } + } + } +} + +// toDestinationInfo returns a validated copyOptions object. +func (opts CopyDestOptions) validate() (err error) { + // Input validation. + if err = s3utils.CheckValidBucketName(opts.Bucket); err != nil { + return err + } + if err = s3utils.CheckValidObjectName(opts.Object); err != nil { + return err + } + if opts.Progress != nil && opts.Size < 0 { + return errInvalidArgument("For progress bar effective size needs to be specified") + } + return nil +} + +// CopySrcOptions represents a source object to be copied, using +// server-side copying APIs. +type CopySrcOptions struct { + Bucket, Object string + VersionID string + MatchETag string + NoMatchETag string + MatchModifiedSince time.Time + MatchUnmodifiedSince time.Time + MatchRange bool + Start, End int64 + Encryption encrypt.ServerSide +} + +// Marshal converts all the CopySrcOptions into their +// equivalent HTTP header representation +func (opts CopySrcOptions) Marshal(header http.Header) { + // Set the source header + header.Set("x-amz-copy-source", s3utils.EncodePath(opts.Bucket+"/"+opts.Object)) + if opts.VersionID != "" { + header.Set("x-amz-copy-source", s3utils.EncodePath(opts.Bucket+"/"+opts.Object)+"?versionId="+opts.VersionID) + } + + if opts.MatchETag != "" { + header.Set("x-amz-copy-source-if-match", opts.MatchETag) + } + if opts.NoMatchETag != "" { + header.Set("x-amz-copy-source-if-none-match", opts.NoMatchETag) + } + + if !opts.MatchModifiedSince.IsZero() { + header.Set("x-amz-copy-source-if-modified-since", opts.MatchModifiedSince.Format(http.TimeFormat)) + } + if !opts.MatchUnmodifiedSince.IsZero() { + header.Set("x-amz-copy-source-if-unmodified-since", opts.MatchUnmodifiedSince.Format(http.TimeFormat)) + } + + if opts.Encryption != nil { + encrypt.SSECopy(opts.Encryption).Marshal(header) + } +} + +func (opts CopySrcOptions) validate() (err error) { + // Input validation. + if err = s3utils.CheckValidBucketName(opts.Bucket); err != nil { + return err + } + if err = s3utils.CheckValidObjectName(opts.Object); err != nil { + return err + } + if opts.Start > opts.End || opts.Start < 0 { + return errInvalidArgument("start must be non-negative, and start must be at most end.") + } + return nil +} + +// Low level implementation of CopyObject API, supports only upto 5GiB worth of copy. +func (c Client) copyObjectDo(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, + metadata map[string]string) (ObjectInfo, error) { + + // Build headers. + headers := make(http.Header) + + // Set all the metadata headers. + for k, v := range metadata { + headers.Set(k, v) + } + + // Set the source header + headers.Set("x-amz-copy-source", s3utils.EncodePath(srcBucket+"/"+srcObject)) + + // Send upload-part-copy request + resp, err := c.executeMethod(ctx, http.MethodPut, requestMetadata{ + bucketName: destBucket, + objectName: destObject, + customHeader: headers, + }) + defer closeResponse(resp) + if err != nil { + return ObjectInfo{}, err + } + + // Check if we got an error response. + if resp.StatusCode != http.StatusOK { + return ObjectInfo{}, httpRespToErrorResponse(resp, srcBucket, srcObject) + } + + cpObjRes := copyObjectResult{} + err = xmlDecoder(resp.Body, &cpObjRes) + if err != nil { + return ObjectInfo{}, err + } + + objInfo := ObjectInfo{ + Key: destObject, + ETag: strings.Trim(cpObjRes.ETag, "\""), + LastModified: cpObjRes.LastModified, + } + return objInfo, nil +} + +func (c Client) copyObjectPartDo(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, + partID int, startOffset int64, length int64, metadata map[string]string) (p CompletePart, err error) { + + headers := make(http.Header) + + // Set source + headers.Set("x-amz-copy-source", s3utils.EncodePath(srcBucket+"/"+srcObject)) + + if startOffset < 0 { + return p, errInvalidArgument("startOffset must be non-negative") + } + + if length >= 0 { + headers.Set("x-amz-copy-source-range", fmt.Sprintf("bytes=%d-%d", startOffset, startOffset+length-1)) + } + + for k, v := range metadata { + headers.Set(k, v) + } + + queryValues := make(url.Values) + queryValues.Set("partNumber", strconv.Itoa(partID)) + queryValues.Set("uploadId", uploadID) + + resp, err := c.executeMethod(ctx, http.MethodPut, requestMetadata{ + bucketName: destBucket, + objectName: destObject, + customHeader: headers, + queryValues: queryValues, + }) + defer closeResponse(resp) + if err != nil { + return + } + + // Check if we got an error response. + if resp.StatusCode != http.StatusOK { + return p, httpRespToErrorResponse(resp, destBucket, destObject) + } + + // Decode copy-part response on success. + cpObjRes := copyObjectResult{} + err = xmlDecoder(resp.Body, &cpObjRes) + if err != nil { + return p, err + } + p.PartNumber, p.ETag = partID, cpObjRes.ETag + return p, nil +} + +// uploadPartCopy - helper function to create a part in a multipart +// upload via an upload-part-copy request +// https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPartCopy.html +func (c Client) uploadPartCopy(ctx context.Context, bucket, object, uploadID string, partNumber int, + headers http.Header) (p CompletePart, err error) { + + // Build query parameters + urlValues := make(url.Values) + urlValues.Set("partNumber", strconv.Itoa(partNumber)) + urlValues.Set("uploadId", uploadID) + + // Send upload-part-copy request + resp, err := c.executeMethod(ctx, http.MethodPut, requestMetadata{ + bucketName: bucket, + objectName: object, + customHeader: headers, + queryValues: urlValues, + }) + defer closeResponse(resp) + if err != nil { + return p, err + } + + // Check if we got an error response. + if resp.StatusCode != http.StatusOK { + return p, httpRespToErrorResponse(resp, bucket, object) + } + + // Decode copy-part response on success. + cpObjRes := copyObjectResult{} + err = xmlDecoder(resp.Body, &cpObjRes) + if err != nil { + return p, err + } + p.PartNumber, p.ETag = partNumber, cpObjRes.ETag + return p, nil +} + +// ComposeObject - creates an object using server-side copying +// of existing objects. It takes a list of source objects (with optional offsets) +// and concatenates them into a new object using only server-side copying +// operations. Optionally takes progress reader hook for applications to +// look at current progress. +func (c Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs ...CopySrcOptions) (UploadInfo, error) { + if len(srcs) < 1 || len(srcs) > maxPartsCount { + return UploadInfo{}, errInvalidArgument("There must be as least one and up to 10000 source objects.") + } + + for _, src := range srcs { + if err := src.validate(); err != nil { + return UploadInfo{}, err + } + } + + if err := dst.validate(); err != nil { + return UploadInfo{}, err + } + + srcObjectInfos := make([]ObjectInfo, len(srcs)) + srcObjectSizes := make([]int64, len(srcs)) + var totalSize, totalParts int64 + var err error + for i, src := range srcs { + opts := StatObjectOptions{ServerSideEncryption: encrypt.SSE(src.Encryption), VersionID: src.VersionID} + srcObjectInfos[i], err = c.statObject(context.Background(), src.Bucket, src.Object, opts) + if err != nil { + return UploadInfo{}, err + } + + srcCopySize := srcObjectInfos[i].Size + // Check if a segment is specified, and if so, is the + // segment within object bounds? + if src.MatchRange { + // Since range is specified, + // 0 <= src.start <= src.end + // so only invalid case to check is: + if src.End >= srcCopySize || src.Start < 0 { + return UploadInfo{}, errInvalidArgument( + fmt.Sprintf("CopySrcOptions %d has invalid segment-to-copy [%d, %d] (size is %d)", + i, src.Start, src.End, srcCopySize)) + } + srcCopySize = src.End - src.Start + 1 + } + + // Only the last source may be less than `absMinPartSize` + if srcCopySize < absMinPartSize && i < len(srcs)-1 { + return UploadInfo{}, errInvalidArgument( + fmt.Sprintf("CopySrcOptions %d is too small (%d) and it is not the last part", i, srcCopySize)) + } + + // Is data to copy too large? + totalSize += srcCopySize + if totalSize > maxMultipartPutObjectSize { + return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Cannot compose an object of size %d (> 5TiB)", totalSize)) + } + + // record source size + srcObjectSizes[i] = srcCopySize + + // calculate parts needed for current source + totalParts += partsRequired(srcCopySize) + // Do we need more parts than we are allowed? + if totalParts > maxPartsCount { + return UploadInfo{}, errInvalidArgument(fmt.Sprintf( + "Your proposed compose object requires more than %d parts", maxPartsCount)) + } + } + + // Single source object case (i.e. when only one source is + // involved, it is being copied wholly and at most 5GiB in + // size, emptyfiles are also supported). + if (totalParts == 1 && srcs[0].Start == -1 && totalSize <= maxPartSize) || (totalSize == 0) { + return c.CopyObject(ctx, dst, srcs[0]) + } + + // Now, handle multipart-copy cases. + + // 1. Ensure that the object has not been changed while + // we are copying data. + for i, src := range srcs { + src.MatchETag = srcObjectInfos[i].ETag + } + + // 2. Initiate a new multipart upload. + + // Set user-metadata on the destination object. If no + // user-metadata is specified, and there is only one source, + // (only) then metadata from source is copied. + var userMeta map[string]string + if dst.ReplaceMetadata { + userMeta = dst.UserMetadata + } else { + userMeta = srcObjectInfos[0].UserMetadata + } + + var userTags map[string]string + if dst.ReplaceTags { + userTags = dst.UserTags + } else { + userTags = srcObjectInfos[0].UserTags + } + + uploadID, err := c.newUploadID(ctx, dst.Bucket, dst.Object, PutObjectOptions{ + ServerSideEncryption: dst.Encryption, + UserMetadata: userMeta, + UserTags: userTags, + Mode: dst.Mode, + RetainUntilDate: dst.RetainUntilDate, + LegalHold: dst.LegalHold, + }) + if err != nil { + return UploadInfo{}, err + } + + // 3. Perform copy part uploads + objParts := []CompletePart{} + partIndex := 1 + for i, src := range srcs { + var h = make(http.Header) + src.Marshal(h) + if dst.Encryption != nil && dst.Encryption.Type() == encrypt.SSEC { + dst.Encryption.Marshal(h) + } + + // calculate start/end indices of parts after + // splitting. + startIdx, endIdx := calculateEvenSplits(srcObjectSizes[i], src) + for j, start := range startIdx { + end := endIdx[j] + + // Add (or reset) source range header for + // upload part copy request. + h.Set("x-amz-copy-source-range", + fmt.Sprintf("bytes=%d-%d", start, end)) + + // make upload-part-copy request + complPart, err := c.uploadPartCopy(ctx, dst.Bucket, + dst.Object, uploadID, partIndex, h) + if err != nil { + return UploadInfo{}, err + } + if dst.Progress != nil { + io.CopyN(ioutil.Discard, dst.Progress, end-start+1) + } + objParts = append(objParts, complPart) + partIndex++ + } + } + + // 4. Make final complete-multipart request. + uploadInfo, err := c.completeMultipartUpload(ctx, dst.Bucket, dst.Object, uploadID, + completeMultipartUpload{Parts: objParts}) + if err != nil { + return UploadInfo{}, err + } + + uploadInfo.Size = totalSize + return uploadInfo, nil +} + +// partsRequired is maximum parts possible with +// max part size of ceiling(maxMultipartPutObjectSize / (maxPartsCount - 1)) +func partsRequired(size int64) int64 { + maxPartSize := maxMultipartPutObjectSize / (maxPartsCount - 1) + r := size / int64(maxPartSize) + if size%int64(maxPartSize) > 0 { + r++ + } + return r +} + +// calculateEvenSplits - computes splits for a source and returns +// start and end index slices. Splits happen evenly to be sure that no +// part is less than 5MiB, as that could fail the multipart request if +// it is not the last part. +func calculateEvenSplits(size int64, src CopySrcOptions) (startIndex, endIndex []int64) { + if size == 0 { + return + } + + reqParts := partsRequired(size) + startIndex = make([]int64, reqParts) + endIndex = make([]int64, reqParts) + // Compute number of required parts `k`, as: + // + // k = ceiling(size / copyPartSize) + // + // Now, distribute the `size` bytes in the source into + // k parts as evenly as possible: + // + // r parts sized (q+1) bytes, and + // (k - r) parts sized q bytes, where + // + // size = q * k + r (by simple division of size by k, + // so that 0 <= r < k) + // + start := src.Start + if start == -1 { + start = 0 + } + quot, rem := size/reqParts, size%reqParts + nextStart := start + for j := int64(0); j < reqParts; j++ { + curPartSize := quot + if j < rem { + curPartSize++ + } + + cStart := nextStart + cEnd := cStart + curPartSize - 1 + nextStart = cEnd + 1 + + startIndex[j], endIndex[j] = cStart, cEnd + } + return +} diff --git a/vendor/github.com/minio/minio-go/v6/api-datatypes.go b/vendor/github.com/minio/minio-go/v7/api-datatypes.go similarity index 77% rename from vendor/github.com/minio/minio-go/v6/api-datatypes.go rename to vendor/github.com/minio/minio-go/v7/api-datatypes.go index 9db73736a3..970e1fa5e3 100644 --- a/vendor/github.com/minio/minio-go/v6/api-datatypes.go +++ b/vendor/github.com/minio/minio-go/v7/api-datatypes.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -62,6 +62,29 @@ func (m *StringMap) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { return nil } +// Owner name. +type Owner struct { + DisplayName string `json:"name"` + ID string `json:"id"` +} + +// UploadInfo contains information about the +// newly uploaded or copied object. +type UploadInfo struct { + Bucket string + Key string + ETag string + Size int64 + LastModified time.Time + Location string + VersionID string + + // Lifecycle expiry-date and ruleID associated with the expiry + // not to be confused with `Expires` HTTP header. + Expiration time.Time + ExpirationRuleID string +} + // ObjectInfo container for object metadata. type ObjectInfo struct { // An ETag is optionally set to md5sum of an object. In case of multipart objects, @@ -85,11 +108,11 @@ type ObjectInfo struct { // x-amz-tagging values in their k/v values. UserTags map[string]string `json:"userTags"` + // x-amz-tagging-count value + UserTagCount int + // Owner name. - Owner struct { - DisplayName string `json:"name"` - ID string `json:"id"` - } `json:"owner"` + Owner Owner // ACL grant. Grant []struct { @@ -104,6 +127,23 @@ type ObjectInfo struct { // The class of storage used to store the object. StorageClass string `json:"storageClass"` + // Versioning related information + IsLatest bool + IsDeleteMarker bool + VersionID string `xml:"VersionId"` + + // x-amz-replication-status value is either in one of the following states + // - COMPLETE + // - PENDING + // - FAILED + // - REPLICA (on the destination) + ReplicationStatus string `xml:"ReplicationStatus"` + + // Lifecycle expiry-date and ruleID associated with the expiry + // not to be confused with `Expires` HTTP header. + Expiration time.Time + ExpirationRuleID string + // Error Err error `json:"-"` } diff --git a/vendor/github.com/minio/minio-go/v6/api-error-response.go b/vendor/github.com/minio/minio-go/v7/api-error-response.go similarity index 82% rename from vendor/github.com/minio/minio-go/v6/api-error-response.go rename to vendor/github.com/minio/minio-go/v7/api-error-response.go index 4fda64856e..f439a8870d 100644 --- a/vendor/github.com/minio/minio-go/v6/api-error-response.go +++ b/vendor/github.com/minio/minio-go/v7/api-error-response.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -63,7 +63,7 @@ type ErrorResponse struct { // // For example: // -// import s3 "github.com/minio/minio-go/v6" +// import s3 "github.com/minio/minio-go/v7" // ... // ... // reader, stat, err := s3.GetObject(...) @@ -103,7 +103,7 @@ const ( func httpRespToErrorResponse(resp *http.Response, bucketName, objectName string) error { if resp == nil { msg := "Response is empty. " + reportIssue - return ErrInvalidArgument(msg) + return errInvalidArgument(msg) } errResp := ErrorResponse{ @@ -183,8 +183,8 @@ func httpRespToErrorResponse(resp *http.Response, bucketName, objectName string) return errResp } -// ErrTransferAccelerationBucket - bucket name is invalid to be used with transfer acceleration. -func ErrTransferAccelerationBucket(bucketName string) error { +// errTransferAccelerationBucket - bucket name is invalid to be used with transfer acceleration. +func errTransferAccelerationBucket(bucketName string) error { return ErrorResponse{ StatusCode: http.StatusBadRequest, Code: "InvalidArgument", @@ -193,8 +193,8 @@ func ErrTransferAccelerationBucket(bucketName string) error { } } -// ErrEntityTooLarge - Input size is larger than supported maximum. -func ErrEntityTooLarge(totalSize, maxObjectSize int64, bucketName, objectName string) error { +// errEntityTooLarge - Input size is larger than supported maximum. +func errEntityTooLarge(totalSize, maxObjectSize int64, bucketName, objectName string) error { msg := fmt.Sprintf("Your proposed upload size ‘%d’ exceeds the maximum allowed object size ‘%d’ for single PUT operation.", totalSize, maxObjectSize) return ErrorResponse{ StatusCode: http.StatusBadRequest, @@ -205,8 +205,8 @@ func ErrEntityTooLarge(totalSize, maxObjectSize int64, bucketName, objectName st } } -// ErrEntityTooSmall - Input size is smaller than supported minimum. -func ErrEntityTooSmall(totalSize int64, bucketName, objectName string) error { +// errEntityTooSmall - Input size is smaller than supported minimum. +func errEntityTooSmall(totalSize int64, bucketName, objectName string) error { msg := fmt.Sprintf("Your proposed upload size ‘%d’ is below the minimum allowed object size ‘0B’ for single PUT operation.", totalSize) return ErrorResponse{ StatusCode: http.StatusBadRequest, @@ -217,8 +217,8 @@ func ErrEntityTooSmall(totalSize int64, bucketName, objectName string) error { } } -// ErrUnexpectedEOF - Unexpected end of file reached. -func ErrUnexpectedEOF(totalRead, totalSize int64, bucketName, objectName string) error { +// errUnexpectedEOF - Unexpected end of file reached. +func errUnexpectedEOF(totalRead, totalSize int64, bucketName, objectName string) error { msg := fmt.Sprintf("Data read ‘%d’ is not equal to the size ‘%d’ of the input Reader.", totalRead, totalSize) return ErrorResponse{ StatusCode: http.StatusBadRequest, @@ -229,8 +229,8 @@ func ErrUnexpectedEOF(totalRead, totalSize int64, bucketName, objectName string) } } -// ErrInvalidBucketName - Invalid bucket name response. -func ErrInvalidBucketName(message string) error { +// errInvalidBucketName - Invalid bucket name response. +func errInvalidBucketName(message string) error { return ErrorResponse{ StatusCode: http.StatusBadRequest, Code: "InvalidBucketName", @@ -239,8 +239,8 @@ func ErrInvalidBucketName(message string) error { } } -// ErrInvalidObjectName - Invalid object name response. -func ErrInvalidObjectName(message string) error { +// errInvalidObjectName - Invalid object name response. +func errInvalidObjectName(message string) error { return ErrorResponse{ StatusCode: http.StatusNotFound, Code: "NoSuchKey", @@ -249,12 +249,8 @@ func ErrInvalidObjectName(message string) error { } } -// ErrInvalidObjectPrefix - Invalid object prefix response is -// similar to object name response. -var ErrInvalidObjectPrefix = ErrInvalidObjectName - -// ErrInvalidArgument - Invalid argument response. -func ErrInvalidArgument(message string) error { +// errInvalidArgument - Invalid argument response. +func errInvalidArgument(message string) error { return ErrorResponse{ StatusCode: http.StatusBadRequest, Code: "InvalidArgument", @@ -263,20 +259,9 @@ func ErrInvalidArgument(message string) error { } } -// ErrNoSuchBucketPolicy - No Such Bucket Policy response -// The specified bucket does not have a bucket policy. -func ErrNoSuchBucketPolicy(message string) error { - return ErrorResponse{ - StatusCode: http.StatusNotFound, - Code: "NoSuchBucketPolicy", - Message: message, - RequestID: "minio", - } -} - -// ErrAPINotSupported - API not supported response +// errAPINotSupported - API not supported response // The specified API call is not supported -func ErrAPINotSupported(message string) error { +func errAPINotSupported(message string) error { return ErrorResponse{ StatusCode: http.StatusNotImplemented, Code: "APINotSupported", diff --git a/vendor/github.com/minio/minio-go/v6/api-get-object-acl-context.go b/vendor/github.com/minio/minio-go/v7/api-get-object-acl.go similarity index 94% rename from vendor/github.com/minio/minio-go/v6/api-get-object-acl-context.go rename to vendor/github.com/minio/minio-go/v7/api-get-object-acl.go index f49984f680..afa53079d3 100644 --- a/vendor/github.com/minio/minio-go/v6/api-get-object-acl-context.go +++ b/vendor/github.com/minio/minio-go/v7/api-get-object-acl.go @@ -40,9 +40,9 @@ type accessControlPolicy struct { } `xml:"AccessControlList"` } -//GetObjectACLWithContext get object ACLs -func (c Client) GetObjectACLWithContext(ctx context.Context, bucketName, objectName string) (*ObjectInfo, error) { - resp, err := c.executeMethod(ctx, "GET", requestMetadata{ +// GetObjectACL get object ACLs +func (c Client) GetObjectACL(ctx context.Context, bucketName, objectName string) (*ObjectInfo, error) { + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: url.Values{ diff --git a/vendor/github.com/minio/minio-go/v6/api-get-object-file.go b/vendor/github.com/minio/minio-go/v7/api-get-object-file.go similarity index 78% rename from vendor/github.com/minio/minio-go/v6/api-get-object-file.go rename to vendor/github.com/minio/minio-go/v7/api-get-object-file.go index c2d277514c..bccff45787 100644 --- a/vendor/github.com/minio/minio-go/v6/api-get-object-file.go +++ b/vendor/github.com/minio/minio-go/v7/api-get-object-file.go @@ -23,22 +23,12 @@ import ( "os" "path/filepath" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) -// FGetObjectWithContext - download contents of an object to a local file. -// The options can be used to specify the GET request further. -func (c Client) FGetObjectWithContext(ctx context.Context, bucketName, objectName, filePath string, opts GetObjectOptions) error { - return c.fGetObjectWithContext(ctx, bucketName, objectName, filePath, opts) -} - // FGetObject - download contents of an object to a local file. -func (c Client) FGetObject(bucketName, objectName, filePath string, opts GetObjectOptions) error { - return c.fGetObjectWithContext(context.Background(), bucketName, objectName, filePath, opts) -} - -// fGetObjectWithContext - fgetObject wrapper function with context -func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectName, filePath string, opts GetObjectOptions) error { +// The options can be used to specify the GET request further. +func (c Client) FGetObject(ctx context.Context, bucketName, objectName, filePath string, opts GetObjectOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -52,7 +42,7 @@ func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectNam if err == nil { // If the destination exists and is a directory. if st.IsDir() { - return ErrInvalidArgument("fileName is a directory.") + return errInvalidArgument("fileName is a directory.") } } @@ -73,7 +63,7 @@ func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectNam } // Gather md5sum. - objectStat, err := c.StatObject(bucketName, objectName, StatObjectOptions{opts}) + objectStat, err := c.StatObject(ctx, bucketName, objectName, StatObjectOptions(opts)) if err != nil { return err } diff --git a/vendor/github.com/minio/minio-go/v6/api-get-object.go b/vendor/github.com/minio/minio-go/v7/api-get-object.go similarity index 93% rename from vendor/github.com/minio/minio-go/v6/api-get-object.go rename to vendor/github.com/minio/minio-go/v7/api-get-object.go index 14370b0cdb..2df1112a91 100644 --- a/vendor/github.com/minio/minio-go/v6/api-get-object.go +++ b/vendor/github.com/minio/minio-go/v7/api-get-object.go @@ -23,18 +23,14 @@ import ( "fmt" "io" "net/http" + "net/url" "sync" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) -// GetObject - returns an seekable, readable object. -func (c Client) GetObject(bucketName, objectName string, opts GetObjectOptions) (*Object, error) { - return c.getObjectWithContext(context.Background(), bucketName, objectName, opts) -} - // GetObject wrapper function that accepts a request context -func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (*Object, error) { +func (c Client) GetObject(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (*Object, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, err @@ -106,7 +102,7 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName etag = objectInfo.ETag // Read at least firstReq.Buffer bytes, if not we have // reached our EOF. - size, err := io.ReadFull(httpReader, req.Buffer) + size, err := readFull(httpReader, req.Buffer) if size > 0 && err == io.ErrUnexpectedEOF { // If an EOF happens after reading some but not // all the bytes ReadFull returns ErrUnexpectedEOF @@ -125,7 +121,7 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName // Remove range header if already set, for stat Operations to get original file size. delete(opts.headers, "Range") - objectInfo, err = c.statObject(ctx, bucketName, objectName, StatObjectOptions{opts}) + objectInfo, err = c.statObject(ctx, bucketName, objectName, StatObjectOptions(opts)) if err != nil { resCh <- getResponse{ Error: err, @@ -148,7 +144,7 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName if etag != "" && !snowball { opts.SetMatchETag(etag) } - objectInfo, err := c.statObject(ctx, bucketName, objectName, StatObjectOptions{opts}) + objectInfo, err := c.statObject(ctx, bucketName, objectName, StatObjectOptions(opts)) if err != nil { resCh <- getResponse{ Error: err, @@ -196,8 +192,8 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName // Read at least req.Buffer bytes, if not we have // reached our EOF. - size, err := io.ReadFull(httpReader, req.Buffer) - if err == io.ErrUnexpectedEOF { + size, err := readFull(httpReader, req.Buffer) + if size > 0 && err == io.ErrUnexpectedEOF { // If an EOF happens after reading some but not // all the bytes ReadFull returns ErrUnexpectedEOF err = io.EOF @@ -322,7 +318,7 @@ func (o *Object) setOffset(bytesRead int64) error { // io.EOF upon end of file. func (o *Object) Read(b []byte) (n int, err error) { if o == nil { - return 0, ErrInvalidArgument("Object is nil") + return 0, errInvalidArgument("Object is nil") } // Locking. @@ -376,7 +372,7 @@ func (o *Object) Read(b []byte) (n int, err error) { // Stat returns the ObjectInfo structure describing Object. func (o *Object) Stat() (ObjectInfo, error) { if o == nil { - return ObjectInfo{}, ErrInvalidArgument("Object is nil") + return ObjectInfo{}, errInvalidArgument("Object is nil") } // Locking. o.mutex.Lock() @@ -408,7 +404,7 @@ func (o *Object) Stat() (ObjectInfo, error) { // file, that error is io.EOF. func (o *Object) ReadAt(b []byte, offset int64) (n int, err error) { if o == nil { - return 0, ErrInvalidArgument("Object is nil") + return 0, errInvalidArgument("Object is nil") } // Locking. @@ -485,7 +481,7 @@ func (o *Object) ReadAt(b []byte, offset int64) (n int, err error) { // underlying object is not closed. func (o *Object) Seek(offset int64, whence int) (n int64, err error) { if o == nil { - return 0, ErrInvalidArgument("Object is nil") + return 0, errInvalidArgument("Object is nil") } // Locking. @@ -499,7 +495,7 @@ func (o *Object) Seek(offset int64, whence int) (n int64, err error) { // Negative offset is valid for whence of '2'. if offset < 0 && whence != 2 { - return 0, ErrInvalidArgument(fmt.Sprintf("Negative position not allowed for %d", whence)) + return 0, errInvalidArgument(fmt.Sprintf("Negative position not allowed for %d", whence)) } // This is the first request. So before anything else @@ -523,7 +519,7 @@ func (o *Object) Seek(offset int64, whence int) (n int64, err error) { // Switch through whence. switch whence { default: - return 0, ErrInvalidArgument(fmt.Sprintf("Invalid whence %d", whence)) + return 0, errInvalidArgument(fmt.Sprintf("Invalid whence %d", whence)) case 0: if o.objectInfo.Size > -1 && offset > o.objectInfo.Size { return 0, io.EOF @@ -537,7 +533,7 @@ func (o *Object) Seek(offset int64, whence int) (n int64, err error) { case 2: // If we don't know the object size return an error for io.SeekEnd if o.objectInfo.Size < 0 { - return 0, ErrInvalidArgument("Whence END is not supported when the object size is unknown") + return 0, errInvalidArgument("Whence END is not supported when the object size is unknown") } // Seeking to positive offset is valid for whence '2', but // since we are backing a Reader we have reached 'EOF' if @@ -547,7 +543,7 @@ func (o *Object) Seek(offset int64, whence int) (n int64, err error) { } // Seeking to negative position not allowed for whence. if o.objectInfo.Size+offset < 0 { - return 0, ErrInvalidArgument(fmt.Sprintf("Seeking at negative offset not allowed for %d", whence)) + return 0, errInvalidArgument(fmt.Sprintf("Seeking at negative offset not allowed for %d", whence)) } o.currOffset = o.objectInfo.Size + offset } @@ -568,7 +564,7 @@ func (o *Object) Seek(offset int64, whence int) (n int64, err error) { // for subsequent Close() calls. func (o *Object) Close() (err error) { if o == nil { - return ErrInvalidArgument("Object is nil") + return errInvalidArgument("Object is nil") } // Locking. o.mutex.Lock() @@ -617,10 +613,16 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, op return nil, ObjectInfo{}, nil, err } + urlValues := make(url.Values) + if opts.VersionID != "" { + urlValues.Set("versionId", opts.VersionID) + } + // Execute GET on objectName. - resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, objectName: objectName, + queryValues: urlValues, customHeader: opts.Header(), contentSHA256Hex: emptySHA256Hex, }) @@ -636,7 +638,7 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, op objectStat, err := ToObjectInfo(bucketName, objectName, resp.Header) if err != nil { closeResponse(resp) - return nil, objectStat, resp.Header, nil + return nil, ObjectInfo{}, nil, err } // do not close body here, caller will close diff --git a/vendor/github.com/minio/minio-go/v6/api-get-options.go b/vendor/github.com/minio/minio-go/v7/api-get-options.go similarity index 88% rename from vendor/github.com/minio/minio-go/v6/api-get-options.go rename to vendor/github.com/minio/minio-go/v7/api-get-options.go index 538fd1a052..5f3ed36566 100644 --- a/vendor/github.com/minio/minio-go/v6/api-get-options.go +++ b/vendor/github.com/minio/minio-go/v7/api-get-options.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ import ( "net/http" "time" - "github.com/minio/minio-go/v6/pkg/encrypt" + "github.com/minio/minio-go/v7/pkg/encrypt" ) // GetObjectOptions are used to specify additional headers or options @@ -30,13 +30,12 @@ import ( type GetObjectOptions struct { headers map[string]string ServerSideEncryption encrypt.ServerSide + VersionID string } // StatObjectOptions are used to specify additional headers or options // during GET info/stat requests. -type StatObjectOptions struct { - GetObjectOptions -} +type StatObjectOptions = GetObjectOptions // Header returns the http.Header representation of the GET options. func (o GetObjectOptions) Header() http.Header { @@ -63,7 +62,7 @@ func (o *GetObjectOptions) Set(key, value string) { // SetMatchETag - set match etag. func (o *GetObjectOptions) SetMatchETag(etag string) error { if etag == "" { - return ErrInvalidArgument("ETag cannot be empty.") + return errInvalidArgument("ETag cannot be empty.") } o.Set("If-Match", "\""+etag+"\"") return nil @@ -72,7 +71,7 @@ func (o *GetObjectOptions) SetMatchETag(etag string) error { // SetMatchETagExcept - set match etag except. func (o *GetObjectOptions) SetMatchETagExcept(etag string) error { if etag == "" { - return ErrInvalidArgument("ETag cannot be empty.") + return errInvalidArgument("ETag cannot be empty.") } o.Set("If-None-Match", "\""+etag+"\"") return nil @@ -81,7 +80,7 @@ func (o *GetObjectOptions) SetMatchETagExcept(etag string) error { // SetUnmodified - set unmodified time since. func (o *GetObjectOptions) SetUnmodified(modTime time.Time) error { if modTime.IsZero() { - return ErrInvalidArgument("Modified since cannot be empty.") + return errInvalidArgument("Modified since cannot be empty.") } o.Set("If-Unmodified-Since", modTime.Format(http.TimeFormat)) return nil @@ -90,7 +89,7 @@ func (o *GetObjectOptions) SetUnmodified(modTime time.Time) error { // SetModified - set modified time since. func (o *GetObjectOptions) SetModified(modTime time.Time) error { if modTime.IsZero() { - return ErrInvalidArgument("Modified since cannot be empty.") + return errInvalidArgument("Modified since cannot be empty.") } o.Set("If-Modified-Since", modTime.Format(http.TimeFormat)) return nil @@ -119,7 +118,7 @@ func (o *GetObjectOptions) SetRange(start, end int64) error { // bytes=-3-0 // bytes=-3--2 // are invalid. - return ErrInvalidArgument( + return errInvalidArgument( fmt.Sprintf( "Invalid range specified: start=%d end=%d", start, end)) diff --git a/vendor/github.com/minio/minio-go/v6/api-list.go b/vendor/github.com/minio/minio-go/v7/api-list.go similarity index 65% rename from vendor/github.com/minio/minio-go/v6/api-list.go rename to vendor/github.com/minio/minio-go/v7/api-list.go index 1bcb64f3dd..7996c11e9d 100644 --- a/vendor/github.com/minio/minio-go/v6/api-list.go +++ b/vendor/github.com/minio/minio-go/v7/api-list.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2019 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ import ( "net/http" "net/url" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // ListBuckets list all buckets owned by this authenticated user. @@ -32,28 +32,13 @@ import ( // allowed for listing buckets. // // api := client.New(....) -// for message := range api.ListBuckets() { +// for message := range api.ListBuckets(context.Background()) { // fmt.Println(message) // } // -func (c Client) ListBuckets() ([]BucketInfo, error) { - return c.ListBucketsWithContext(context.Background()) -} - -// ListBucketsWithContext list all buckets owned by this authenticated user, -// accepts a context for facilitate cancellation. -// -// This call requires explicit authentication, no anonymous requests are -// allowed for listing buckets. -// -// api := client.New(....) -// for message := range api.ListBucketsWithContext(context.Background()) { -// fmt.Println(message) -// } -// -func (c Client) ListBucketsWithContext(ctx context.Context) ([]BucketInfo, error) { +func (c Client) ListBuckets(ctx context.Context) ([]BucketInfo, error) { // Execute GET on service. - resp, err := c.executeMethod(ctx, "GET", requestMetadata{contentSHA256Hex: emptySHA256Hex}) + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{contentSHA256Hex: emptySHA256Hex}) defer closeResponse(resp) if err != nil { return nil, err @@ -73,44 +58,7 @@ func (c Client) ListBucketsWithContext(ctx context.Context) ([]BucketInfo, error /// Bucket Read Operations. -// ListObjectsV2WithMetadata lists all objects matching the objectPrefix -// from the specified bucket. If recursion is enabled it would list -// all subdirectories and all its contents. This call adds -// UserMetadata information as well for each object. -// -// This is a MinIO extension, this will not work against other S3 -// compatible object storage vendors. -// -// Your input parameters are just bucketName, objectPrefix, recursive -// and a done channel for pro-actively closing the internal go -// routine. If you enable recursive as 'true' this function will -// return back all the objects in a given bucket name and object -// prefix. -// -// api := client.New(....) -// // Create a done channel. -// doneCh := make(chan struct{}) -// defer close(doneCh) -// // Recursively list all objects in 'mytestbucket' -// recursive := true -// // Add metadata -// metadata := true -// for message := range api.ListObjectsV2WithMetadata("mytestbucket", "starthere", recursive, doneCh) { -// fmt.Println(message) -// } -// -func (c Client) ListObjectsV2WithMetadata(bucketName, objectPrefix string, recursive bool, - doneCh <-chan struct{}) <-chan ObjectInfo { - // Check whether this is snowball region, if yes ListObjectsV2 doesn't work, fallback to listObjectsV1. - if location, ok := c.bucketLocCache.Get(bucketName); ok { - if location == "snowball" { - return c.ListObjects(bucketName, objectPrefix, recursive, doneCh) - } - } - return c.listObjectsV2(bucketName, objectPrefix, recursive, true, doneCh) -} - -func (c Client) listObjectsV2(bucketName, objectPrefix string, recursive, metadata bool, doneCh <-chan struct{}) <-chan ObjectInfo { +func (c Client) listObjectsV2(ctx context.Context, bucketName, objectPrefix string, recursive, metadata bool, maxKeys int) <-chan ObjectInfo { // Allocate new list objects channel. objectStatCh := make(chan ObjectInfo, 1) // Default listing is delimited at "/" @@ -148,8 +96,8 @@ func (c Client) listObjectsV2(bucketName, objectPrefix string, recursive, metada var continuationToken string for { // Get list of objects a maximum of 1000 per request. - result, err := c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, - fetchOwner, metadata, delimiter, 0, "") + result, err := c.listObjectsV2Query(ctx, bucketName, objectPrefix, continuationToken, + fetchOwner, metadata, delimiter, maxKeys) if err != nil { objectStatCh <- ObjectInfo{ Err: err, @@ -164,7 +112,7 @@ func (c Client) listObjectsV2(bucketName, objectPrefix string, recursive, metada // Send object content. case objectStatCh <- object: // If receives done from the caller, return here. - case <-doneCh: + case <-ctx.Done(): return } } @@ -176,7 +124,7 @@ func (c Client) listObjectsV2(bucketName, objectPrefix string, recursive, metada // Send object prefixes. case objectStatCh <- ObjectInfo{Key: obj.Prefix}: // If receives done from the caller, return here. - case <-doneCh: + case <-ctx.Done(): return } } @@ -195,36 +143,6 @@ func (c Client) listObjectsV2(bucketName, objectPrefix string, recursive, metada return objectStatCh } -// ListObjectsV2 lists all objects matching the objectPrefix from -// the specified bucket. If recursion is enabled it would list -// all subdirectories and all its contents. -// -// Your input parameters are just bucketName, objectPrefix, recursive -// and a done channel for pro-actively closing the internal go -// routine. If you enable recursive as 'true' this function will -// return back all the objects in a given bucket name and object -// prefix. -// -// api := client.New(....) -// // Create a done channel. -// doneCh := make(chan struct{}) -// defer close(doneCh) -// // Recursively list all objects in 'mytestbucket' -// recursive := true -// for message := range api.ListObjectsV2("mytestbucket", "starthere", recursive, doneCh) { -// fmt.Println(message) -// } -// -func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { - // Check whether this is snowball region, if yes ListObjectsV2 doesn't work, fallback to listObjectsV1. - if location, ok := c.bucketLocCache.Get(bucketName); ok { - if location == "snowball" { - return c.ListObjects(bucketName, objectPrefix, recursive, doneCh) - } - } - return c.listObjectsV2(bucketName, objectPrefix, recursive, false, doneCh) -} - // listObjectsV2Query - (List Objects V2) - List some or all (up to 1000) of the objects in a bucket. // // You can use the request parameters as selection criteria to return a subset of the objects in a bucket. @@ -234,9 +152,8 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d // ?delimiter - A delimiter is a character you use to group keys. // ?prefix - Limits the response to keys that begin with the specified prefix. // ?max-keys - Sets the maximum number of keys returned in the response body. -// ?start-after - Specifies the key to start after when listing objects in a bucket. // ?metadata - Specifies if we want metadata for the objects as part of list operation. -func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken string, fetchOwner, metadata bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) { +func (c Client) listObjectsV2Query(ctx context.Context, bucketName, objectPrefix, continuationToken string, fetchOwner, metadata bool, delimiter string, maxkeys int) (ListBucketV2Result, error) { // Validate bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ListBucketV2Result{}, err @@ -280,13 +197,8 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) } - // Set start-after - if startAfter != "" { - urlValues.Set("start-after", startAfter) - } - // Execute GET on bucket to list objects. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, @@ -334,29 +246,7 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s return listBucketResult, nil } -// ListObjects - (List Objects) - List some objects or all recursively. -// -// ListObjects lists all objects matching the objectPrefix from -// the specified bucket. If recursion is enabled it would list -// all subdirectories and all its contents. -// -// Your input parameters are just bucketName, objectPrefix, recursive -// and a done channel for pro-actively closing the internal go -// routine. If you enable recursive as 'true' this function will -// return back all the objects in a given bucket name and object -// prefix. -// -// api := client.New(....) -// // Create a done channel. -// doneCh := make(chan struct{}) -// defer close(doneCh) -// // Recurively list all objects in 'mytestbucket' -// recursive := true -// for message := range api.ListObjects("mytestbucket", "starthere", recursive, doneCh) { -// fmt.Println(message) -// } -// -func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { +func (c Client) listObjects(ctx context.Context, bucketName, objectPrefix string, recursive bool, maxKeys int) <-chan ObjectInfo { // Allocate new list objects channel. objectStatCh := make(chan ObjectInfo, 1) // Default listing is delimited at "/" @@ -385,11 +275,11 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // Initiate list objects goroutine here. go func(objectStatCh chan<- ObjectInfo) { defer close(objectStatCh) - // Save marker for next request. - var marker string + + marker := "" for { // Get list of objects a maximum of 1000 per request. - result, err := c.listObjectsQuery(bucketName, objectPrefix, marker, delimiter, 0) + result, err := c.listObjectsQuery(ctx, bucketName, objectPrefix, marker, delimiter, maxKeys) if err != nil { objectStatCh <- ObjectInfo{ Err: err, @@ -405,7 +295,7 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // Send object content. case objectStatCh <- object: // If receives done from the caller, return here. - case <-doneCh: + case <-ctx.Done(): return } } @@ -417,7 +307,7 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // Send object prefixes. case objectStatCh <- ObjectInfo{Key: obj.Prefix}: // If receives done from the caller, return here. - case <-doneCh: + case <-ctx.Done(): return } } @@ -436,6 +326,205 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don return objectStatCh } +func (c Client) listObjectVersions(ctx context.Context, bucketName, prefix string, recursive bool, maxKeys int) <-chan ObjectInfo { + // Allocate new list objects channel. + resultCh := make(chan ObjectInfo, 1) + // Default listing is delimited at "/" + delimiter := "/" + if recursive { + // If recursive we do not delimit. + delimiter = "" + } + + // Validate bucket name. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + defer close(resultCh) + resultCh <- ObjectInfo{ + Err: err, + } + return resultCh + } + + // Validate incoming object prefix. + if err := s3utils.CheckValidObjectNamePrefix(prefix); err != nil { + defer close(resultCh) + resultCh <- ObjectInfo{ + Err: err, + } + return resultCh + } + + // Initiate list objects goroutine here. + go func(resultCh chan<- ObjectInfo) { + defer close(resultCh) + + var ( + keyMarker = "" + versionIDMarker = "" + ) + + for { + // Get list of objects a maximum of 1000 per request. + result, err := c.listObjectVersionsQuery(ctx, bucketName, prefix, keyMarker, versionIDMarker, delimiter, maxKeys) + if err != nil { + resultCh <- ObjectInfo{ + Err: err, + } + return + } + + // If contents are available loop through and send over channel. + for _, version := range result.Versions { + info := ObjectInfo{ + ETag: trimEtag(version.ETag), + Key: version.Key, + LastModified: version.LastModified, + Size: version.Size, + Owner: version.Owner, + StorageClass: version.StorageClass, + IsLatest: version.IsLatest, + VersionID: version.VersionID, + + IsDeleteMarker: version.isDeleteMarker, + } + select { + // Send object version info. + case resultCh <- info: + // If receives done from the caller, return here. + case <-ctx.Done(): + return + } + } + + // Send all common prefixes if any. + // NOTE: prefixes are only present if the request is delimited. + for _, obj := range result.CommonPrefixes { + select { + // Send object prefixes. + case resultCh <- ObjectInfo{Key: obj.Prefix}: + // If receives done from the caller, return here. + case <-ctx.Done(): + return + } + } + + // If next key marker is present, save it for next request. + if result.NextKeyMarker != "" { + keyMarker = result.NextKeyMarker + } + + // If next version id marker is present, save it for next request. + if result.NextVersionIDMarker != "" { + versionIDMarker = result.NextVersionIDMarker + } + + // Listing ends result is not truncated, return right here. + if !result.IsTruncated { + return + } + } + }(resultCh) + return resultCh +} + +// listObjectVersions - (List Object Versions) - List some or all (up to 1000) of the existing objects +// and their versions in a bucket. +// +// You can use the request parameters as selection criteria to return a subset of the objects in a bucket. +// request parameters :- +// --------- +// ?key-marker - Specifies the key to start with when listing objects in a bucket. +// ?version-id-marker - Specifies the version id marker to start with when listing objects with versions in a bucket. +// ?delimiter - A delimiter is a character you use to group keys. +// ?prefix - Limits the response to keys that begin with the specified prefix. +// ?max-keys - Sets the maximum number of keys returned in the response body. +func (c Client) listObjectVersionsQuery(ctx context.Context, bucketName, prefix, keyMarker, versionIDMarker, delimiter string, maxkeys int) (ListVersionsResult, error) { + // Validate bucket name. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return ListVersionsResult{}, err + } + // Validate object prefix. + if err := s3utils.CheckValidObjectNamePrefix(prefix); err != nil { + return ListVersionsResult{}, err + } + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + + // Set versions to trigger versioning API + urlValues.Set("versions", "") + + // Set object prefix, prefix value to be set to empty is okay. + urlValues.Set("prefix", prefix) + + // Set delimiter, delimiter value to be set to empty is okay. + urlValues.Set("delimiter", delimiter) + + // Set object marker. + if keyMarker != "" { + urlValues.Set("key-marker", keyMarker) + } + + // Set max keys. + if maxkeys > 0 { + urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) + } + + // Set version ID marker + if versionIDMarker != "" { + urlValues.Set("version-id-marker", versionIDMarker) + } + + // Always set encoding-type + urlValues.Set("encoding-type", "url") + + // Execute GET on bucket to list objects. + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return ListVersionsResult{}, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return ListVersionsResult{}, httpRespToErrorResponse(resp, bucketName, "") + } + } + + // Decode ListVersionsResult XML. + listObjectVersionsOutput := ListVersionsResult{} + err = xmlDecoder(resp.Body, &listObjectVersionsOutput) + if err != nil { + return ListVersionsResult{}, err + } + + for i, obj := range listObjectVersionsOutput.Versions { + listObjectVersionsOutput.Versions[i].Key, err = decodeS3Name(obj.Key, listObjectVersionsOutput.EncodingType) + if err != nil { + return listObjectVersionsOutput, err + } + } + + for i, obj := range listObjectVersionsOutput.CommonPrefixes { + listObjectVersionsOutput.CommonPrefixes[i].Prefix, err = decodeS3Name(obj.Prefix, listObjectVersionsOutput.EncodingType) + if err != nil { + return listObjectVersionsOutput, err + } + } + + if listObjectVersionsOutput.NextKeyMarker != "" { + listObjectVersionsOutput.NextKeyMarker, err = decodeS3Name(listObjectVersionsOutput.NextKeyMarker, listObjectVersionsOutput.EncodingType) + if err != nil { + return listObjectVersionsOutput, err + } + } + + return listObjectVersionsOutput, nil +} + // listObjects - (List Objects) - List some or all (up to 1000) of the objects in a bucket. // // You can use the request parameters as selection criteria to return a subset of the objects in a bucket. @@ -445,7 +534,7 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // ?delimiter - A delimiter is a character you use to group keys. // ?prefix - Limits the response to keys that begin with the specified prefix. // ?max-keys - Sets the maximum number of keys returned in the response body. -func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimiter string, maxkeys int) (ListBucketResult, error) { +func (c Client) listObjectsQuery(ctx context.Context, bucketName, objectPrefix, objectMarker, delimiter string, maxkeys int) (ListBucketResult, error) { // Validate bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ListBucketResult{}, err @@ -478,7 +567,7 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit urlValues.Set("encoding-type", "url") // Execute GET on bucket to list objects. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, @@ -523,35 +612,74 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit return listBucketResult, nil } +// ListObjectsOptions holds all options of a list object request +type ListObjectsOptions struct { + // Include objects versions in the listing + WithVersions bool + // Include objects metadata in the listing + WithMetadata bool + // Only list objects with the prefix + Prefix string + // Ignore '/' delimiter + Recursive bool + // The maximum number of objects requested per + // batch, advanced use-case not useful for most + // applications + MaxKeys int + + // Use the deprecated list objects V1 API + UseV1 bool +} + +// ListObjects returns objects list after evaluating the passed options. +// +// api := client.New(....) +// for object := range api.ListObjects(ctx, "mytestbucket", minio.ListObjectsOptions{Prefix: "starthere", Recursive:true}) { +// fmt.Println(object) +// } +// +func (c Client) ListObjects(ctx context.Context, bucketName string, opts ListObjectsOptions) <-chan ObjectInfo { + if opts.WithVersions { + return c.listObjectVersions(ctx, bucketName, opts.Prefix, opts.Recursive, opts.MaxKeys) + } + + // Use legacy list objects v1 API + if opts.UseV1 { + return c.listObjects(ctx, bucketName, opts.Prefix, opts.Recursive, opts.MaxKeys) + } + + // Check whether this is snowball region, if yes ListObjectsV2 doesn't work, fallback to listObjectsV1. + if location, ok := c.bucketLocCache.Get(bucketName); ok { + if location == "snowball" { + return c.listObjects(ctx, bucketName, opts.Prefix, opts.Recursive, opts.MaxKeys) + } + } + + return c.listObjectsV2(ctx, bucketName, opts.Prefix, opts.Recursive, opts.WithMetadata, opts.MaxKeys) +} + // ListIncompleteUploads - List incompletely uploaded multipart objects. // // ListIncompleteUploads lists all incompleted objects matching the // objectPrefix from the specified bucket. If recursion is enabled // it would list all subdirectories and all its contents. // -// Your input parameters are just bucketName, objectPrefix, recursive -// and a done channel to pro-actively close the internal go routine. +// Your input parameters are just bucketName, objectPrefix, recursive. // If you enable recursive as 'true' this function will return back all // the multipart objects in a given bucket name. // // api := client.New(....) -// // Create a done channel. -// doneCh := make(chan struct{}) -// defer close(doneCh) // // Recurively list all objects in 'mytestbucket' // recursive := true -// for message := range api.ListIncompleteUploads("mytestbucket", "starthere", recursive) { +// for message := range api.ListIncompleteUploads(context.Background(), "mytestbucket", "starthere", recursive) { // fmt.Println(message) // } -// -func (c Client) ListIncompleteUploads(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo { - // Turn on size aggregation of individual parts. - isAggregateSize := true - return c.listIncompleteUploads(bucketName, objectPrefix, recursive, isAggregateSize, doneCh) +func (c Client) ListIncompleteUploads(ctx context.Context, bucketName, objectPrefix string, recursive bool) <-chan ObjectMultipartInfo { + return c.listIncompleteUploads(ctx, bucketName, objectPrefix, recursive) } // listIncompleteUploads lists all incomplete uploads. -func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive, aggregateSize bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo { +func (c Client) listIncompleteUploads(ctx context.Context, bucketName, objectPrefix string, recursive bool) <-chan ObjectMultipartInfo { // Allocate channel for multipart uploads. objectMultipartStatCh := make(chan ObjectMultipartInfo, 1) // Delimiter is set to "/" by default. @@ -583,7 +711,7 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive var uploadIDMarker string for { // list all multipart uploads. - result, err := c.listMultipartUploadsQuery(bucketName, objectMarker, uploadIDMarker, objectPrefix, delimiter, 0) + result, err := c.listMultipartUploadsQuery(ctx, bucketName, objectMarker, uploadIDMarker, objectPrefix, delimiter, 0) if err != nil { objectMultipartStatCh <- ObjectMultipartInfo{ Err: err, @@ -596,21 +724,11 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive // Send all multipart uploads. for _, obj := range result.Uploads { // Calculate total size of the uploaded parts if 'aggregateSize' is enabled. - if aggregateSize { - // Get total multipart size. - obj.Size, err = c.getTotalMultipartSize(bucketName, obj.Key, obj.UploadID) - if err != nil { - objectMultipartStatCh <- ObjectMultipartInfo{ - Err: err, - } - continue - } - } select { // Send individual uploads here. case objectMultipartStatCh <- obj: - // If done channel return here. - case <-doneCh: + // If the context is canceled + case <-ctx.Done(): return } } @@ -620,8 +738,8 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive select { // Send delimited prefixes here. case objectMultipartStatCh <- ObjectMultipartInfo{Key: obj.Prefix, Size: 0}: - // If done channel return here. - case <-doneCh: + // If context is canceled. + case <-ctx.Done(): return } } @@ -633,9 +751,10 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive }(objectMultipartStatCh) // return. return objectMultipartStatCh + } -// listMultipartUploads - (List Multipart Uploads). +// listMultipartUploadsQuery - (List Multipart Uploads). // - Lists some or all (up to 1000) in-progress multipart uploads in a bucket. // // You can use the request parameters as selection criteria to return a subset of the uploads in a bucket. @@ -646,7 +765,7 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive // ?delimiter - A delimiter is a character you use to group keys. // ?prefix - Limits the response to keys that begin with the specified prefix. // ?max-uploads - Sets the maximum number of multipart uploads returned in the response body. -func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, prefix, delimiter string, maxUploads int) (ListMultipartUploadsResult, error) { +func (c Client) listMultipartUploadsQuery(ctx context.Context, bucketName, keyMarker, uploadIDMarker, prefix, delimiter string, maxUploads int) (ListMultipartUploadsResult, error) { // Get resources properly escaped and lined up before using them in http request. urlValues := make(url.Values) // Set uploads. @@ -676,7 +795,7 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, } // Execute GET on bucketName to list multipart uploads. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, @@ -725,13 +844,13 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, } // listObjectParts list all object parts recursively. -func (c Client) listObjectParts(bucketName, objectName, uploadID string) (partsInfo map[int]ObjectPart, err error) { +func (c Client) listObjectParts(ctx context.Context, bucketName, objectName, uploadID string) (partsInfo map[int]ObjectPart, err error) { // Part number marker for the next batch of request. var nextPartNumberMarker int partsInfo = make(map[int]ObjectPart) for { // Get list of uploaded parts a maximum of 1000 per request. - listObjPartsResult, err := c.listObjectPartsQuery(bucketName, objectName, uploadID, nextPartNumberMarker, 1000) + listObjPartsResult, err := c.listObjectPartsQuery(ctx, bucketName, objectName, uploadID, nextPartNumberMarker, 1000) if err != nil { return nil, err } @@ -754,17 +873,12 @@ func (c Client) listObjectParts(bucketName, objectName, uploadID string) (partsI } // findUploadIDs lists all incomplete uploads and find the uploadIDs of the matching object name. -func (c Client) findUploadIDs(bucketName, objectName string) ([]string, error) { +func (c Client) findUploadIDs(ctx context.Context, bucketName, objectName string) ([]string, error) { var uploadIDs []string // Make list incomplete uploads recursive. isRecursive := true - // Turn off size aggregation of individual parts, in this request. - isAggregateSize := false - // Create done channel to cleanup the routine. - doneCh := make(chan struct{}) - defer close(doneCh) // List all incomplete uploads. - for mpUpload := range c.listIncompleteUploads(bucketName, objectName, isRecursive, isAggregateSize, doneCh) { + for mpUpload := range c.listIncompleteUploads(ctx, bucketName, objectName, isRecursive) { if mpUpload.Err != nil { return nil, mpUpload.Err } @@ -776,19 +890,6 @@ func (c Client) findUploadIDs(bucketName, objectName string) ([]string, error) { return uploadIDs, nil } -// getTotalMultipartSize - calculate total uploaded size for the a given multipart object. -func (c Client) getTotalMultipartSize(bucketName, objectName, uploadID string) (size int64, err error) { - // Iterate over all parts and aggregate the size. - partsInfo, err := c.listObjectParts(bucketName, objectName, uploadID) - if err != nil { - return 0, err - } - for _, partInfo := range partsInfo { - size += partInfo.Size - } - return size, nil -} - // listObjectPartsQuery (List Parts query) // - lists some or all (up to 1000) parts that have been uploaded // for a specific multipart upload @@ -799,7 +900,7 @@ func (c Client) getTotalMultipartSize(bucketName, objectName, uploadID string) ( // ?part-number-marker - Specifies the part after which listing should // begin. // ?max-parts - Maximum parts to be listed per request. -func (c Client) listObjectPartsQuery(bucketName, objectName, uploadID string, partNumberMarker, maxParts int) (ListObjectPartsResult, error) { +func (c Client) listObjectPartsQuery(ctx context.Context, bucketName, objectName, uploadID string, partNumberMarker, maxParts int) (ListObjectPartsResult, error) { // Get resources properly escaped and lined up before using them in http request. urlValues := make(url.Values) // Set part number marker. @@ -814,7 +915,7 @@ func (c Client) listObjectPartsQuery(bucketName, objectName, uploadID string, pa } // Execute GET on objectName to get list of parts. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, diff --git a/vendor/github.com/minio/minio-go/v6/api-object-legal-hold.go b/vendor/github.com/minio/minio-go/v7/api-object-legal-hold.go similarity index 90% rename from vendor/github.com/minio/minio-go/v6/api-object-legal-hold.go rename to vendor/github.com/minio/minio-go/v7/api-object-legal-hold.go index 2b43575ace..b139c1687b 100644 --- a/vendor/github.com/minio/minio-go/v6/api-object-legal-hold.go +++ b/vendor/github.com/minio/minio-go/v7/api-object-legal-hold.go @@ -25,7 +25,7 @@ import ( "net/http" "net/url" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // objectLegalHold - object legal hold specified in @@ -81,7 +81,7 @@ func newObjectLegalHold(status *LegalHoldStatus) (*objectLegalHold, error) { } // PutObjectLegalHold : sets object legal hold for a given object and versionID. -func (c Client) PutObjectLegalHold(bucketName, objectName string, opts PutObjectLegalHoldOptions) error { +func (c Client) PutObjectLegalHold(ctx context.Context, bucketName, objectName string, opts PutObjectLegalHoldOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -121,7 +121,7 @@ func (c Client) PutObjectLegalHold(bucketName, objectName string, opts PutObject } // Execute PUT Object Legal Hold. - resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err @@ -135,7 +135,7 @@ func (c Client) PutObjectLegalHold(bucketName, objectName string, opts PutObject } // GetObjectLegalHold gets legal-hold status of given object. -func (c Client) GetObjectLegalHold(bucketName, objectName string, opts GetObjectLegalHoldOptions) (status *LegalHoldStatus, err error) { +func (c Client) GetObjectLegalHold(ctx context.Context, bucketName, objectName string, opts GetObjectLegalHoldOptions) (status *LegalHoldStatus, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, err @@ -152,7 +152,7 @@ func (c Client) GetObjectLegalHold(bucketName, objectName string, opts GetObject } // Execute GET on bucket to list objects. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, diff --git a/vendor/github.com/minio/minio-go/v6/api-object-lock.go b/vendor/github.com/minio/minio-go/v7/api-object-lock.go similarity index 86% rename from vendor/github.com/minio/minio-go/v6/api-object-lock.go rename to vendor/github.com/minio/minio-go/v7/api-object-lock.go index bdd2b07c40..29f52b0546 100644 --- a/vendor/github.com/minio/minio-go/v6/api-object-lock.go +++ b/vendor/github.com/minio/minio-go/v7/api-object-lock.go @@ -26,7 +26,7 @@ import ( "net/url" "time" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // RetentionMode - object retention mode. @@ -139,7 +139,7 @@ func newObjectLockConfig(mode *RetentionMode, validity *uint, unit *ValidityUnit } // SetBucketObjectLockConfig sets object lock configuration in given bucket. mode, validity and unit are either all set or all nil. -func (c Client) SetBucketObjectLockConfig(bucketName string, mode *RetentionMode, validity *uint, unit *ValidityUnit) error { +func (c Client) SetBucketObjectLockConfig(ctx context.Context, bucketName string, mode *RetentionMode, validity *uint, unit *ValidityUnit) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -170,7 +170,7 @@ func (c Client) SetBucketObjectLockConfig(bucketName string, mode *RetentionMode } // Execute PUT bucket object lock configuration. - resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err @@ -184,7 +184,7 @@ func (c Client) SetBucketObjectLockConfig(bucketName string, mode *RetentionMode } // GetObjectLockConfig gets object lock configuration of given bucket. -func (c Client) GetObjectLockConfig(bucketName string) (objectLock string, mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { +func (c Client) GetObjectLockConfig(ctx context.Context, bucketName string) (objectLock string, mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return "", nil, nil, nil, err @@ -194,7 +194,7 @@ func (c Client) GetObjectLockConfig(bucketName string) (objectLock string, mode urlValues.Set("object-lock", "") // Execute GET on bucket to list objects. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, @@ -230,12 +230,12 @@ func (c Client) GetObjectLockConfig(bucketName string) (objectLock string, mode } // GetBucketObjectLockConfig gets object lock configuration of given bucket. -func (c Client) GetBucketObjectLockConfig(bucketName string) (mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { - _, mode, validity, unit, err = c.GetObjectLockConfig(bucketName) +func (c Client) GetBucketObjectLockConfig(ctx context.Context, bucketName string) (mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { + _, mode, validity, unit, err = c.GetObjectLockConfig(ctx, bucketName) return mode, validity, unit, err } // SetObjectLockConfig sets object lock configuration in given bucket. mode, validity and unit are either all set or all nil. -func (c Client) SetObjectLockConfig(bucketName string, mode *RetentionMode, validity *uint, unit *ValidityUnit) error { - return c.SetBucketObjectLockConfig(bucketName, mode, validity, unit) +func (c Client) SetObjectLockConfig(ctx context.Context, bucketName string, mode *RetentionMode, validity *uint, unit *ValidityUnit) error { + return c.SetBucketObjectLockConfig(ctx, bucketName, mode, validity, unit) } diff --git a/vendor/github.com/minio/minio-go/v6/api-object-retention.go b/vendor/github.com/minio/minio-go/v7/api-object-retention.go similarity index 77% rename from vendor/github.com/minio/minio-go/v6/api-object-retention.go rename to vendor/github.com/minio/minio-go/v7/api-object-retention.go index 822bcc6042..54f2762de9 100644 --- a/vendor/github.com/minio/minio-go/v6/api-object-retention.go +++ b/vendor/github.com/minio/minio-go/v7/api-object-retention.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2019 MinIO, Inc. + * Copyright 2019-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ import ( "net/url" "time" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // objectRetention - object retention specified in @@ -34,26 +34,23 @@ import ( type objectRetention struct { XMLNS string `xml:"xmlns,attr,omitempty"` XMLName xml.Name `xml:"Retention"` - Mode RetentionMode `xml:"Mode"` - RetainUntilDate time.Time `type:"timestamp" timestampFormat:"iso8601" xml:"RetainUntilDate"` + Mode RetentionMode `xml:"Mode,omitempty"` + RetainUntilDate *time.Time `type:"timestamp" timestampFormat:"iso8601" xml:"RetainUntilDate,omitempty"` } func newObjectRetention(mode *RetentionMode, date *time.Time) (*objectRetention, error) { - if mode == nil { - return nil, fmt.Errorf("Mode not set") + objectRetention := &objectRetention{} + + if date != nil && !date.IsZero() { + objectRetention.RetainUntilDate = date + } + if mode != nil { + if !mode.IsValid() { + return nil, fmt.Errorf("invalid retention mode `%v`", mode) + } + objectRetention.Mode = *mode } - if date == nil { - return nil, fmt.Errorf("RetainUntilDate not set") - } - - if !mode.IsValid() { - return nil, fmt.Errorf("invalid retention mode `%v`", mode) - } - objectRetention := &objectRetention{ - Mode: *mode, - RetainUntilDate: *date, - } return objectRetention, nil } @@ -65,8 +62,8 @@ type PutObjectRetentionOptions struct { VersionID string } -// PutObjectRetention : sets object retention for a given object and versionID. -func (c Client) PutObjectRetention(bucketName, objectName string, opts PutObjectRetentionOptions) error { +// PutObjectRetention sets object retention for a given object and versionID. +func (c Client) PutObjectRetention(ctx context.Context, bucketName, objectName string, opts PutObjectRetentionOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -115,7 +112,7 @@ func (c Client) PutObjectRetention(bucketName, objectName string, opts PutObject } // Execute PUT Object Retention. - resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err @@ -129,7 +126,7 @@ func (c Client) PutObjectRetention(bucketName, objectName string, opts PutObject } // GetObjectRetention gets retention of given object. -func (c Client) GetObjectRetention(bucketName, objectName, versionID string) (mode *RetentionMode, retainUntilDate *time.Time, err error) { +func (c Client) GetObjectRetention(ctx context.Context, bucketName, objectName, versionID string) (mode *RetentionMode, retainUntilDate *time.Time, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, nil, err @@ -144,7 +141,7 @@ func (c Client) GetObjectRetention(bucketName, objectName, versionID string) (mo urlValues.Set("versionId", versionID) } // Execute GET on bucket to list objects. - resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, @@ -164,5 +161,5 @@ func (c Client) GetObjectRetention(bucketName, objectName, versionID string) (mo return nil, nil, err } - return &retention.Mode, &retention.RetainUntilDate, nil + return &retention.Mode, retention.RetainUntilDate, nil } diff --git a/vendor/github.com/minio/minio-go/v6/api-object-tagging.go b/vendor/github.com/minio/minio-go/v7/api-object-tagging.go similarity index 56% rename from vendor/github.com/minio/minio-go/v6/api-object-tagging.go rename to vendor/github.com/minio/minio-go/v7/api-object-tagging.go index 65df4f3e7d..2709efcd12 100644 --- a/vendor/github.com/minio/minio-go/v6/api-object-tagging.go +++ b/vendor/github.com/minio/minio-go/v7/api-object-tagging.go @@ -21,22 +21,22 @@ import ( "bytes" "context" "encoding/xml" - "io/ioutil" "net/http" "net/url" - "github.com/minio/minio-go/v6/pkg/s3utils" - "github.com/minio/minio-go/v6/pkg/tags" + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/tags" ) -// PutObjectTagging replaces or creates object tag(s) -func (c Client) PutObjectTagging(bucketName, objectName string, objectTags map[string]string) error { - return c.PutObjectTaggingWithContext(context.Background(), bucketName, objectName, objectTags) +// PutObjectTaggingOptions holds an object version id +// to update tag(s) of a specific object version +type PutObjectTaggingOptions struct { + VersionID string } -// PutObjectTaggingWithContext replaces or creates object tag(s) with a context to control cancellations -// and timeouts. -func (c Client) PutObjectTaggingWithContext(ctx context.Context, bucketName, objectName string, objectTags map[string]string) error { +// PutObjectTagging replaces or creates object tag(s) and can target +// a specific object version in a versioned bucket. +func (c Client) PutObjectTagging(ctx context.Context, bucketName, objectName string, otags *tags.Tags, opts PutObjectTaggingOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -47,12 +47,11 @@ func (c Client) PutObjectTaggingWithContext(ctx context.Context, bucketName, obj urlValues := make(url.Values) urlValues.Set("tagging", "") - tags, err := tags.NewTags(objectTags, true) - if err != nil { - return err + if opts.VersionID != "" { + urlValues.Set("versionId", opts.VersionID) } - reqBytes, err := xml.Marshal(tags) + reqBytes, err := xml.Marshal(otags) if err != nil { return err } @@ -67,7 +66,7 @@ func (c Client) PutObjectTaggingWithContext(ctx context.Context, bucketName, obj } // Execute PUT to set a object tagging. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err @@ -80,21 +79,26 @@ func (c Client) PutObjectTaggingWithContext(ctx context.Context, bucketName, obj return nil } -// GetObjectTagging fetches object tag(s) -func (c Client) GetObjectTagging(bucketName, objectName string) (string, error) { - return c.GetObjectTaggingWithContext(context.Background(), bucketName, objectName) +// GetObjectTaggingOptions holds the object version ID +// to fetch the tagging key/value pairs +type GetObjectTaggingOptions struct { + VersionID string } -// GetObjectTaggingWithContext fetches object tag(s) with a context to control cancellations -// and timeouts. -func (c Client) GetObjectTaggingWithContext(ctx context.Context, bucketName, objectName string) (string, error) { +// GetObjectTagging fetches object tag(s) with options to target +// a specific object version in a versioned bucket. +func (c Client) GetObjectTagging(ctx context.Context, bucketName, objectName string, opts GetObjectTaggingOptions) (*tags.Tags, error) { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("tagging", "") + if opts.VersionID != "" { + urlValues.Set("versionId", opts.VersionID) + } + // Execute GET on object to get object tag(s) - resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, @@ -102,38 +106,37 @@ func (c Client) GetObjectTaggingWithContext(ctx context.Context, bucketName, obj defer closeResponse(resp) if err != nil { - return "", err + return nil, err } if resp != nil { if resp.StatusCode != http.StatusOK { - return "", httpRespToErrorResponse(resp, bucketName, objectName) + return nil, httpRespToErrorResponse(resp, bucketName, objectName) } } - tagBuf, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", err - } - - return string(tagBuf), err + return tags.ParseObjectXML(resp.Body) } -// RemoveObjectTagging deletes object tag(s) -func (c Client) RemoveObjectTagging(bucketName, objectName string) error { - return c.RemoveObjectTaggingWithContext(context.Background(), bucketName, objectName) +// RemoveObjectTaggingOptions holds the version id of the object to remove +type RemoveObjectTaggingOptions struct { + VersionID string } -// RemoveObjectTaggingWithContext removes object tag(s) with a context to control cancellations -// and timeouts. -func (c Client) RemoveObjectTaggingWithContext(ctx context.Context, bucketName, objectName string) error { +// RemoveObjectTagging removes object tag(s) with options to control a specific object +// version in a versioned bucket +func (c Client) RemoveObjectTagging(ctx context.Context, bucketName, objectName string, opts RemoveObjectTaggingOptions) error { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("tagging", "") + if opts.VersionID != "" { + urlValues.Set("versionId", opts.VersionID) + } + // Execute DELETE on object to remove object tag(s) - resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, diff --git a/vendor/github.com/minio/minio-go/v6/api-presigned.go b/vendor/github.com/minio/minio-go/v7/api-presigned.go similarity index 75% rename from vendor/github.com/minio/minio-go/v6/api-presigned.go rename to vendor/github.com/minio/minio-go/v7/api-presigned.go index 2e27c3f97a..80c363da5b 100644 --- a/vendor/github.com/minio/minio-go/v6/api-presigned.go +++ b/vendor/github.com/minio/minio-go/v7/api-presigned.go @@ -18,21 +18,22 @@ package minio import ( + "context" "errors" "net/http" "net/url" "time" - "github.com/minio/minio-go/v6/pkg/s3utils" - "github.com/minio/minio-go/v6/pkg/signer" + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/signer" ) // presignURL - Returns a presigned URL for an input 'method'. // Expires maximum is 7days - ie. 604800 and minimum is 1. -func (c Client) presignURL(method string, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { +func (c Client) presignURL(ctx context.Context, method string, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { // Input validation. if method == "" { - return nil, ErrInvalidArgument("method cannot be empty.") + return nil, errInvalidArgument("method cannot be empty.") } if err = s3utils.CheckValidBucketName(bucketName); err != nil { return nil, err @@ -54,7 +55,7 @@ func (c Client) presignURL(method string, bucketName string, objectName string, // Instantiate a new request. // Since expires is set newRequest will presign the request. var req *http.Request - if req, err = c.newRequest(method, reqMetadata); err != nil { + if req, err = c.newRequest(ctx, method, reqMetadata); err != nil { return nil, err } return req.URL, nil @@ -64,43 +65,43 @@ func (c Client) presignURL(method string, bucketName string, objectName string, // data without credentials. URL can have a maximum expiry of // upto 7days or a minimum of 1sec. Additionally you can override // a set of response headers using the query parameters. -func (c Client) PresignedGetObject(bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { +func (c Client) PresignedGetObject(ctx context.Context, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { if err = s3utils.CheckValidObjectName(objectName); err != nil { return nil, err } - return c.presignURL("GET", bucketName, objectName, expires, reqParams) + return c.presignURL(ctx, http.MethodGet, bucketName, objectName, expires, reqParams) } -// PresignedHeadObject - Returns a presigned URL to access object -// metadata without credentials. URL can have a maximum expiry of -// upto 7days or a minimum of 1sec. Additionally you can override +// PresignedHeadObject - Returns a presigned URL to access +// object metadata without credentials. URL can have a maximum expiry +// of upto 7days or a minimum of 1sec. Additionally you can override // a set of response headers using the query parameters. -func (c Client) PresignedHeadObject(bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { +func (c Client) PresignedHeadObject(ctx context.Context, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { if err = s3utils.CheckValidObjectName(objectName); err != nil { return nil, err } - return c.presignURL("HEAD", bucketName, objectName, expires, reqParams) + return c.presignURL(ctx, http.MethodHead, bucketName, objectName, expires, reqParams) } // PresignedPutObject - Returns a presigned URL to upload an object // without credentials. URL can have a maximum expiry of upto 7days // or a minimum of 1sec. -func (c Client) PresignedPutObject(bucketName string, objectName string, expires time.Duration) (u *url.URL, err error) { +func (c Client) PresignedPutObject(ctx context.Context, bucketName string, objectName string, expires time.Duration) (u *url.URL, err error) { if err = s3utils.CheckValidObjectName(objectName); err != nil { return nil, err } - return c.presignURL("PUT", bucketName, objectName, expires, nil) + return c.presignURL(ctx, http.MethodPut, bucketName, objectName, expires, nil) } // Presign - returns a presigned URL for any http method of your choice // along with custom request params. URL can have a maximum expiry of // upto 7days or a minimum of 1sec. -func (c Client) Presign(method string, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { - return c.presignURL(method, bucketName, objectName, expires, reqParams) +func (c Client) Presign(ctx context.Context, method string, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { + return c.presignURL(ctx, method, bucketName, objectName, expires, reqParams) } // PresignedPostPolicy - Returns POST urlString, form data to upload an object. -func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[string]string, err error) { +func (c Client) PresignedPostPolicy(ctx context.Context, p *PostPolicy) (u *url.URL, formData map[string]string, err error) { // Validate input arguments. if p.expiration.IsZero() { return nil, nil, errors.New("Expiration time must be specified") @@ -114,7 +115,7 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[str bucketName := p.formData["bucket"] // Fetch the bucket location. - location, err := c.getBucketLocation(bucketName) + location, err := c.getBucketLocation(ctx, bucketName) if err != nil { return nil, nil, err } @@ -140,7 +141,7 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[str ) if signerType.IsAnonymous() { - return nil, nil, ErrInvalidArgument("Presigned operations are not supported for anonymous credentials") + return nil, nil, errInvalidArgument("Presigned operations are not supported for anonymous credentials") } // Keep time. diff --git a/vendor/github.com/minio/minio-go/v7/api-put-bucket.go b/vendor/github.com/minio/minio-go/v7/api-put-bucket.go new file mode 100644 index 0000000000..df9fe98afa --- /dev/null +++ b/vendor/github.com/minio/minio-go/v7/api-put-bucket.go @@ -0,0 +1,123 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/xml" + "net/http" + + "github.com/minio/minio-go/v7/pkg/s3utils" +) + +/// Bucket operations +func (c Client) makeBucket(ctx context.Context, bucketName string, opts MakeBucketOptions) (err error) { + // Validate the input arguments. + if err := s3utils.CheckValidBucketNameStrict(bucketName); err != nil { + return err + } + + err = c.doMakeBucket(ctx, bucketName, opts.Region, opts.ObjectLocking) + if err != nil && (opts.Region == "" || opts.Region == "us-east-1") { + if resp, ok := err.(ErrorResponse); ok && resp.Code == "AuthorizationHeaderMalformed" && resp.Region != "" { + err = c.doMakeBucket(ctx, bucketName, resp.Region, opts.ObjectLocking) + } + } + return err +} + +func (c Client) doMakeBucket(ctx context.Context, bucketName string, location string, objectLockEnabled bool) (err error) { + defer func() { + // Save the location into cache on a successful makeBucket response. + if err == nil { + c.bucketLocCache.Set(bucketName, location) + } + }() + + // If location is empty, treat is a default region 'us-east-1'. + if location == "" { + location = "us-east-1" + // For custom region clients, default + // to custom region instead not 'us-east-1'. + if c.region != "" { + location = c.region + } + } + // PUT bucket request metadata. + reqMetadata := requestMetadata{ + bucketName: bucketName, + bucketLocation: location, + } + + if objectLockEnabled { + headers := make(http.Header) + headers.Add("x-amz-bucket-object-lock-enabled", "true") + reqMetadata.customHeader = headers + } + + // If location is not 'us-east-1' create bucket location config. + if location != "us-east-1" && location != "" { + createBucketConfig := createBucketConfiguration{} + createBucketConfig.Location = location + var createBucketConfigBytes []byte + createBucketConfigBytes, err = xml.Marshal(createBucketConfig) + if err != nil { + return err + } + reqMetadata.contentMD5Base64 = sumMD5Base64(createBucketConfigBytes) + reqMetadata.contentSHA256Hex = sum256Hex(createBucketConfigBytes) + reqMetadata.contentBody = bytes.NewReader(createBucketConfigBytes) + reqMetadata.contentLength = int64(len(createBucketConfigBytes)) + } + + // Execute PUT to create a new bucket. + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + + if resp != nil { + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + } + + // Success. + return nil +} + +// MakeBucketOptions holds all options to tweak bucket creation +type MakeBucketOptions struct { + // Bucket location + Region string + // Enable object locking + ObjectLocking bool +} + +// MakeBucket creates a new bucket with bucketName with a context to control cancellations and timeouts. +// +// Location is an optional argument, by default all buckets are +// created in US Standard Region. +// +// For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html +// For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations +func (c Client) MakeBucket(ctx context.Context, bucketName string, opts MakeBucketOptions) (err error) { + return c.makeBucket(ctx, bucketName, opts) +} diff --git a/vendor/github.com/minio/minio-go/v6/api-put-object-common.go b/vendor/github.com/minio/minio-go/v7/api-put-object-common.go similarity index 91% rename from vendor/github.com/minio/minio-go/v6/api-put-object-common.go rename to vendor/github.com/minio/minio-go/v7/api-put-object-common.go index 78e0a2fdcd..3d0408e534 100644 --- a/vendor/github.com/minio/minio-go/v6/api-put-object-common.go +++ b/vendor/github.com/minio/minio-go/v7/api-put-object-common.go @@ -23,7 +23,7 @@ import ( "math" "os" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // Verify if reader is *minio.Object @@ -77,31 +77,31 @@ func optimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCou // object size is larger than supported maximum. if objectSize > maxMultipartPutObjectSize { - err = ErrEntityTooLarge(objectSize, maxMultipartPutObjectSize, "", "") + err = errEntityTooLarge(objectSize, maxMultipartPutObjectSize, "", "") return } var partSizeFlt float64 if configuredPartSize > 0 { if int64(configuredPartSize) > objectSize { - err = ErrEntityTooLarge(int64(configuredPartSize), objectSize, "", "") + err = errEntityTooLarge(int64(configuredPartSize), objectSize, "", "") return } if !unknownSize { if objectSize > (int64(configuredPartSize) * maxPartsCount) { - err = ErrInvalidArgument("Part size * max_parts(10000) is lesser than input objectSize.") + err = errInvalidArgument("Part size * max_parts(10000) is lesser than input objectSize.") return } } if configuredPartSize < absMinPartSize { - err = ErrInvalidArgument("Input part size is smaller than allowed minimum of 5MiB.") + err = errInvalidArgument("Input part size is smaller than allowed minimum of 5MiB.") return } if configuredPartSize > maxPartSize { - err = ErrInvalidArgument("Input part size is bigger than allowed maximum of 5GiB.") + err = errInvalidArgument("Input part size is bigger than allowed maximum of 5GiB.") return } diff --git a/vendor/github.com/minio/minio-go/v7/api-put-object-copy.go b/vendor/github.com/minio/minio-go/v7/api-put-object-copy.go new file mode 100644 index 0000000000..9af036ec05 --- /dev/null +++ b/vendor/github.com/minio/minio-go/v7/api-put-object-copy.go @@ -0,0 +1,77 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017, 2018 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "io" + "io/ioutil" + "net/http" +) + +// CopyObject - copy a source object into a new object +func (c Client) CopyObject(ctx context.Context, dst CopyDestOptions, src CopySrcOptions) (UploadInfo, error) { + if err := src.validate(); err != nil { + return UploadInfo{}, err + } + + if err := dst.validate(); err != nil { + return UploadInfo{}, err + } + + header := make(http.Header) + dst.Marshal(header) + src.Marshal(header) + + resp, err := c.executeMethod(ctx, http.MethodPut, requestMetadata{ + bucketName: dst.Bucket, + objectName: dst.Object, + customHeader: header, + }) + if err != nil { + return UploadInfo{}, err + } + defer closeResponse(resp) + + if resp.StatusCode != http.StatusOK { + return UploadInfo{}, httpRespToErrorResponse(resp, dst.Bucket, dst.Object) + } + + // Update the progress properly after successful copy. + if dst.Progress != nil { + io.Copy(ioutil.Discard, io.LimitReader(dst.Progress, dst.Size)) + } + + cpObjRes := copyObjectResult{} + if err = xmlDecoder(resp.Body, &cpObjRes); err != nil { + return UploadInfo{}, err + } + + // extract lifecycle expiry date and rule ID + expTime, ruleID := amzExpirationToExpiryDateRuleID(resp.Header.Get(amzExpiration)) + + return UploadInfo{ + Bucket: dst.Bucket, + Key: dst.Object, + LastModified: cpObjRes.LastModified, + ETag: trimEtag(resp.Header.Get("ETag")), + VersionID: resp.Header.Get(amzVersionID), + Expiration: expTime, + ExpirationRuleID: ruleID, + }, nil +} diff --git a/vendor/github.com/minio/minio-go/v6/api-put-object-file-context.go b/vendor/github.com/minio/minio-go/v7/api-put-object-file-context.go similarity index 75% rename from vendor/github.com/minio/minio-go/v6/api-put-object-file-context.go rename to vendor/github.com/minio/minio-go/v7/api-put-object-file-context.go index fb22c0d64b..6c0f20df37 100644 --- a/vendor/github.com/minio/minio-go/v6/api-put-object-file-context.go +++ b/vendor/github.com/minio/minio-go/v7/api-put-object-file-context.go @@ -23,31 +23,31 @@ import ( "os" "path/filepath" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) -// FPutObjectWithContext - Create an object in a bucket, with contents from file at filePath. Allows request cancellation. -func (c Client) FPutObjectWithContext(ctx context.Context, bucketName, objectName, filePath string, opts PutObjectOptions) (n int64, err error) { +// FPutObject - Create an object in a bucket, with contents from file at filePath. Allows request cancellation. +func (c Client) FPutObject(ctx context.Context, bucketName, objectName, filePath string, opts PutObjectOptions) (info UploadInfo, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return 0, err + return UploadInfo{}, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { - return 0, err + return UploadInfo{}, err } // Open the referenced file. fileReader, err := os.Open(filePath) // If any error fail quickly here. if err != nil { - return 0, err + return UploadInfo{}, err } defer fileReader.Close() // Save the file stat. fileStat, err := fileReader.Stat() if err != nil { - return 0, err + return UploadInfo{}, err } // Save the file size. @@ -60,5 +60,5 @@ func (c Client) FPutObjectWithContext(ctx context.Context, bucketName, objectNam opts.ContentType = "application/octet-stream" } } - return c.PutObjectWithContext(ctx, bucketName, objectName, fileReader, fileSize, opts) + return c.PutObject(ctx, bucketName, objectName, fileReader, fileSize, opts) } diff --git a/vendor/github.com/minio/minio-go/v6/api-put-object-multipart.go b/vendor/github.com/minio/minio-go/v7/api-put-object-multipart.go similarity index 81% rename from vendor/github.com/minio/minio-go/v6/api-put-object-multipart.go rename to vendor/github.com/minio/minio-go/v7/api-put-object-multipart.go index 62af16cc2d..1c862ad968 100644 --- a/vendor/github.com/minio/minio-go/v6/api-put-object-multipart.go +++ b/vendor/github.com/minio/minio-go/v7/api-put-object-multipart.go @@ -32,13 +32,13 @@ import ( "strconv" "strings" - "github.com/minio/minio-go/v6/pkg/encrypt" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/encrypt" + "github.com/minio/minio-go/v7/pkg/s3utils" ) func (c Client) putObjectMultipart(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, - opts PutObjectOptions) (n int64, err error) { - n, err = c.putObjectMultipartNoStream(ctx, bucketName, objectName, reader, opts) + opts PutObjectOptions) (info UploadInfo, err error) { + info, err = c.putObjectMultipartNoStream(ctx, bucketName, objectName, reader, opts) if err != nil { errResp := ToErrorResponse(err) // Verify if multipart functionality is not available, if not @@ -46,22 +46,22 @@ func (c Client) putObjectMultipart(ctx context.Context, bucketName, objectName s if errResp.Code == "AccessDenied" && strings.Contains(errResp.Message, "Access Denied") { // Verify if size of reader is greater than '5GiB'. if size > maxSinglePutObjectSize { - return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) + return UploadInfo{}, errEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) } // Fall back to uploading as single PutObject operation. return c.putObject(ctx, bucketName, objectName, reader, size, opts) } } - return n, err + return info, err } -func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, objectName string, reader io.Reader, opts PutObjectOptions) (n int64, err error) { +func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, objectName string, reader io.Reader, opts PutObjectOptions) (info UploadInfo, err error) { // Input validation. if err = s3utils.CheckValidBucketName(bucketName); err != nil { - return 0, err + return UploadInfo{}, err } if err = s3utils.CheckValidObjectName(objectName); err != nil { - return 0, err + return UploadInfo{}, err } // Total data read and written to server. should be equal to @@ -74,13 +74,13 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje // Calculate the optimal parts info for a given size. totalPartsCount, partSize, _, err := optimalPartInfo(-1, opts.PartSize) if err != nil { - return 0, err + return UploadInfo{}, err } // Initiate a new multipart upload. uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts) if err != nil { - return 0, err + return UploadInfo{}, err } defer func() { @@ -104,12 +104,13 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje // HTTPS connection. hashAlgos, hashSums := c.hashMaterials(opts.SendContentMd5) - length, rErr := io.ReadFull(reader, buf) - if rErr == io.EOF { + length, rErr := readFull(reader, buf) + if rErr == io.EOF && partNumber > 1 { break } - if rErr != nil && rErr != io.ErrUnexpectedEOF { - return 0, rErr + + if rErr != nil && rErr != io.ErrUnexpectedEOF && rErr != io.EOF { + return UploadInfo{}, rErr } // Calculates hash sums while copying partSize bytes into cw. @@ -139,7 +140,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje objPart, uerr := c.uploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber, md5Base64, sha256Hex, int64(length), opts.ServerSideEncryption) if uerr != nil { - return totalUploadedSize, uerr + return UploadInfo{}, uerr } // Save successfully uploaded part metadata. @@ -163,7 +164,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje for i := 1; i < partNumber; i++ { part, ok := partsInfo[i] if !ok { - return 0, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", i)) + return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Missing part number %d", i)) } complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ ETag: part.ETag, @@ -173,12 +174,14 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) - if _, err = c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload); err != nil { - return totalUploadedSize, err + + uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) + if err != nil { + return UploadInfo{}, err } - // Return final size. - return totalUploadedSize, nil + uploadInfo.Size = totalUploadedSize + return uploadInfo, nil } // initiateMultipartUpload - Initiates a multipart upload and returns an upload ID. @@ -206,7 +209,7 @@ func (c Client) initiateMultipartUpload(ctx context.Context, bucketName, objectN } // Execute POST on an objectName to initiate multipart upload. - resp, err := c.executeMethod(ctx, "POST", reqMetadata) + resp, err := c.executeMethod(ctx, http.MethodPost, reqMetadata) defer closeResponse(resp) if err != nil { return initiateMultipartUploadResult{}, err @@ -236,16 +239,16 @@ func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID return ObjectPart{}, err } if size > maxPartSize { - return ObjectPart{}, ErrEntityTooLarge(size, maxPartSize, bucketName, objectName) + return ObjectPart{}, errEntityTooLarge(size, maxPartSize, bucketName, objectName) } if size <= -1 { - return ObjectPart{}, ErrEntityTooSmall(size, bucketName, objectName) + return ObjectPart{}, errEntityTooSmall(size, bucketName, objectName) } if partNumber <= 0 { - return ObjectPart{}, ErrInvalidArgument("Part number cannot be negative or equal to zero.") + return ObjectPart{}, errInvalidArgument("Part number cannot be negative or equal to zero.") } if uploadID == "" { - return ObjectPart{}, ErrInvalidArgument("UploadID cannot be empty.") + return ObjectPart{}, errInvalidArgument("UploadID cannot be empty.") } // Get resources properly escaped and lined up before using them in http request. @@ -277,7 +280,7 @@ func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID } // Execute PUT on each part. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return ObjectPart{}, err @@ -298,13 +301,13 @@ func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID // completeMultipartUpload - Completes a multipart upload by assembling previously uploaded parts. func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectName, uploadID string, - complete completeMultipartUpload) (completeMultipartUploadResult, error) { + complete completeMultipartUpload) (UploadInfo, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return completeMultipartUploadResult{}, err + return UploadInfo{}, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { - return completeMultipartUploadResult{}, err + return UploadInfo{}, err } // Initialize url queries. @@ -313,7 +316,7 @@ func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectN // Marshal complete multipart body. completeMultipartUploadBytes, err := xml.Marshal(complete) if err != nil { - return completeMultipartUploadResult{}, err + return UploadInfo{}, err } // Instantiate all the complete multipart buffer. @@ -328,14 +331,14 @@ func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectN } // Execute POST to complete multipart upload for an objectName. - resp, err := c.executeMethod(ctx, "POST", reqMetadata) + resp, err := c.executeMethod(ctx, http.MethodPost, reqMetadata) defer closeResponse(resp) if err != nil { - return completeMultipartUploadResult{}, err + return UploadInfo{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { - return completeMultipartUploadResult{}, httpRespToErrorResponse(resp, bucketName, objectName) + return UploadInfo{}, httpRespToErrorResponse(resp, bucketName, objectName) } } @@ -343,14 +346,14 @@ func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectN var b []byte b, err = ioutil.ReadAll(resp.Body) if err != nil { - return completeMultipartUploadResult{}, err + return UploadInfo{}, err } // Decode completed multipart upload response on success. completeMultipartUploadResult := completeMultipartUploadResult{} err = xmlDecoder(bytes.NewReader(b), &completeMultipartUploadResult) if err != nil { // xml parsing failure due to presence an ill-formed xml fragment - return completeMultipartUploadResult, err + return UploadInfo{}, err } else if completeMultipartUploadResult.Bucket == "" { // xml's Decode method ignores well-formed xml that don't apply to the type of value supplied. // In this case, it would leave completeMultipartUploadResult with the corresponding zero-values @@ -361,9 +364,22 @@ func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectN err = xmlDecoder(bytes.NewReader(b), &completeMultipartUploadErr) if err != nil { // xml parsing failure due to presence an ill-formed xml fragment - return completeMultipartUploadResult, err + return UploadInfo{}, err } - return completeMultipartUploadResult, completeMultipartUploadErr + return UploadInfo{}, completeMultipartUploadErr } - return completeMultipartUploadResult, nil + + // extract lifecycle expiry date and rule ID + expTime, ruleID := amzExpirationToExpiryDateRuleID(resp.Header.Get(amzExpiration)) + + return UploadInfo{ + Bucket: completeMultipartUploadResult.Bucket, + Key: completeMultipartUploadResult.Key, + ETag: trimEtag(completeMultipartUploadResult.ETag), + VersionID: resp.Header.Get(amzVersionID), + Location: completeMultipartUploadResult.Location, + Expiration: expTime, + ExpirationRuleID: ruleID, + }, nil + } diff --git a/vendor/github.com/minio/minio-go/v6/api-put-object-streaming.go b/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go similarity index 75% rename from vendor/github.com/minio/minio-go/v6/api-put-object-streaming.go rename to vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go index f0c5b13313..251fee9f84 100644 --- a/vendor/github.com/minio/minio-go/v6/api-put-object-streaming.go +++ b/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go @@ -24,10 +24,12 @@ import ( "fmt" "io" "net/http" + "net/url" "sort" "strings" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/google/uuid" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // putObjectMultipartStream - upload a large object using @@ -40,13 +42,13 @@ import ( // - Any reader which has a method 'ReadAt()' // func (c Client) putObjectMultipartStream(ctx context.Context, bucketName, objectName string, - reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { + reader io.Reader, size int64, opts PutObjectOptions) (info UploadInfo, err error) { if !isObject(reader) && isReadAt(reader) && !opts.SendContentMd5 { // Verify if the reader implements ReadAt and it is not a *minio.Object then we will use parallel uploader. - n, err = c.putObjectMultipartStreamFromReadAt(ctx, bucketName, objectName, reader.(io.ReaderAt), size, opts) + info, err = c.putObjectMultipartStreamFromReadAt(ctx, bucketName, objectName, reader.(io.ReaderAt), size, opts) } else { - n, err = c.putObjectMultipartStreamOptionalChecksum(ctx, bucketName, objectName, reader, size, opts) + info, err = c.putObjectMultipartStreamOptionalChecksum(ctx, bucketName, objectName, reader, size, opts) } if err != nil { errResp := ToErrorResponse(err) @@ -55,13 +57,13 @@ func (c Client) putObjectMultipartStream(ctx context.Context, bucketName, object if errResp.Code == "AccessDenied" && strings.Contains(errResp.Message, "Access Denied") { // Verify if size of reader is greater than '5GiB'. if size > maxSinglePutObjectSize { - return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) + return UploadInfo{}, errEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) } // Fall back to uploading as single PutObject operation. return c.putObject(ctx, bucketName, objectName, reader, size, opts) } } - return n, err + return info, err } // uploadedPartRes - the response received from a part upload. @@ -89,25 +91,25 @@ type uploadPartReq struct { // cleaned automatically when the caller i.e http client closes the // stream after uploading all the contents successfully. func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketName, objectName string, - reader io.ReaderAt, size int64, opts PutObjectOptions) (n int64, err error) { + reader io.ReaderAt, size int64, opts PutObjectOptions) (info UploadInfo, err error) { // Input validation. if err = s3utils.CheckValidBucketName(bucketName); err != nil { - return 0, err + return UploadInfo{}, err } if err = s3utils.CheckValidObjectName(objectName); err != nil { - return 0, err + return UploadInfo{}, err } // Calculate the optimal parts info for a given size. totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size, opts.PartSize) if err != nil { - return 0, err + return UploadInfo{}, err } // Initiate a new multipart upload. uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts) if err != nil { - return 0, err + return UploadInfo{}, err } // Aborts the multipart upload in progress, if the @@ -144,9 +146,15 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa uploadPartsCh <- uploadPartReq{PartNum: p} } close(uploadPartsCh) + + var partsBuf = make([][]byte, opts.getNumThreads()) + for i := range partsBuf { + partsBuf[i] = make([]byte, partSize) + } + // Receive each part number from the channel allowing three parallel uploads. for w := 1; w <= opts.getNumThreads(); w++ { - go func(partSize int64) { + go func(w int, partSize int64) { // Each worker will draw from the part channel and upload in parallel. for uploadReq := range uploadPartsCh { @@ -162,12 +170,21 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa partSize = lastPartSize } + n, rerr := readFull(io.NewSectionReader(reader, readOffset, partSize), partsBuf[w-1][:partSize]) + if rerr != nil && rerr != io.ErrUnexpectedEOF && err != io.EOF { + uploadedPartsCh <- uploadedPartRes{ + Error: rerr, + } + // Exit the goroutine. + return + } + // Get a section reader on a particular offset. - sectionReader := newHook(io.NewSectionReader(reader, readOffset, partSize), opts.Progress) + hookReader := newHook(bytes.NewReader(partsBuf[w-1][:n]), opts.Progress) // Proceed to upload the part. - objPart, err := c.uploadPart(ctx, bucketName, objectName, uploadID, - sectionReader, uploadReq.PartNum, + objPart, err := c.uploadPart(ctx, bucketName, objectName, + uploadID, hookReader, uploadReq.PartNum, "", "", partSize, opts.ServerSideEncryption) if err != nil { uploadedPartsCh <- uploadedPartRes{ @@ -187,7 +204,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa Part: uploadReq.Part, } } - }(partSize) + }(w, partSize) } // Gather the responses as they occur and update any @@ -195,7 +212,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa for u := 1; u <= totalPartsCount; u++ { uploadRes := <-uploadedPartsCh if uploadRes.Error != nil { - return totalUploadedSize, uploadRes.Error + return UploadInfo{}, uploadRes.Error } // Update the totalUploadedSize. totalUploadedSize += uploadRes.Size @@ -208,39 +225,40 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa // Verify if we uploaded all the data. if totalUploadedSize != size { - return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) + return UploadInfo{}, errUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) } // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) - _, err = c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) + + uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) if err != nil { - return totalUploadedSize, err + return UploadInfo{}, err } - // Return final size. - return totalUploadedSize, nil + uploadInfo.Size = totalUploadedSize + return uploadInfo, nil } func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bucketName, objectName string, - reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { + reader io.Reader, size int64, opts PutObjectOptions) (info UploadInfo, err error) { // Input validation. if err = s3utils.CheckValidBucketName(bucketName); err != nil { - return 0, err + return UploadInfo{}, err } if err = s3utils.CheckValidObjectName(objectName); err != nil { - return 0, err + return UploadInfo{}, err } // Calculate the optimal parts info for a given size. totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size, opts.PartSize) if err != nil { - return 0, err + return UploadInfo{}, err } // Initiates a new multipart request uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts) if err != nil { - return 0, err + return UploadInfo{}, err } // Aborts the multipart upload if the function returns @@ -276,13 +294,15 @@ func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bu } if opts.SendContentMd5 { - length, rerr := io.ReadFull(reader, buf) + length, rerr := readFull(reader, buf) if rerr == io.EOF && partNumber > 1 { break } - if rerr != nil && rerr != io.ErrUnexpectedEOF && rerr != io.EOF { - return 0, rerr + + if rerr != nil && rerr != io.ErrUnexpectedEOF && err != io.EOF { + return UploadInfo{}, rerr } + // Calculate md5sum. hash := c.md5Hasher() hash.Write(buf[:length]) @@ -302,7 +322,7 @@ func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bu io.LimitReader(hookReader, partSize), partNumber, md5Base64, "", partSize, opts.ServerSideEncryption) if uerr != nil { - return totalUploadedSize, uerr + return UploadInfo{}, uerr } // Save successfully uploaded part metadata. @@ -315,7 +335,7 @@ func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bu // Verify if we uploaded all the data. if size > 0 { if totalUploadedSize != size { - return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) + return UploadInfo{}, errUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) } } @@ -327,7 +347,7 @@ func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bu for i := 1; i < partNumber; i++ { part, ok := partsInfo[i] if !ok { - return 0, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", i)) + return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Missing part number %d", i)) } complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ ETag: part.ETag, @@ -337,34 +357,35 @@ func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bu // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) - _, err = c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) + + uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) if err != nil { - return totalUploadedSize, err + return UploadInfo{}, err } - // Return final size. - return totalUploadedSize, nil + uploadInfo.Size = totalUploadedSize + return uploadInfo, nil } // putObject special function used Google Cloud Storage. This special function // is used for Google Cloud Storage since Google's multipart API is not S3 compatible. -func (c Client) putObject(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { +func (c Client) putObject(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (info UploadInfo, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return 0, err + return UploadInfo{}, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { - return 0, err + return UploadInfo{}, err } // Size -1 is only supported on Google Cloud Storage, we error // out in all other situations. if size < 0 && !s3utils.IsGoogleEndpoint(*c.endpointURL) { - return 0, ErrEntityTooSmall(size, bucketName, objectName) + return UploadInfo{}, errEntityTooSmall(size, bucketName, objectName) } if opts.SendContentMd5 && s3utils.IsGoogleEndpoint(*c.endpointURL) && size < 0 { - return 0, ErrInvalidArgument("MD5Sum cannot be calculated with size '-1'") + return UploadInfo{}, errInvalidArgument("MD5Sum cannot be calculated with size '-1'") } if size > 0 { @@ -373,7 +394,7 @@ func (c Client) putObject(ctx context.Context, bucketName, objectName string, re if ok { offset, err := seeker.Seek(0, io.SeekCurrent) if err != nil { - return 0, ErrInvalidArgument(err.Error()) + return UploadInfo{}, errInvalidArgument(err.Error()) } reader = io.NewSectionReader(reader.(io.ReaderAt), offset, size) } @@ -385,9 +406,9 @@ func (c Client) putObject(ctx context.Context, bucketName, objectName string, re // Create a buffer. buf := make([]byte, size) - length, rErr := io.ReadFull(reader, buf) - if rErr != nil && rErr != io.ErrUnexpectedEOF { - return 0, rErr + length, rErr := readFull(reader, buf) + if rErr != nil && rErr != io.ErrUnexpectedEOF && rErr != io.EOF { + return UploadInfo{}, rErr } // Calculate md5sum. @@ -404,25 +425,18 @@ func (c Client) putObject(ctx context.Context, bucketName, objectName string, re // This function does not calculate sha256 and md5sum for payload. // Execute put object. - st, err := c.putObjectDo(ctx, bucketName, objectName, readSeeker, md5Base64, "", size, opts) - if err != nil { - return 0, err - } - if st.Size != size { - return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName) - } - return size, nil + return c.putObjectDo(ctx, bucketName, objectName, readSeeker, md5Base64, "", size, opts) } // putObjectDo - executes the put object http operation. // NOTE: You must have WRITE permissions on a bucket to add an object to it. -func (c Client) putObjectDo(ctx context.Context, bucketName, objectName string, reader io.Reader, md5Base64, sha256Hex string, size int64, opts PutObjectOptions) (ObjectInfo, error) { +func (c Client) putObjectDo(ctx context.Context, bucketName, objectName string, reader io.Reader, md5Base64, sha256Hex string, size int64, opts PutObjectOptions) (UploadInfo, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return ObjectInfo{}, err + return UploadInfo{}, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { - return ObjectInfo{}, err + return UploadInfo{}, err } // Set headers. customHeader := opts.Header() @@ -437,25 +451,36 @@ func (c Client) putObjectDo(ctx context.Context, bucketName, objectName string, contentMD5Base64: md5Base64, contentSHA256Hex: sha256Hex, } - + if opts.ReplicationVersionID != "" { + if _, err := uuid.Parse(opts.ReplicationVersionID); err != nil { + return UploadInfo{}, errInvalidArgument(err.Error()) + } + urlValues := make(url.Values) + urlValues.Set("versionId", opts.ReplicationVersionID) + reqMetadata.queryValues = urlValues + } // Execute PUT an objectName. - resp, err := c.executeMethod(ctx, "PUT", reqMetadata) + resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { - return ObjectInfo{}, err + return UploadInfo{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { - return ObjectInfo{}, httpRespToErrorResponse(resp, bucketName, objectName) + return UploadInfo{}, httpRespToErrorResponse(resp, bucketName, objectName) } } - var objInfo ObjectInfo - // Trim off the odd double quotes from ETag in the beginning and end. - objInfo.ETag = trimEtag(resp.Header.Get("ETag")) - // A success here means data was written to server successfully. - objInfo.Size = size + // extract lifecycle expiry date and rule ID + expTime, ruleID := amzExpirationToExpiryDateRuleID(resp.Header.Get(amzExpiration)) - // Return here. - return objInfo, nil + return UploadInfo{ + Bucket: bucketName, + Key: objectName, + ETag: trimEtag(resp.Header.Get("ETag")), + VersionID: resp.Header.Get(amzVersionID), + Size: size, + Expiration: expTime, + ExpirationRuleID: ruleID, + }, nil } diff --git a/vendor/github.com/minio/minio-go/v6/api-put-object.go b/vendor/github.com/minio/minio-go/v7/api-put-object.go similarity index 74% rename from vendor/github.com/minio/minio-go/v6/api-put-object.go rename to vendor/github.com/minio/minio-go/v7/api-put-object.go index dbb7c8d53d..b0d5af466b 100644 --- a/vendor/github.com/minio/minio-go/v6/api-put-object.go +++ b/vendor/github.com/minio/minio-go/v7/api-put-object.go @@ -28,11 +28,30 @@ import ( "sort" "time" - "github.com/minio/minio-go/v6/pkg/encrypt" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/encrypt" + "github.com/minio/minio-go/v7/pkg/s3utils" "golang.org/x/net/http/httpguts" ) +// ReplicationStatus represents replication status of object +type ReplicationStatus string + +const ( + // ReplicationStatusPending indicates replication is pending + ReplicationStatusPending ReplicationStatus = "PENDING" + // ReplicationStatusComplete indicates replication completed ok + ReplicationStatusComplete ReplicationStatus = "COMPLETE" + // ReplicationStatusFailed indicates replication failed + ReplicationStatusFailed ReplicationStatus = "FAILED" + // ReplicationStatusReplica indicates object is a replica of a source + ReplicationStatusReplica ReplicationStatus = "REPLICA" +) + +// Empty returns true if no replication status set. +func (r ReplicationStatus) Empty() bool { + return r == "" +} + // PutObjectOptions represents options specified by user for PutObject call type PutObjectOptions struct { UserMetadata map[string]string @@ -43,8 +62,8 @@ type PutObjectOptions struct { ContentDisposition string ContentLanguage string CacheControl string - Mode *RetentionMode - RetainUntilDate *time.Time + Mode RetentionMode + RetainUntilDate time.Time ServerSideEncryption encrypt.ServerSide NumThreads uint StorageClass string @@ -53,6 +72,9 @@ type PutObjectOptions struct { LegalHold LegalHoldStatus SendContentMd5 bool DisableMultipart bool + ReplicationVersionID string + ReplicationStatus ReplicationStatus + ReplicationMTime time.Time } // getNumThreads - gets the number of threads to be used in the multipart @@ -90,11 +112,11 @@ func (opts PutObjectOptions) Header() (header http.Header) { header.Set("Cache-Control", opts.CacheControl) } - if opts.Mode != nil { + if opts.Mode != "" { header.Set(amzLockMode, opts.Mode.String()) } - if opts.RetainUntilDate != nil { + if !opts.RetainUntilDate.IsZero() { header.Set("X-Amz-Object-Lock-Retain-Until-Date", opts.RetainUntilDate.Format(time.RFC3339)) } @@ -114,15 +136,21 @@ func (opts PutObjectOptions) Header() (header http.Header) { header.Set(amzWebsiteRedirectLocation, opts.WebsiteRedirectLocation) } + if !opts.ReplicationStatus.Empty() { + header.Set(amzBucketReplicationStatus, string(opts.ReplicationStatus)) + } + if !opts.ReplicationMTime.IsZero() { + header.Set(minIOBucketReplicationSourceMTime, opts.ReplicationMTime.Format(time.RFC3339)) + } if len(opts.UserTags) != 0 { header.Set(amzTaggingHeader, s3utils.TagEncode(opts.UserTags)) } for k, v := range opts.UserMetadata { - if !isAmzHeader(k) && !isStandardHeader(k) && !isStorageClassHeader(k) { - header.Set("X-Amz-Meta-"+k, v) - } else { + if isAmzHeader(k) || isStandardHeader(k) || isStorageClassHeader(k) { header.Set(k, v) + } else { + header.Set("x-amz-meta-"+k, v) } } return @@ -132,19 +160,17 @@ func (opts PutObjectOptions) Header() (header http.Header) { func (opts PutObjectOptions) validate() (err error) { for k, v := range opts.UserMetadata { if !httpguts.ValidHeaderFieldName(k) || isStandardHeader(k) || isSSEHeader(k) || isStorageClassHeader(k) { - return ErrInvalidArgument(k + " unsupported user defined metadata name") + return errInvalidArgument(k + " unsupported user defined metadata name") } if !httpguts.ValidHeaderFieldValue(v) { - return ErrInvalidArgument(v + " unsupported user defined metadata value") + return errInvalidArgument(v + " unsupported user defined metadata value") } } - if opts.Mode != nil { - if !opts.Mode.IsValid() { - return ErrInvalidArgument(opts.Mode.String() + " unsupported retention mode") - } + if opts.Mode != "" && !opts.Mode.IsValid() { + return errInvalidArgument(opts.Mode.String() + " unsupported retention mode") } if opts.LegalHold != "" && !opts.LegalHold.IsValid() { - return ErrInvalidArgument(opts.LegalHold.String() + " unsupported legal-hold status") + return errInvalidArgument(opts.LegalHold.String() + " unsupported legal-hold status") } return nil } @@ -168,19 +194,24 @@ func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].Part // - For size input as -1 PutObject does a multipart Put operation // until input stream reaches EOF. Maximum object size that can // be uploaded through this operation will be 5TiB. -func (c Client) PutObject(bucketName, objectName string, reader io.Reader, objectSize int64, - opts PutObjectOptions) (n int64, err error) { +func (c Client) PutObject(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64, + opts PutObjectOptions) (info UploadInfo, err error) { if objectSize < 0 && opts.DisableMultipart { - return 0, errors.New("object size must be provided with disable multipart upload") + return UploadInfo{}, errors.New("object size must be provided with disable multipart upload") } - return c.PutObjectWithContext(context.Background(), bucketName, objectName, reader, objectSize, opts) + err = opts.validate() + if err != nil { + return UploadInfo{}, err + } + + return c.putObjectCommon(ctx, bucketName, objectName, reader, objectSize, opts) } -func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { +func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (info UploadInfo, err error) { // Check for largest object size allowed. if size > int64(maxMultipartPutObjectSize) { - return 0, ErrEntityTooLarge(size, maxMultipartPutObjectSize, bucketName, objectName) + return UploadInfo{}, errEntityTooLarge(size, maxMultipartPutObjectSize, bucketName, objectName) } // NOTE: Streaming signature is not supported by GCS. @@ -199,6 +230,7 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri } return c.putObjectMultipart(ctx, bucketName, objectName, reader, size, opts) } + if size < 0 { return c.putObjectMultipartStreamNoLength(ctx, bucketName, objectName, reader, opts) } @@ -210,13 +242,13 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri return c.putObjectMultipartStream(ctx, bucketName, objectName, reader, size, opts) } -func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName, objectName string, reader io.Reader, opts PutObjectOptions) (n int64, err error) { +func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName, objectName string, reader io.Reader, opts PutObjectOptions) (info UploadInfo, err error) { // Input validation. if err = s3utils.CheckValidBucketName(bucketName); err != nil { - return 0, err + return UploadInfo{}, err } if err = s3utils.CheckValidObjectName(objectName); err != nil { - return 0, err + return UploadInfo{}, err } // Total data read and written to server. should be equal to @@ -229,12 +261,12 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName // Calculate the optimal parts info for a given size. totalPartsCount, partSize, _, err := optimalPartInfo(-1, opts.PartSize) if err != nil { - return 0, err + return UploadInfo{}, err } // Initiate a new multipart upload. uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts) if err != nil { - return 0, err + return UploadInfo{}, err } defer func() { @@ -253,12 +285,13 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName buf := make([]byte, partSize) for partNumber <= totalPartsCount { - length, rerr := io.ReadFull(reader, buf) + length, rerr := readFull(reader, buf) if rerr == io.EOF && partNumber > 1 { break } + if rerr != nil && rerr != io.ErrUnexpectedEOF && rerr != io.EOF { - return 0, rerr + return UploadInfo{}, rerr } var md5Base64 string @@ -278,7 +311,7 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName objPart, uerr := c.uploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber, md5Base64, "", int64(length), opts.ServerSideEncryption) if uerr != nil { - return totalUploadedSize, uerr + return UploadInfo{}, uerr } // Save successfully uploaded part metadata. @@ -302,7 +335,7 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName for i := 1; i < partNumber; i++ { part, ok := partsInfo[i] if !ok { - return 0, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", i)) + return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Missing part number %d", i)) } complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ ETag: part.ETag, @@ -312,10 +345,12 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) - if _, err = c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload); err != nil { - return totalUploadedSize, err + + uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) + if err != nil { + return UploadInfo{}, err } - // Return final size. - return totalUploadedSize, nil + uploadInfo.Size = totalUploadedSize + return uploadInfo, nil } diff --git a/vendor/github.com/minio/minio-go/v6/api-remove.go b/vendor/github.com/minio/minio-go/v7/api-remove.go similarity index 70% rename from vendor/github.com/minio/minio-go/v6/api-remove.go rename to vendor/github.com/minio/minio-go/v7/api-remove.go index f2879e8b58..6c2ab78020 100644 --- a/vendor/github.com/minio/minio-go/v6/api-remove.go +++ b/vendor/github.com/minio/minio-go/v7/api-remove.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,20 +25,20 @@ import ( "net/http" "net/url" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // RemoveBucket deletes the bucket name. // // All objects (including all object versions and delete markers). // in the bucket must be deleted before successfully attempting this request. -func (c Client) RemoveBucket(bucketName string) error { +func (c Client) RemoveBucket(ctx context.Context, bucketName string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } // Execute DELETE on bucket. - resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, contentSHA256Hex: emptySHA256Hex, }) @@ -58,19 +58,14 @@ func (c Client) RemoveBucket(bucketName string) error { return nil } -// RemoveObject remove an object from a bucket. -func (c Client) RemoveObject(bucketName, objectName string) error { - return c.RemoveObjectWithOptions(bucketName, objectName, RemoveObjectOptions{}) -} - // RemoveObjectOptions represents options specified by user for RemoveObject call type RemoveObjectOptions struct { GovernanceBypass bool VersionID string } -// RemoveObjectWithOptions removes an object from a bucket. -func (c Client) RemoveObjectWithOptions(bucketName, objectName string, opts RemoveObjectOptions) error { +// RemoveObject removes an object from a bucket. +func (c Client) RemoveObject(ctx context.Context, bucketName, objectName string, opts RemoveObjectOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -95,7 +90,7 @@ func (c Client) RemoveObjectWithOptions(bucketName, objectName string, opts Remo headers.Set(amzBypassGovernance, "true") } // Execute DELETE on objectName. - resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, objectName: objectName, contentSHA256Hex: emptySHA256Hex, @@ -122,22 +117,26 @@ func (c Client) RemoveObjectWithOptions(bucketName, objectName string, opts Remo // RemoveObjectError - container of Multi Delete S3 API error type RemoveObjectError struct { ObjectName string + VersionID string Err error } // generateRemoveMultiObjects - generate the XML request for remove multi objects request -func generateRemoveMultiObjectsRequest(objects []string) []byte { - rmObjects := []deleteObject{} +func generateRemoveMultiObjectsRequest(objects []ObjectInfo) []byte { + delObjects := []deleteObject{} for _, obj := range objects { - rmObjects = append(rmObjects, deleteObject{Key: obj}) + delObjects = append(delObjects, deleteObject{ + Key: obj.Key, + VersionID: obj.VersionID, + }) } - xmlBytes, _ := xml.Marshal(deleteMultiObjects{Objects: rmObjects, Quiet: true}) + xmlBytes, _ := xml.Marshal(deleteMultiObjects{Objects: delObjects, Quiet: true}) return xmlBytes } // processRemoveMultiObjectsResponse - parse the remove multi objects web service // and return the success/failure result status for each object -func processRemoveMultiObjectsResponse(body io.Reader, objects []string, errorCh chan<- RemoveObjectError) { +func processRemoveMultiObjectsResponse(body io.Reader, objects []ObjectInfo, errorCh chan<- RemoveObjectError) { // Parse multi delete XML response rmResult := &deleteMultiObjectsResult{} err := xmlDecoder(body, rmResult) @@ -158,34 +157,15 @@ func processRemoveMultiObjectsResponse(body io.Reader, objects []string, errorCh } } -// RemoveObjectsWithContext - Identical to RemoveObjects call, but accepts context to facilitate request cancellation. -func (c Client) RemoveObjectsWithContext(ctx context.Context, bucketName string, objectsCh <-chan string) <-chan RemoveObjectError { - errorCh := make(chan RemoveObjectError, 1) - - // Validate if bucket name is valid. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - defer close(errorCh) - errorCh <- RemoveObjectError{ - Err: err, - } - return errorCh - } - // Validate objects channel to be properly allocated. - if objectsCh == nil { - defer close(errorCh) - errorCh <- RemoveObjectError{ - Err: ErrInvalidArgument("Objects channel cannot be nil"), - } - return errorCh - } - - go c.removeObjects(ctx, bucketName, objectsCh, errorCh, RemoveObjectsOptions{}) - return errorCh +// RemoveObjectsOptions represents options specified by user for RemoveObjects call +type RemoveObjectsOptions struct { + GovernanceBypass bool } -// RemoveObjectsWithOptionsContext - Identical to RemoveObjects call, but accepts context to -// facilitate request cancellation and options to bypass governance retention -func (c Client) RemoveObjectsWithOptionsContext(ctx context.Context, bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError { +// RemoveObjects removes multiple objects from a bucket while +// it is possible to specify objects versions which are received from +// objectsCh. Remove failures are sent back via error channel. +func (c Client) RemoveObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, opts RemoveObjectsOptions) <-chan RemoveObjectError { errorCh := make(chan RemoveObjectError, 1) // Validate if bucket name is valid. @@ -200,7 +180,7 @@ func (c Client) RemoveObjectsWithOptionsContext(ctx context.Context, bucketName if objectsCh == nil { defer close(errorCh) errorCh <- RemoveObjectError{ - Err: ErrInvalidArgument("Objects channel cannot be nil"), + Err: errInvalidArgument("Objects channel cannot be nil"), } return errorCh } @@ -210,7 +190,7 @@ func (c Client) RemoveObjectsWithOptionsContext(ctx context.Context, bucketName } // Generate and call MultiDelete S3 requests based on entries received from objectsCh -func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh <-chan string, errorCh chan<- RemoveObjectError, opts RemoveObjectsOptions) { +func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, errorCh chan<- RemoveObjectError, opts RemoveObjectsOptions) { maxEntries := 1000 finish := false urlValues := make(url.Values) @@ -225,7 +205,7 @@ func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh break } count := 0 - var batch []string + var batch []ObjectInfo // Try to gather 1000 entries for object := range objectsCh { @@ -270,7 +250,11 @@ func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh } if err != nil { for _, b := range batch { - errorCh <- RemoveObjectError{ObjectName: b, Err: err} + errorCh <- RemoveObjectError{ + ObjectName: b.Key, + VersionID: b.VersionID, + Err: err, + } } continue } @@ -282,27 +266,8 @@ func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh } } -// RemoveObjects removes multiple objects from a bucket. -// The list of objects to remove are received from objectsCh. -// Remove failures are sent back via error channel. -func (c Client) RemoveObjects(bucketName string, objectsCh <-chan string) <-chan RemoveObjectError { - return c.RemoveObjectsWithContext(context.Background(), bucketName, objectsCh) -} - -// RemoveObjectsOptions represents options specified by user for RemoveObjects call -type RemoveObjectsOptions struct { - GovernanceBypass bool -} - -// RemoveObjectsWithOptions removes multiple objects from a bucket. -// The list of objects to remove are received from objectsCh. -// Remove failures are sent back via error channel. -func (c Client) RemoveObjectsWithOptions(bucketName string, objectsCh <-chan string, opts RemoveObjectsOptions) <-chan RemoveObjectError { - return c.RemoveObjectsWithOptionsContext(context.Background(), bucketName, objectsCh, opts) -} - // RemoveIncompleteUpload aborts an partially uploaded object. -func (c Client) RemoveIncompleteUpload(bucketName, objectName string) error { +func (c Client) RemoveIncompleteUpload(ctx context.Context, bucketName, objectName string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -311,14 +276,14 @@ func (c Client) RemoveIncompleteUpload(bucketName, objectName string) error { return err } // Find multipart upload ids of the object to be aborted. - uploadIDs, err := c.findUploadIDs(bucketName, objectName) + uploadIDs, err := c.findUploadIDs(ctx, bucketName, objectName) if err != nil { return err } for _, uploadID := range uploadIDs { // abort incomplete multipart upload, based on the upload id passed. - err := c.abortMultipartUpload(context.Background(), bucketName, objectName, uploadID) + err := c.abortMultipartUpload(ctx, bucketName, objectName, uploadID) if err != nil { return err } @@ -343,7 +308,7 @@ func (c Client) abortMultipartUpload(ctx context.Context, bucketName, objectName urlValues.Set("uploadId", uploadID) // Execute DELETE on multipart upload. - resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, diff --git a/vendor/github.com/minio/minio-go/v6/api-s3-datatypes.go b/vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go similarity index 73% rename from vendor/github.com/minio/minio-go/v6/api-s3-datatypes.go rename to vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go index a6b125522a..ac34457455 100644 --- a/vendor/github.com/minio/minio-go/v6/api-s3-datatypes.go +++ b/vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,9 @@ package minio import ( "encoding/xml" + "errors" + "io" + "reflect" "time" ) @@ -71,6 +74,103 @@ type ListBucketV2Result struct { StartAfter string } +// Version is an element in the list object versions response +type Version struct { + ETag string + IsLatest bool + Key string + LastModified time.Time + Owner Owner + Size int64 + StorageClass string + VersionID string `xml:"VersionId"` + + isDeleteMarker bool +} + +// ListVersionsResult is an element in the list object versions response +type ListVersionsResult struct { + Versions []Version + + CommonPrefixes []CommonPrefix + Name string + Prefix string + Delimiter string + MaxKeys int64 + EncodingType string + IsTruncated bool + KeyMarker string + VersionIDMarker string + NextKeyMarker string + NextVersionIDMarker string +} + +// UnmarshalXML is a custom unmarshal code for the response of ListObjectVersions, the custom +// code will unmarshal and tags and save them in Versions field to +// preserve the lexical order of the listing. +func (l *ListVersionsResult) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err error) { + for { + // Read tokens from the XML document in a stream. + t, err := d.Token() + if err != nil { + if err == io.EOF { + break + } + return err + } + + switch se := t.(type) { + case xml.StartElement: + tagName := se.Name.Local + switch tagName { + case "Name", "Prefix", + "Delimiter", "EncodingType", + "KeyMarker", "VersionIdMarker", + "NextKeyMarker", "NextVersionIdMarker": + var s string + if err = d.DecodeElement(&s, &se); err != nil { + return err + } + v := reflect.ValueOf(l).Elem().FieldByName(tagName) + if v.IsValid() { + v.SetString(s) + } + case "IsTruncated": // bool + var b bool + if err = d.DecodeElement(&b, &se); err != nil { + return err + } + l.IsTruncated = b + case "MaxKeys": // int64 + var i int64 + if err = d.DecodeElement(&i, &se); err != nil { + return err + } + l.MaxKeys = i + case "CommonPrefixes": + var cp CommonPrefix + if err = d.DecodeElement(&cp, &se); err != nil { + return err + } + l.CommonPrefixes = append(l.CommonPrefixes, cp) + case "DeleteMarker", "Version": + var v Version + if err = d.DecodeElement(&v, &se); err != nil { + return err + } + if tagName == "DeleteMarker" { + v.isDeleteMarker = true + } + l.Versions = append(l.Versions, v) + default: + return errors.New("unrecognized option:" + tagName) + } + + } + } + return nil +} + // ListBucketResult container for listObjects response. type ListBucketResult struct { // A response can contain CommonPrefixes only if you have diff --git a/vendor/github.com/minio/minio-go/v6/api-select.go b/vendor/github.com/minio/minio-go/v7/api-select.go similarity index 98% rename from vendor/github.com/minio/minio-go/v6/api-select.go rename to vendor/github.com/minio/minio-go/v7/api-select.go index f9f4dda58a..e35cf02bf3 100644 --- a/vendor/github.com/minio/minio-go/v6/api-select.go +++ b/vendor/github.com/minio/minio-go/v7/api-select.go @@ -31,8 +31,8 @@ import ( "net/url" "strings" - "github.com/minio/minio-go/v6/pkg/encrypt" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/encrypt" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // CSVFileHeaderInfo - is the parameter for whether to utilize headers. @@ -450,7 +450,7 @@ func (c Client) SelectObjectContent(ctx context.Context, bucketName, objectName urlValues.Set("select-type", "2") // Execute POST on bucket/object. - resp, err := c.executeMethod(ctx, "POST", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodPost, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, @@ -709,7 +709,7 @@ func extractString(source io.Reader, lenBytes int) (string, error) { // extractUint32 extracts a 4 byte integer from the byte array. func extractUint32(r io.Reader) (uint32, error) { buf := make([]byte, 4) - _, err := io.ReadFull(r, buf) + _, err := readFull(r, buf) if err != nil { return 0, err } @@ -719,7 +719,7 @@ func extractUint32(r io.Reader) (uint32, error) { // extractUint16 extracts a 2 byte integer from the byte array. func extractUint16(r io.Reader) (uint16, error) { buf := make([]byte, 2) - _, err := io.ReadFull(r, buf) + _, err := readFull(r, buf) if err != nil { return 0, err } @@ -729,7 +729,7 @@ func extractUint16(r io.Reader) (uint16, error) { // extractUint8 extracts a 1 byte integer from the byte array. func extractUint8(r io.Reader) (uint8, error) { buf := make([]byte, 1) - _, err := io.ReadFull(r, buf) + _, err := readFull(r, buf) if err != nil { return 0, err } diff --git a/vendor/github.com/minio/minio-go/v6/api-stat.go b/vendor/github.com/minio/minio-go/v7/api-stat.go similarity index 70% rename from vendor/github.com/minio/minio-go/v6/api-stat.go rename to vendor/github.com/minio/minio-go/v7/api-stat.go index 5da981b6c3..ea9c30970a 100644 --- a/vendor/github.com/minio/minio-go/v6/api-stat.go +++ b/vendor/github.com/minio/minio-go/v7/api-stat.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,25 +20,21 @@ package minio import ( "context" "net/http" + "net/url" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) -// BucketExists verify if bucket exists and you have permission to access it. -func (c Client) BucketExists(bucketName string) (bool, error) { - return c.BucketExistsWithContext(context.Background(), bucketName) -} - -// BucketExistsWithContext verify if bucket exists and you have permission to access it. Allows for a Context to +// BucketExists verifies if bucket exists and you have permission to access it. Allows for a Context to // control cancellations and timeouts. -func (c Client) BucketExistsWithContext(ctx context.Context, bucketName string) (bool, error) { +func (c Client) BucketExists(ctx context.Context, bucketName string) (bool, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return false, err } // Execute HEAD on bucketName. - resp, err := c.executeMethod(ctx, "HEAD", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodHead, requestMetadata{ bucketName: bucketName, contentSHA256Hex: emptySHA256Hex, }) @@ -62,13 +58,7 @@ func (c Client) BucketExistsWithContext(ctx context.Context, bucketName string) } // StatObject verifies if object exists and you have permission to access. -func (c Client) StatObject(bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { - return c.StatObjectWithContext(context.Background(), bucketName, objectName, opts) -} - -// StatObjectWithContext verifies if object exists and you have permission to access with a context to control -// cancellations and timeouts. -func (c Client) StatObjectWithContext(ctx context.Context, bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { +func (c Client) StatObject(ctx context.Context, bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ObjectInfo{}, err @@ -89,10 +79,16 @@ func (c Client) statObject(ctx context.Context, bucketName, objectName string, o return ObjectInfo{}, err } + urlValues := make(url.Values) + if opts.VersionID != "" { + urlValues.Set("versionId", opts.VersionID) + } + // Execute HEAD on objectName. - resp, err := c.executeMethod(ctx, "HEAD", requestMetadata{ + resp, err := c.executeMethod(ctx, http.MethodHead, requestMetadata{ bucketName: bucketName, objectName: objectName, + queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, customHeader: opts.Header(), }) diff --git a/vendor/github.com/minio/minio-go/v6/api.go b/vendor/github.com/minio/minio-go/v7/api.go similarity index 85% rename from vendor/github.com/minio/minio-go/v6/api.go rename to vendor/github.com/minio/minio-go/v7/api.go index 22778dafa8..09b1e0bb19 100644 --- a/vendor/github.com/minio/minio-go/v6/api.go +++ b/vendor/github.com/minio/minio-go/v7/api.go @@ -37,9 +37,9 @@ import ( "time" md5simd "github.com/minio/md5-simd" - "github.com/minio/minio-go/v6/pkg/credentials" - "github.com/minio/minio-go/v6/pkg/s3utils" - "github.com/minio/minio-go/v6/pkg/signer" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/signer" "golang.org/x/net/publicsuffix" ) @@ -96,6 +96,7 @@ type Client struct { type Options struct { Creds *credentials.Credentials Secure bool + Transport http.RoundTripper Region string BucketLookup BucketLookupType @@ -107,7 +108,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.57" + libraryVersion = "v7.0.3" ) // User Agent should always following the below style. @@ -129,43 +130,12 @@ const ( BucketLookupPath ) -// NewV2 - instantiate minio client with Amazon S3 signature version -// '2' compatibility. -func NewV2(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { - clnt, err := privateNew(endpoint, Options{ - Creds: credentials.NewStaticV2(accessKeyID, secretAccessKey, ""), - Secure: secure, - BucketLookup: BucketLookupAuto, - }) - if err != nil { - return nil, err +// New - instantiate minio client with options +func New(endpoint string, opts *Options) (*Client, error) { + if opts == nil { + return nil, errors.New("no options provided") } - clnt.overrideSignerType = credentials.SignatureV2 - return clnt, nil -} - -// NewV4 - instantiate minio client with Amazon S3 signature version -// '4' compatibility. -func NewV4(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { - clnt, err := privateNew(endpoint, Options{ - Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), - Secure: secure, - BucketLookup: BucketLookupAuto, - }) - if err != nil { - return nil, err - } - clnt.overrideSignerType = credentials.SignatureV4 - return clnt, nil -} - -// New - instantiate minio client, adds automatic verification of signature. -func New(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { - clnt, err := privateNew(endpoint, Options{ - Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), - Secure: secure, - BucketLookup: BucketLookupAuto, - }) + clnt, err := privateNew(endpoint, opts) if err != nil { return nil, err } @@ -177,42 +147,10 @@ func New(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Client, e if s3utils.IsAmazonEndpoint(*clnt.endpointURL) { clnt.overrideSignerType = credentials.SignatureV4 } + return clnt, nil } -// NewWithCredentials - instantiate minio client with credentials provider -// for retrieving credentials from various credentials provider such as -// IAM, File, Env etc. -func NewWithCredentials(endpoint string, creds *credentials.Credentials, secure bool, region string) (*Client, error) { - return privateNew(endpoint, Options{ - Creds: creds, - Secure: secure, - BucketLookup: BucketLookupAuto, - Region: region, - }) -} - -// NewWithRegion - instantiate minio client, with region configured. Unlike New(), -// NewWithRegion avoids bucket-location lookup operations and it is slightly faster. -// Use this function when if your application deals with single region. -func NewWithRegion(endpoint, accessKeyID, secretAccessKey string, secure bool, region string) (*Client, error) { - creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "") - return privateNew(endpoint, Options{ - Creds: creds, - Secure: secure, - Region: region, - BucketLookup: BucketLookupAuto, - }) -} - -// NewWithOptions - instantiate minio client with options -func NewWithOptions(endpoint string, opts *Options) (*Client, error) { - if opts == nil { - return nil, errors.New("no options provided") - } - return privateNew(endpoint, *opts) -} - // EndpointURL returns the URL of the S3 endpoint. func (c *Client) EndpointURL() *url.URL { endpoint := *c.endpointURL // copy to prevent callers from modifying internal state @@ -302,7 +240,7 @@ func (c *Client) redirectHeaders(req *http.Request, via []*http.Request) error { return nil } -func privateNew(endpoint string, opts Options) (*Client, error) { +func privateNew(endpoint string, opts *Options) (*Client, error) { // construct endpoint. endpointURL, err := getEndpointURL(endpoint, opts.Secure) if err != nil { @@ -328,9 +266,12 @@ func privateNew(endpoint string, opts Options) (*Client, error) { // Save endpoint URL, user agent for future uses. clnt.endpointURL = endpointURL - transport, err := DefaultTransport(opts.Secure) - if err != nil { - return nil, err + transport := opts.Transport + if transport == nil { + transport, err = DefaultTransport(opts.Secure) + if err != nil { + return nil, err + } } // Instantiate http client and bucket location cache. @@ -377,27 +318,6 @@ func (c *Client) SetAppInfo(appName string, appVersion string) { } } -// SetCustomTransport - set new custom transport. -func (c *Client) SetCustomTransport(customHTTPTransport http.RoundTripper) { - // Set this to override default transport - // ``http.DefaultTransport``. - // - // This transport is usually needed for debugging OR to add your - // own custom TLS certificates on the client transport, for custom - // CA's and certs which are not part of standard certificate - // authority follow this example :- - // - // tr := &http.Transport{ - // TLSClientConfig: &tls.Config{RootCAs: pool}, - // DisableCompression: true, - // } - // api.SetCustomTransport(tr) - // - if c.httpClient != nil { - c.httpClient.Transport = customHTTPTransport - } -} - // TraceOn - enable HTTP tracing. func (c *Client) TraceOn(outputStream io.Writer) { // if outputStream is nil then default to os.Stdout. @@ -567,7 +487,7 @@ func (c Client) do(req *http.Request) (*http.Response, error) { // Response cannot be non-nil, report error if thats the case. if resp == nil { msg := "Response is empty. " + reportIssue - return nil, ErrInvalidArgument(msg) + return nil, errInvalidArgument(msg) } // If trace is enabled, dump http request and response, @@ -593,7 +513,7 @@ var successStatus = []int{ // request upon any error up to maxRetries attempts in a binomially // delayed manner using a standard back off algorithm. func (c Client) executeMethod(ctx context.Context, method string, metadata requestMetadata) (res *http.Response, err error) { - var isRetryable bool // Indicates if request can be retried. + var retryable bool // Indicates if request can be retried. var bodySeeker io.Seeker // Extracted seeker from io.Reader. var reqRetry = MaxRetry // Indicates how many times we can retry the request @@ -606,13 +526,13 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque if metadata.contentBody != nil { // Check if body is seekable then it is retryable. - bodySeeker, isRetryable = metadata.contentBody.(io.Seeker) + bodySeeker, retryable = metadata.contentBody.(io.Seeker) switch bodySeeker { case os.Stdin, os.Stdout, os.Stderr: - isRetryable = false + retryable = false } // Retry only when reader is seekable - if !isRetryable { + if !retryable { reqRetry = 1 } @@ -639,7 +559,7 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque // error until maxRetries have been exhausted, retry attempts are // performed after waiting for a given period of time in a // binomial fashion. - if isRetryable { + if retryable { // Seek back to beginning for each attempt. if _, err = bodySeeker.Seek(0, 0); err != nil { // If seek failed, no need to retry. @@ -649,7 +569,7 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque // Instantiate a new request. var req *http.Request - req, err = c.newRequest(method, metadata) + req, err = c.newRequest(ctx, method, metadata) if err != nil { errResponse := ToErrorResponse(err) if isS3CodeRetryable(errResponse.Code) { @@ -756,17 +676,17 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque } // newRequest - instantiate a new HTTP request for a given method. -func (c Client) newRequest(method string, metadata requestMetadata) (req *http.Request, err error) { +func (c Client) newRequest(ctx context.Context, method string, metadata requestMetadata) (req *http.Request, err error) { // If no method is supplied default to 'POST'. if method == "" { - method = "POST" + method = http.MethodPost } location := metadata.bucketLocation if location == "" { if metadata.bucketName != "" { // Gather location only if bucketName is present. - location, err = c.getBucketLocation(metadata.bucketName) + location, err = c.getBucketLocation(ctx, metadata.bucketName) if err != nil { return nil, err } @@ -779,7 +699,7 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R // Look if target url supports virtual host. // We explicitly disallow MakeBucket calls to not use virtual DNS style, // since the resolution may fail. - isMakeBucket := (metadata.objectName == "" && method == "PUT" && len(metadata.queryValues) == 0) + isMakeBucket := (metadata.objectName == "" && method == http.MethodPut && len(metadata.queryValues) == 0) isVirtualHost := c.isVirtualHostStyleRequest(*c.endpointURL, metadata.bucketName) && !isMakeBucket // Construct a new target URL. @@ -822,7 +742,7 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R // Generate presign url if needed, return right here. if metadata.expires != 0 && metadata.presignURL { if signerType.IsAnonymous() { - return nil, ErrInvalidArgument("Presigned URLs cannot be generated with anonymous credentials.") + return nil, errInvalidArgument("Presigned URLs cannot be generated with anonymous credentials.") } if signerType.IsV2() { // Presign URL with signature v2. @@ -873,7 +793,7 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R case signerType.IsV2(): // Add signature version '2' authorization header. req = signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualHost) - case metadata.objectName != "" && metadata.queryValues == nil && method == "PUT" && metadata.customHeader.Get("X-Amz-Copy-Source") == "" && !c.secure: + case metadata.objectName != "" && metadata.queryValues == nil && method == http.MethodPut && metadata.customHeader.Get("X-Amz-Copy-Source") == "" && !c.secure: // Streaming signature is used by default for a PUT object request. Additionally we also // look if the initialized client is secure, if yes then we don't need to perform // streaming signature. @@ -912,7 +832,7 @@ func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, isV // http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html // Disable transfer acceleration for non-compliant bucket names. if strings.Contains(bucketName, ".") { - return nil, ErrTransferAccelerationBucket(bucketName) + return nil, errTransferAccelerationBucket(bucketName) } // If transfer acceleration is requested set new host. // For more details about enabling transfer acceleration read here. diff --git a/vendor/github.com/minio/minio-go/v6/bucket-cache.go b/vendor/github.com/minio/minio-go/v7/bucket-cache.go similarity index 93% rename from vendor/github.com/minio/minio-go/v6/bucket-cache.go rename to vendor/github.com/minio/minio-go/v7/bucket-cache.go index 305367bb5a..7d485a6b1c 100644 --- a/vendor/github.com/minio/minio-go/v6/bucket-cache.go +++ b/vendor/github.com/minio/minio-go/v7/bucket-cache.go @@ -18,15 +18,16 @@ package minio import ( + "context" "net" "net/http" "net/url" "path" "sync" - "github.com/minio/minio-go/v6/pkg/credentials" - "github.com/minio/minio-go/v6/pkg/s3utils" - "github.com/minio/minio-go/v6/pkg/signer" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/signer" ) // bucketLocationCache - Provides simple mechanism to hold bucket @@ -72,16 +73,16 @@ func (r *bucketLocationCache) Delete(bucketName string) { // GetBucketLocation - get location for the bucket name from location cache, if not // fetch freshly by making a new request. -func (c Client) GetBucketLocation(bucketName string) (string, error) { +func (c Client) GetBucketLocation(ctx context.Context, bucketName string) (string, error) { if err := s3utils.CheckValidBucketName(bucketName); err != nil { return "", err } - return c.getBucketLocation(bucketName) + return c.getBucketLocation(ctx, bucketName) } // getBucketLocation - Get location for the bucketName from location map cache, if not // fetch freshly by making a new request. -func (c Client) getBucketLocation(bucketName string) (string, error) { +func (c Client) getBucketLocation(ctx context.Context, bucketName string) (string, error) { if err := s3utils.CheckValidBucketName(bucketName); err != nil { return "", err } @@ -197,7 +198,7 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro } // Get a new HTTP request for the method. - req, err := http.NewRequest("GET", urlStr, nil) + req, err := http.NewRequest(http.MethodGet, urlStr, nil) if err != nil { return nil, err } diff --git a/vendor/github.com/minio/minio-go/v7/code_of_conduct.md b/vendor/github.com/minio/minio-go/v7/code_of_conduct.md new file mode 100644 index 0000000000..cb232c3c69 --- /dev/null +++ b/vendor/github.com/minio/minio-go/v7/code_of_conduct.md @@ -0,0 +1,80 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior, in compliance with the +licensing terms applying to the Project developments. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. However, these actions shall respect the +licensing terms of the Project Developments that will always supersede such +Code of Conduct. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at dev@min.io. The project team +will review and investigate all complaints, and will respond in a way that it deems +appropriate to the circumstances. The project team is obligated to maintain +confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [http://contributor-covenant.org/version/1/4][version] + +This version includes a clarification to ensure that the code of conduct is in +compliance with the free software licensing terms of the project. + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/vendor/github.com/minio/minio-go/v6/constants.go b/vendor/github.com/minio/minio-go/v7/constants.go similarity index 87% rename from vendor/github.com/minio/minio-go/v6/constants.go rename to vendor/github.com/minio/minio-go/v7/constants.go index 78ba5a114e..e1da72e79f 100644 --- a/vendor/github.com/minio/minio-go/v6/constants.go +++ b/vendor/github.com/minio/minio-go/v7/constants.go @@ -66,6 +66,11 @@ const ( amzTaggingHeader = "X-Amz-Tagging" amzTaggingHeaderDirective = "X-Amz-Tagging-Directive" + amzVersionID = "X-Amz-Version-Id" + amzTaggingCount = "X-Amz-Tagging-Count" + amzExpiration = "X-Amz-Expiration" + amzReplicationStatus = "X-Amz-Replication-Status" + // Object legal hold header amzLegalHoldHeader = "X-Amz-Object-Lock-Legal-Hold" @@ -73,4 +78,9 @@ const ( amzLockMode = "X-Amz-Object-Lock-Mode" amzLockRetainUntil = "X-Amz-Object-Lock-Retain-Until-Date" amzBypassGovernance = "X-Amz-Bypass-Governance-Retention" + + // Replication status + amzBucketReplicationStatus = "X-Amz-Replication-Status" + // Minio specific Replication extension + minIOBucketReplicationSourceMTime = "X-Minio-Source-Mtime" ) diff --git a/vendor/github.com/minio/minio-go/v7/core.go b/vendor/github.com/minio/minio-go/v7/core.go new file mode 100644 index 0000000000..954e1a1bb5 --- /dev/null +++ b/vendor/github.com/minio/minio-go/v7/core.go @@ -0,0 +1,132 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "io" + "net/http" + + "github.com/minio/minio-go/v7/pkg/encrypt" +) + +// Core - Inherits Client and adds new methods to expose the low level S3 APIs. +type Core struct { + *Client +} + +// NewCore - Returns new initialized a Core client, this CoreClient should be +// only used under special conditions such as need to access lower primitives +// and being able to use them to write your own wrappers. +func NewCore(endpoint string, opts *Options) (*Core, error) { + var s3Client Core + client, err := New(endpoint, opts) + if err != nil { + return nil, err + } + s3Client.Client = client + return &s3Client, nil +} + +// ListObjects - List all the objects at a prefix, optionally with marker and delimiter +// you can further filter the results. +func (c Core) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListBucketResult, err error) { + return c.listObjectsQuery(context.Background(), bucket, prefix, marker, delimiter, maxKeys) +} + +// ListObjectsV2 - Lists all the objects at a prefix, similar to ListObjects() but uses +// continuationToken instead of marker to support iteration over the results. +func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int) (ListBucketV2Result, error) { + return c.listObjectsV2Query(context.Background(), bucketName, objectPrefix, continuationToken, fetchOwner, false, delimiter, maxkeys) +} + +// CopyObject - copies an object from source object to destination object on server side. +func (c Core) CopyObject(ctx context.Context, sourceBucket, sourceObject, destBucket, destObject string, metadata map[string]string) (ObjectInfo, error) { + return c.copyObjectDo(ctx, sourceBucket, sourceObject, destBucket, destObject, metadata) +} + +// CopyObjectPart - creates a part in a multipart upload by copying (a +// part of) an existing object. +func (c Core) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, + partID int, startOffset, length int64, metadata map[string]string) (p CompletePart, err error) { + + return c.copyObjectPartDo(ctx, srcBucket, srcObject, destBucket, destObject, uploadID, + partID, startOffset, length, metadata) +} + +// PutObject - Upload object. Uploads using single PUT call. +func (c Core) PutObject(ctx context.Context, bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, opts PutObjectOptions) (UploadInfo, error) { + return c.putObjectDo(ctx, bucket, object, data, md5Base64, sha256Hex, size, opts) +} + +// NewMultipartUpload - Initiates new multipart upload and returns the new uploadID. +func (c Core) NewMultipartUpload(ctx context.Context, bucket, object string, opts PutObjectOptions) (uploadID string, err error) { + result, err := c.initiateMultipartUpload(ctx, bucket, object, opts) + return result.UploadID, err +} + +// ListMultipartUploads - List incomplete uploads. +func (c Core) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartUploadsResult, err error) { + return c.listMultipartUploadsQuery(ctx, bucket, keyMarker, uploadIDMarker, prefix, delimiter, maxUploads) +} + +// PutObjectPart - Upload an object part. +func (c Core) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string, sse encrypt.ServerSide) (ObjectPart, error) { + return c.uploadPart(ctx, bucket, object, uploadID, data, partID, md5Base64, sha256Hex, size, sse) +} + +// ListObjectParts - List uploaded parts of an incomplete upload.x +func (c Core) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListObjectPartsResult, err error) { + return c.listObjectPartsQuery(ctx, bucket, object, uploadID, partNumberMarker, maxParts) +} + +// CompleteMultipartUpload - Concatenate uploaded parts and commit to an object. +func (c Core) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, parts []CompletePart) (string, error) { + res, err := c.completeMultipartUpload(ctx, bucket, object, uploadID, completeMultipartUpload{ + Parts: parts, + }) + return res.ETag, err +} + +// AbortMultipartUpload - Abort an incomplete upload. +func (c Core) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error { + return c.abortMultipartUpload(ctx, bucket, object, uploadID) +} + +// GetBucketPolicy - fetches bucket access policy for a given bucket. +func (c Core) GetBucketPolicy(ctx context.Context, bucket string) (string, error) { + return c.getBucketPolicy(ctx, bucket) +} + +// PutBucketPolicy - applies a new bucket access policy for a given bucket. +func (c Core) PutBucketPolicy(ctx context.Context, bucket, bucketPolicy string) error { + return c.putBucketPolicy(ctx, bucket, bucketPolicy) +} + +// GetObject is a lower level API implemented to support reading +// partial objects and also downloading objects with special conditions +// matching etag, modtime etc. +func (c Core) GetObject(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, http.Header, error) { + return c.getObject(ctx, bucketName, objectName, opts) +} + +// StatObject is a lower level API implemented to support special +// conditions matching etag, modtime on a request. +func (c Core) StatObject(ctx context.Context, bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { + return c.statObject(ctx, bucketName, objectName, opts) +} diff --git a/vendor/github.com/minio/minio-go/v7/go.mod b/vendor/github.com/minio/minio-go/v7/go.mod new file mode 100644 index 0000000000..448b4bbf17 --- /dev/null +++ b/vendor/github.com/minio/minio-go/v7/go.mod @@ -0,0 +1,25 @@ +module github.com/minio/minio-go/v7 + +go 1.12 + +require ( + github.com/google/uuid v1.1.1 + github.com/json-iterator/go v1.1.10 + github.com/klauspost/cpuid v1.3.1 // indirect + github.com/kr/pretty v0.1.0 // indirect + github.com/minio/md5-simd v1.1.0 + github.com/minio/sha256-simd v0.1.1 + github.com/mitchellh/go-homedir v1.1.0 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/rs/xid v1.2.1 + github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect + github.com/stretchr/testify v1.4.0 // indirect + golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 + golang.org/x/net v0.0.0-20200707034311-ab3426394381 + golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect + golang.org/x/text v0.3.3 // indirect + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect + gopkg.in/ini.v1 v1.57.0 + gopkg.in/yaml.v2 v2.2.8 // indirect +) diff --git a/vendor/github.com/minio/minio-go/v6/go.sum b/vendor/github.com/minio/minio-go/v7/go.sum similarity index 52% rename from vendor/github.com/minio/minio-go/v6/go.sum rename to vendor/github.com/minio/minio-go/v7/go.sum index f158252bb6..1858b6ff38 100644 --- a/vendor/github.com/minio/minio-go/v6/go.sum +++ b/vendor/github.com/minio/minio-go/v7/go.sum @@ -1,20 +1,24 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/minio/md5-simd v1.0.1 h1:tj/FH8APTKxIkOGUX2YGAVJVXXC3AJ5T2SkHoT/dUFI= -github.com/minio/md5-simd v1.0.1/go.mod h1:EhdyA+Dr0guvfyc8d6yrgs9YzHGfaI+YIjA6gt/7mJk= +github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= +github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= @@ -23,35 +27,50 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= -github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= -github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= +github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= -golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= -gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/vendor/github.com/minio/minio-go/v6/hook-reader.go b/vendor/github.com/minio/minio-go/v7/hook-reader.go similarity index 100% rename from vendor/github.com/minio/minio-go/v6/hook-reader.go rename to vendor/github.com/minio/minio-go/v7/hook-reader.go diff --git a/vendor/github.com/minio/minio-go/v6/pkg/credentials/assume_role.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/assume_role.go similarity index 98% rename from vendor/github.com/minio/minio-go/v6/pkg/credentials/assume_role.go rename to vendor/github.com/minio/minio-go/v7/pkg/credentials/assume_role.go index f3702d8d67..cc88a9ab67 100644 --- a/vendor/github.com/minio/minio-go/v6/pkg/credentials/assume_role.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/assume_role.go @@ -29,7 +29,7 @@ import ( "strings" "time" - "github.com/minio/minio-go/v6/pkg/signer" + "github.com/minio/minio-go/v7/pkg/signer" sha256 "github.com/minio/sha256-simd" ) @@ -170,7 +170,7 @@ func getAssumeRoleCredentials(clnt *http.Client, endpoint string, opts STSAssume } postBody.Seek(0, 0) - req, err := http.NewRequest("POST", u.String(), postBody) + req, err := http.NewRequest(http.MethodPost, u.String(), postBody) if err != nil { return AssumeRoleResponse{}, err } diff --git a/vendor/github.com/minio/minio-go/v6/pkg/credentials/chain.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/chain.go similarity index 100% rename from vendor/github.com/minio/minio-go/v6/pkg/credentials/chain.go rename to vendor/github.com/minio/minio-go/v7/pkg/credentials/chain.go diff --git a/vendor/github.com/minio/minio-go/v6/pkg/credentials/config.json.sample b/vendor/github.com/minio/minio-go/v7/pkg/credentials/config.json.sample similarity index 100% rename from vendor/github.com/minio/minio-go/v6/pkg/credentials/config.json.sample rename to vendor/github.com/minio/minio-go/v7/pkg/credentials/config.json.sample diff --git a/vendor/github.com/minio/minio-go/v6/pkg/credentials/credentials.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/credentials.go similarity index 100% rename from vendor/github.com/minio/minio-go/v6/pkg/credentials/credentials.go rename to vendor/github.com/minio/minio-go/v7/pkg/credentials/credentials.go diff --git a/vendor/github.com/minio/minio-go/v6/pkg/credentials/credentials.sample b/vendor/github.com/minio/minio-go/v7/pkg/credentials/credentials.sample similarity index 100% rename from vendor/github.com/minio/minio-go/v6/pkg/credentials/credentials.sample rename to vendor/github.com/minio/minio-go/v7/pkg/credentials/credentials.sample diff --git a/vendor/github.com/minio/minio-go/v6/pkg/credentials/doc.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/doc.go similarity index 100% rename from vendor/github.com/minio/minio-go/v6/pkg/credentials/doc.go rename to vendor/github.com/minio/minio-go/v7/pkg/credentials/doc.go diff --git a/vendor/github.com/minio/minio-go/v6/pkg/credentials/env_aws.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/env_aws.go similarity index 100% rename from vendor/github.com/minio/minio-go/v6/pkg/credentials/env_aws.go rename to vendor/github.com/minio/minio-go/v7/pkg/credentials/env_aws.go diff --git a/vendor/github.com/minio/minio-go/v6/pkg/credentials/env_minio.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/env_minio.go similarity index 100% rename from vendor/github.com/minio/minio-go/v6/pkg/credentials/env_minio.go rename to vendor/github.com/minio/minio-go/v7/pkg/credentials/env_minio.go diff --git a/vendor/github.com/minio/minio-go/v6/pkg/credentials/file_aws_credentials.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/file_aws_credentials.go similarity index 100% rename from vendor/github.com/minio/minio-go/v6/pkg/credentials/file_aws_credentials.go rename to vendor/github.com/minio/minio-go/v7/pkg/credentials/file_aws_credentials.go diff --git a/vendor/github.com/minio/minio-go/v6/pkg/credentials/file_minio_client.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/file_minio_client.go similarity index 100% rename from vendor/github.com/minio/minio-go/v6/pkg/credentials/file_minio_client.go rename to vendor/github.com/minio/minio-go/v7/pkg/credentials/file_minio_client.go diff --git a/vendor/github.com/minio/minio-go/v6/pkg/credentials/iam_aws.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go similarity index 96% rename from vendor/github.com/minio/minio-go/v6/pkg/credentials/iam_aws.go rename to vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go index a7eb8cb9a0..ceeab84ddb 100644 --- a/vendor/github.com/minio/minio-go/v6/pkg/credentials/iam_aws.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go @@ -104,7 +104,11 @@ func (m *IAM) Retrieve() (Value, error) { }, } - return creds.Retrieve() + stsWebIdentityCreds, err := creds.Retrieve() + if err == nil { + m.SetExpiration(creds.Expiration(), DefaultExpiryWindow) + } + return stsWebIdentityCreds, err case len(os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI")) > 0: if len(endpoint) == 0 { @@ -186,7 +190,7 @@ func getIAMRoleURL(endpoint string) (*url.URL, error) { // or there is an error making or receiving the request. // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html func listRoleNames(client *http.Client, u *url.URL) ([]string, error) { - req, err := http.NewRequest("GET", u.String(), nil) + req, err := http.NewRequest(http.MethodGet, u.String(), nil) if err != nil { return nil, err } @@ -213,7 +217,7 @@ func listRoleNames(client *http.Client, u *url.URL) ([]string, error) { } func getEcsTaskCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody, error) { - req, err := http.NewRequest("GET", endpoint, nil) + req, err := http.NewRequest(http.MethodGet, endpoint, nil) if err != nil { return ec2RoleCredRespBody{}, err } @@ -269,7 +273,7 @@ func getCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody, // $ curl http://169.254.169.254/latest/meta-data/iam/security-credentials/s3access // u.Path = path.Join(u.Path, roleName) - req, err := http.NewRequest("GET", u.String(), nil) + req, err := http.NewRequest(http.MethodGet, u.String(), nil) if err != nil { return ec2RoleCredRespBody{}, err } diff --git a/vendor/github.com/minio/minio-go/v6/pkg/credentials/signature-type.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/signature-type.go similarity index 100% rename from vendor/github.com/minio/minio-go/v6/pkg/credentials/signature-type.go rename to vendor/github.com/minio/minio-go/v7/pkg/credentials/signature-type.go diff --git a/vendor/github.com/minio/minio-go/v6/pkg/credentials/static.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/static.go similarity index 100% rename from vendor/github.com/minio/minio-go/v6/pkg/credentials/static.go rename to vendor/github.com/minio/minio-go/v7/pkg/credentials/static.go diff --git a/vendor/github.com/minio/minio-go/v6/pkg/credentials/sts_client_grants.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_client_grants.go similarity index 98% rename from vendor/github.com/minio/minio-go/v6/pkg/credentials/sts_client_grants.go rename to vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_client_grants.go index 03134c3d2c..e89d5d4d87 100644 --- a/vendor/github.com/minio/minio-go/v6/pkg/credentials/sts_client_grants.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_client_grants.go @@ -122,7 +122,7 @@ func getClientGrantsCredentials(clnt *http.Client, endpoint string, } u.RawQuery = v.Encode() - req, err := http.NewRequest("POST", u.String(), nil) + req, err := http.NewRequest(http.MethodPost, u.String(), nil) if err != nil { return AssumeRoleWithClientGrantsResponse{}, err } diff --git a/vendor/github.com/minio/minio-go/v6/pkg/credentials/sts_ldap_identity.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go similarity index 98% rename from vendor/github.com/minio/minio-go/v6/pkg/credentials/sts_ldap_identity.go rename to vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go index b72ac061c3..abbf616413 100644 --- a/vendor/github.com/minio/minio-go/v6/pkg/credentials/sts_ldap_identity.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go @@ -85,7 +85,7 @@ func (k *LDAPIdentity) Retrieve() (value Value, err error) { u.RawQuery = v.Encode() - req, kerr := http.NewRequest("POST", u.String(), nil) + req, kerr := http.NewRequest(http.MethodPost, u.String(), nil) if kerr != nil { err = kerr return diff --git a/vendor/github.com/minio/minio-go/v6/pkg/credentials/sts_web_identity.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go similarity index 96% rename from vendor/github.com/minio/minio-go/v6/pkg/credentials/sts_web_identity.go rename to vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go index 216f48ac37..5a5f6405ee 100644 --- a/vendor/github.com/minio/minio-go/v6/pkg/credentials/sts_web_identity.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go @@ -133,7 +133,7 @@ func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSession u.RawQuery = v.Encode() - req, err := http.NewRequest("POST", u.String(), nil) + req, err := http.NewRequest(http.MethodPost, u.String(), nil) if err != nil { return AssumeRoleWithWebIdentityResponse{}, err } @@ -174,3 +174,8 @@ func (m *STSWebIdentity) Retrieve() (Value, error) { SignerType: SignatureV4, }, nil } + +// Expiration returns the expiration time of the credentials +func (m *STSWebIdentity) Expiration() time.Time { + return m.expiration +} diff --git a/vendor/github.com/minio/minio-go/v6/pkg/encrypt/server-side.go b/vendor/github.com/minio/minio-go/v7/pkg/encrypt/server-side.go similarity index 100% rename from vendor/github.com/minio/minio-go/v6/pkg/encrypt/server-side.go rename to vendor/github.com/minio/minio-go/v7/pkg/encrypt/server-side.go diff --git a/vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go b/vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go new file mode 100644 index 0000000000..3493003f7d --- /dev/null +++ b/vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go @@ -0,0 +1,282 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package lifecycle contains all the lifecycle related data types and marshallers. +package lifecycle + +import ( + "encoding/xml" + "time" +) + +// AbortIncompleteMultipartUpload structure, not supported yet on MinIO +type AbortIncompleteMultipartUpload struct { + XMLName xml.Name `xml:"AbortIncompleteMultipartUpload,omitempty" json:"-"` + DaysAfterInitiation ExpirationDays `xml:"DaysAfterInitiation,omitempty" json:"DaysAfterInitiation,omitempty"` +} + +// IsDaysNull returns true if days field is null +func (n AbortIncompleteMultipartUpload) IsDaysNull() bool { + return n.DaysAfterInitiation == ExpirationDays(0) +} + +// MarshalXML if days after initiation is set to non-zero value +func (n AbortIncompleteMultipartUpload) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if n.IsDaysNull() { + return nil + } + type abortIncompleteMultipartUploadWrapper AbortIncompleteMultipartUpload + return e.EncodeElement(abortIncompleteMultipartUploadWrapper(n), start) +} + +// NoncurrentVersionExpiration - Specifies when noncurrent object versions expire. +// Upon expiration, server permanently deletes the noncurrent object versions. +// Set this lifecycle configuration action on a bucket that has versioning enabled +// (or suspended) to request server delete noncurrent object versions at a +// specific period in the object's lifetime. +type NoncurrentVersionExpiration struct { + XMLName xml.Name `xml:"NoncurrentVersionExpiration" json:"-"` + NoncurrentDays ExpirationDays `xml:"NoncurrentDays,omitempty"` +} + +// NoncurrentVersionTransition structure, set this action to request server to +// transition noncurrent object versions to different set storage classes +// at a specific period in the object's lifetime. +type NoncurrentVersionTransition struct { + XMLName xml.Name `xml:"NoncurrentVersionTransition,omitempty" json:"-"` + StorageClass string `xml:"StorageClass,omitempty" json:"StorageClass,omitempty"` + NoncurrentDays ExpirationDays `xml:"NoncurrentDays,omitempty" json:"NoncurrentDays,omitempty"` +} + +// MarshalXML if non-current days not set to non zero value +func (n NoncurrentVersionExpiration) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if n.IsDaysNull() { + return nil + } + type noncurrentVersionExpirationWrapper NoncurrentVersionExpiration + return e.EncodeElement(noncurrentVersionExpirationWrapper(n), start) +} + +// IsDaysNull returns true if days field is null +func (n NoncurrentVersionExpiration) IsDaysNull() bool { + return n.NoncurrentDays == ExpirationDays(0) +} + +// MarshalXML is extended to leave out +// tags +func (n NoncurrentVersionTransition) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if n.NoncurrentDays == ExpirationDays(0) { + return nil + } + return e.EncodeElement(&n, start) +} + +// Tag structure key/value pair representing an object tag to apply lifecycle configuration +type Tag struct { + XMLName xml.Name `xml:"Tag,omitempty" json:"-"` + Key string `xml:"Key,omitempty" json:"Key,omitempty"` + Value string `xml:"Value,omitempty" json:"Value,omitempty"` +} + +// IsEmpty returns whether this tag is empty or not. +func (tag Tag) IsEmpty() bool { + return tag.Key == "" +} + +// Transition structure - transition details of lifecycle configuration +type Transition struct { + XMLName xml.Name `xml:"Transition" json:"-"` + Date ExpirationDate `xml:"Date,omitempty" json:"Date,omitempty"` + StorageClass string `xml:"StorageClass,omitempty" json:"StorageClass,omitempty"` + Days ExpirationDays `xml:"Days,omitempty" json:"Days,omitempty"` +} + +// IsDaysNull returns true if days field is null +func (t Transition) IsDaysNull() bool { + return t.Days == ExpirationDays(0) +} + +// IsDateNull returns true if date field is null +func (t Transition) IsDateNull() bool { + return t.Date.Time.IsZero() +} + +// IsNull returns true if both date and days fields are null +func (t Transition) IsNull() bool { + return t.IsDaysNull() && t.IsDateNull() +} + +// MarshalXML is transition is non null +func (t Transition) MarshalXML(en *xml.Encoder, startElement xml.StartElement) error { + if t.IsNull() { + return nil + } + type transitionWrapper Transition + return en.EncodeElement(transitionWrapper(t), startElement) +} + +// And And Rule for LifecycleTag, to be used in LifecycleRuleFilter +type And struct { + XMLName xml.Name `xml:"And,omitempty" json:"-"` + Prefix string `xml:"Prefix,omitempty" json:"Prefix,omitempty"` + Tags []Tag `xml:"Tag,omitempty" json:"Tags,omitempty"` +} + +// IsEmpty returns true if Tags field is null +func (a And) IsEmpty() bool { + return len(a.Tags) == 0 && a.Prefix == "" +} + +// Filter will be used in selecting rule(s) for lifecycle configuration +type Filter struct { + XMLName xml.Name `xml:"Filter" json:"-"` + And And `xml:"And,omitempty" json:"And,omitempty"` + Prefix string `xml:"Prefix,omitempty" json:"Prefix,omitempty"` + Tag Tag `xml:"Tag,omitempty" json:"-"` +} + +// MarshalXML - produces the xml representation of the Filter struct +// only one of Prefix, And and Tag should be present in the output. +func (f Filter) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if err := e.EncodeToken(start); err != nil { + return err + } + + switch { + case !f.And.IsEmpty(): + if err := e.EncodeElement(f.And, xml.StartElement{Name: xml.Name{Local: "And"}}); err != nil { + return err + } + case !f.Tag.IsEmpty(): + if err := e.EncodeElement(f.Tag, xml.StartElement{Name: xml.Name{Local: "Tag"}}); err != nil { + return err + } + default: + // Always print Prefix field when both And & Tag are empty + if err := e.EncodeElement(f.Prefix, xml.StartElement{Name: xml.Name{Local: "Prefix"}}); err != nil { + return err + } + } + + return e.EncodeToken(xml.EndElement{Name: start.Name}) +} + +// ExpirationDays is a type alias to unmarshal Days in Expiration +type ExpirationDays int + +// MarshalXML encodes number of days to expire if it is non-zero and +// encodes empty string otherwise +func (eDays ExpirationDays) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { + if eDays == 0 { + return nil + } + return e.EncodeElement(int(eDays), startElement) +} + +// ExpirationDate is a embedded type containing time.Time to unmarshal +// Date in Expiration +type ExpirationDate struct { + time.Time +} + +// MarshalXML encodes expiration date if it is non-zero and encodes +// empty string otherwise +func (eDate ExpirationDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { + if eDate.Time.IsZero() { + return nil + } + return e.EncodeElement(eDate.Format(time.RFC3339), startElement) +} + +// ExpireDeleteMarker represents value of ExpiredObjectDeleteMarker field in Expiration XML element. +type ExpireDeleteMarker bool + +// MarshalXML encodes delete marker boolean into an XML form. +func (b ExpireDeleteMarker) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { + if !b { + return nil + } + type expireDeleteMarkerWrapper ExpireDeleteMarker + return e.EncodeElement(expireDeleteMarkerWrapper(b), startElement) +} + +// Expiration structure - expiration details of lifecycle configuration +type Expiration struct { + XMLName xml.Name `xml:"Expiration,omitempty" json:"-"` + Date ExpirationDate `xml:"Date,omitempty" json:"Date,omitempty"` + Days ExpirationDays `xml:"Days,omitempty" json:"Days,omitempty"` + DeleteMarker ExpireDeleteMarker `xml:"ExpiredObjectDeleteMarker,omitempty"` +} + +// IsDaysNull returns true if days field is null +func (e Expiration) IsDaysNull() bool { + return e.Days == ExpirationDays(0) +} + +// IsDateNull returns true if date field is null +func (e Expiration) IsDateNull() bool { + return e.Date.Time.IsZero() +} + +// IsNull returns true if both date and days fields are null +func (e Expiration) IsNull() bool { + return e.IsDaysNull() && e.IsDateNull() +} + +// MarshalXML is expiration is non null +func (e Expiration) MarshalXML(en *xml.Encoder, startElement xml.StartElement) error { + if e.IsNull() { + return nil + } + type expirationWrapper Expiration + return en.EncodeElement(expirationWrapper(e), startElement) +} + +// Rule represents a single rule in lifecycle configuration +type Rule struct { + XMLName xml.Name `xml:"Rule,omitempty" json:"-"` + AbortIncompleteMultipartUpload AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty" json:"AbortIncompleteMultipartUpload,omitempty"` + Expiration Expiration `xml:"Expiration,omitempty" json:"Expiration,omitempty"` + ID string `xml:"ID" json:"ID"` + RuleFilter Filter `xml:"Filter,omitempty" json:"Filter,omitempty"` + NoncurrentVersionExpiration NoncurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty" json:"NoncurrentVersionExpiration,omitempty"` + NoncurrentVersionTransition NoncurrentVersionTransition `xml:"NoncurrentVersionTransition,omitempty" json:"NoncurrentVersionTransition,omitempty"` + Prefix string `xml:"Prefix,omitempty" json:"Prefix,omitempty"` + Status string `xml:"Status" json:"Status"` + Transition Transition `xml:"Transition,omitempty" json:"Transition,omitempty"` +} + +// Configuration is a collection of Rule objects. +type Configuration struct { + XMLName xml.Name `xml:"LifecycleConfiguration,omitempty" json:"-"` + Rules []Rule `xml:"Rule"` +} + +// Empty check if lifecycle configuration is empty +func (c *Configuration) Empty() bool { + if c == nil { + return true + } + return len(c.Rules) == 0 +} + +// NewConfiguration initializes a fresh lifecycle configuration +// for manipulation, such as setting and removing lifecycle rules +// and filters. +func NewConfiguration() *Configuration { + return &Configuration{} +} diff --git a/vendor/github.com/minio/minio-go/v7/pkg/notification/info.go b/vendor/github.com/minio/minio-go/v7/pkg/notification/info.go new file mode 100644 index 0000000000..d0a4716383 --- /dev/null +++ b/vendor/github.com/minio/minio-go/v7/pkg/notification/info.go @@ -0,0 +1,78 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017-2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package notification + +// Indentity represents the user id, this is a compliance field. +type identity struct { + PrincipalID string `json:"principalId"` +} + +// event bucket metadata. +type bucketMeta struct { + Name string `json:"name"` + OwnerIdentity identity `json:"ownerIdentity"` + ARN string `json:"arn"` +} + +// event object metadata. +type objectMeta struct { + Key string `json:"key"` + Size int64 `json:"size,omitempty"` + ETag string `json:"eTag,omitempty"` + ContentType string `json:"contentType,omitempty"` + UserMetadata map[string]string `json:"userMetadata,omitempty"` + VersionID string `json:"versionId,omitempty"` + Sequencer string `json:"sequencer"` +} + +// event server specific metadata. +type eventMeta struct { + SchemaVersion string `json:"s3SchemaVersion"` + ConfigurationID string `json:"configurationId"` + Bucket bucketMeta `json:"bucket"` + Object objectMeta `json:"object"` +} + +// sourceInfo represents information on the client that +// triggered the event notification. +type sourceInfo struct { + Host string `json:"host"` + Port string `json:"port"` + UserAgent string `json:"userAgent"` +} + +// Event represents an Amazon an S3 bucket notification event. +type Event struct { + EventVersion string `json:"eventVersion"` + EventSource string `json:"eventSource"` + AwsRegion string `json:"awsRegion"` + EventTime string `json:"eventTime"` + EventName string `json:"eventName"` + UserIdentity identity `json:"userIdentity"` + RequestParameters map[string]string `json:"requestParameters"` + ResponseElements map[string]string `json:"responseElements"` + S3 eventMeta `json:"s3"` + Source sourceInfo `json:"source"` +} + +// Info - represents the collection of notification events, additionally +// also reports errors if any while listening on bucket notifications. +type Info struct { + Records []Event + Err error +} diff --git a/vendor/github.com/minio/minio-go/v6/bucket-notification.go b/vendor/github.com/minio/minio-go/v7/pkg/notification/notification.go similarity index 68% rename from vendor/github.com/minio/minio-go/v6/bucket-notification.go rename to vendor/github.com/minio/minio-go/v7/pkg/notification/notification.go index 7503b01b9a..55c58cb3b8 100644 --- a/vendor/github.com/minio/minio-go/v6/bucket-notification.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/notification/notification.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2020 MinIO, Inc. + * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,34 +15,34 @@ * limitations under the License. */ -package minio +package notification import ( "encoding/xml" "errors" "fmt" - "github.com/minio/minio-go/v6/pkg/set" + "github.com/minio/minio-go/v7/pkg/set" ) -// NotificationEventType is a S3 notification event associated to the bucket notification configuration -type NotificationEventType string +// EventType is a S3 notification event associated to the bucket notification configuration +type EventType string // The role of all event types are described in : // http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html#notification-how-to-event-types-and-destinations const ( - ObjectCreatedAll NotificationEventType = "s3:ObjectCreated:*" - ObjectCreatedPut = "s3:ObjectCreated:Put" - ObjectCreatedPost = "s3:ObjectCreated:Post" - ObjectCreatedCopy = "s3:ObjectCreated:Copy" - ObjectCreatedCompleteMultipartUpload = "s3:ObjectCreated:CompleteMultipartUpload" - ObjectAccessedGet = "s3:ObjectAccessed:Get" - ObjectAccessedHead = "s3:ObjectAccessed:Head" - ObjectAccessedAll = "s3:ObjectAccessed:*" - ObjectRemovedAll = "s3:ObjectRemoved:*" - ObjectRemovedDelete = "s3:ObjectRemoved:Delete" - ObjectRemovedDeleteMarkerCreated = "s3:ObjectRemoved:DeleteMarkerCreated" - ObjectReducedRedundancyLostObject = "s3:ReducedRedundancyLostObject" + ObjectCreatedAll EventType = "s3:ObjectCreated:*" + ObjectCreatedPut = "s3:ObjectCreated:Put" + ObjectCreatedPost = "s3:ObjectCreated:Post" + ObjectCreatedCopy = "s3:ObjectCreated:Copy" + ObjectCreatedCompleteMultipartUpload = "s3:ObjectCreated:CompleteMultipartUpload" + ObjectAccessedGet = "s3:ObjectAccessed:Get" + ObjectAccessedHead = "s3:ObjectAccessed:Head" + ObjectAccessedAll = "s3:ObjectAccessed:*" + ObjectRemovedAll = "s3:ObjectRemoved:*" + ObjectRemovedDelete = "s3:ObjectRemoved:Delete" + ObjectRemovedDeleteMarkerCreated = "s3:ObjectRemoved:DeleteMarkerCreated" + ObjectReducedRedundancyLostObject = "s3:ReducedRedundancyLostObject" ) // FilterRule - child of S3Key, a tag in the notification xml which @@ -88,27 +88,27 @@ func (arn Arn) String() string { return "arn:" + arn.Partition + ":" + arn.Service + ":" + arn.Region + ":" + arn.AccountID + ":" + arn.Resource } -// NotificationConfig - represents one single notification configuration +// Config - represents one single notification configuration // such as topic, queue or lambda configuration. -type NotificationConfig struct { - ID string `xml:"Id,omitempty"` - Arn Arn `xml:"-"` - Events []NotificationEventType `xml:"Event"` - Filter *Filter `xml:"Filter,omitempty"` +type Config struct { + ID string `xml:"Id,omitempty"` + Arn Arn `xml:"-"` + Events []EventType `xml:"Event"` + Filter *Filter `xml:"Filter,omitempty"` } -// NewNotificationConfig creates one notification config and sets the given ARN -func NewNotificationConfig(arn Arn) NotificationConfig { - return NotificationConfig{Arn: arn, Filter: &Filter{}} +// NewConfig creates one notification config and sets the given ARN +func NewConfig(arn Arn) Config { + return Config{Arn: arn, Filter: &Filter{}} } // AddEvents adds one event to the current notification config -func (t *NotificationConfig) AddEvents(events ...NotificationEventType) { +func (t *Config) AddEvents(events ...EventType) { t.Events = append(t.Events, events...) } // AddFilterSuffix sets the suffix configuration to the current notification config -func (t *NotificationConfig) AddFilterSuffix(suffix string) { +func (t *Config) AddFilterSuffix(suffix string) { if t.Filter == nil { t.Filter = &Filter{} } @@ -124,7 +124,7 @@ func (t *NotificationConfig) AddFilterSuffix(suffix string) { } // AddFilterPrefix sets the prefix configuration to the current notification config -func (t *NotificationConfig) AddFilterPrefix(prefix string) { +func (t *Config) AddFilterPrefix(prefix string) { if t.Filter == nil { t.Filter = &Filter{} } @@ -139,8 +139,8 @@ func (t *NotificationConfig) AddFilterPrefix(prefix string) { t.Filter.S3Key.FilterRules = append(t.Filter.S3Key.FilterRules, newFilterRule) } -// EqualNotificationEventTypeList tells whether a and b contain the same events -func EqualNotificationEventTypeList(a, b []NotificationEventType) bool { +// EqualEventTypeList tells whether a and b contain the same events +func EqualEventTypeList(a, b []EventType) bool { if len(a) != len(b) { return false } @@ -176,10 +176,10 @@ func EqualFilterRuleList(a, b []FilterRule) bool { return setA.Difference(setB).IsEmpty() } -// Equal returns whether this `NotificationConfig` is equal to another defined by the passed parameters -func (t *NotificationConfig) Equal(events []NotificationEventType, prefix, suffix string) bool { +// Equal returns whether this `Config` is equal to another defined by the passed parameters +func (t *Config) Equal(events []EventType, prefix, suffix string) bool { //Compare events - passEvents := EqualNotificationEventTypeList(t.Events, events) + passEvents := EqualEventTypeList(t.Events, events) //Compare filters var newFilter []FilterRule @@ -197,24 +197,24 @@ func (t *NotificationConfig) Equal(events []NotificationEventType, prefix, suffi // TopicConfig carries one single topic notification configuration type TopicConfig struct { - NotificationConfig + Config Topic string `xml:"Topic"` } // QueueConfig carries one single queue notification configuration type QueueConfig struct { - NotificationConfig + Config Queue string `xml:"Queue"` } // LambdaConfig carries one single cloudfunction notification configuration type LambdaConfig struct { - NotificationConfig + Config Lambda string `xml:"CloudFunction"` } -// BucketNotification - the struct that represents the whole XML to be sent to the web service -type BucketNotification struct { +// Configuration - the struct that represents the whole XML to be sent to the web service +type Configuration struct { XMLName xml.Name `xml:"NotificationConfiguration"` LambdaConfigs []LambdaConfig `xml:"CloudFunctionConfiguration"` TopicConfigs []TopicConfig `xml:"TopicConfiguration"` @@ -222,8 +222,8 @@ type BucketNotification struct { } // AddTopic adds a given topic config to the general bucket notification config -func (b *BucketNotification) AddTopic(topicConfig NotificationConfig) bool { - newTopicConfig := TopicConfig{NotificationConfig: topicConfig, Topic: topicConfig.Arn.String()} +func (b *Configuration) AddTopic(topicConfig Config) bool { + newTopicConfig := TopicConfig{Config: topicConfig, Topic: topicConfig.Arn.String()} for _, n := range b.TopicConfigs { // If new config matches existing one if n.Topic == newTopicConfig.Arn.String() && newTopicConfig.Filter == n.Filter { @@ -248,8 +248,8 @@ func (b *BucketNotification) AddTopic(topicConfig NotificationConfig) bool { } // AddQueue adds a given queue config to the general bucket notification config -func (b *BucketNotification) AddQueue(queueConfig NotificationConfig) bool { - newQueueConfig := QueueConfig{NotificationConfig: queueConfig, Queue: queueConfig.Arn.String()} +func (b *Configuration) AddQueue(queueConfig Config) bool { + newQueueConfig := QueueConfig{Config: queueConfig, Queue: queueConfig.Arn.String()} for _, n := range b.QueueConfigs { if n.Queue == newQueueConfig.Arn.String() && newQueueConfig.Filter == n.Filter { @@ -273,8 +273,8 @@ func (b *BucketNotification) AddQueue(queueConfig NotificationConfig) bool { } // AddLambda adds a given lambda config to the general bucket notification config -func (b *BucketNotification) AddLambda(lambdaConfig NotificationConfig) bool { - newLambdaConfig := LambdaConfig{NotificationConfig: lambdaConfig, Lambda: lambdaConfig.Arn.String()} +func (b *Configuration) AddLambda(lambdaConfig Config) bool { + newLambdaConfig := LambdaConfig{Config: lambdaConfig, Lambda: lambdaConfig.Arn.String()} for _, n := range b.LambdaConfigs { if n.Lambda == newLambdaConfig.Arn.String() && newLambdaConfig.Filter == n.Filter { @@ -298,7 +298,7 @@ func (b *BucketNotification) AddLambda(lambdaConfig NotificationConfig) bool { } // RemoveTopicByArn removes all topic configurations that match the exact specified ARN -func (b *BucketNotification) RemoveTopicByArn(arn Arn) { +func (b *Configuration) RemoveTopicByArn(arn Arn) { var topics []TopicConfig for _, topic := range b.TopicConfigs { if topic.Topic != arn.String() { @@ -308,15 +308,15 @@ func (b *BucketNotification) RemoveTopicByArn(arn Arn) { b.TopicConfigs = topics } -// ErrNoNotificationConfigMatch is returned when a notification configuration (sqs,sns,lambda) is not found when trying to delete -var ErrNoNotificationConfigMatch = errors.New("no notification configuration matched") +// ErrNoConfigMatch is returned when a notification configuration (sqs,sns,lambda) is not found when trying to delete +var ErrNoConfigMatch = errors.New("no notification configuration matched") // RemoveTopicByArnEventsPrefixSuffix removes a topic configuration that match the exact specified ARN, events, prefix and suffix -func (b *BucketNotification) RemoveTopicByArnEventsPrefixSuffix(arn Arn, events []NotificationEventType, prefix, suffix string) error { +func (b *Configuration) RemoveTopicByArnEventsPrefixSuffix(arn Arn, events []EventType, prefix, suffix string) error { removeIndex := -1 for i, v := range b.TopicConfigs { // if it matches events and filters, mark the index for deletion - if v.Topic == arn.String() && v.NotificationConfig.Equal(events, prefix, suffix) { + if v.Topic == arn.String() && v.Config.Equal(events, prefix, suffix) { removeIndex = i break // since we have at most one matching config } @@ -325,11 +325,11 @@ func (b *BucketNotification) RemoveTopicByArnEventsPrefixSuffix(arn Arn, events b.TopicConfigs = append(b.TopicConfigs[:removeIndex], b.TopicConfigs[removeIndex+1:]...) return nil } - return ErrNoNotificationConfigMatch + return ErrNoConfigMatch } // RemoveQueueByArn removes all queue configurations that match the exact specified ARN -func (b *BucketNotification) RemoveQueueByArn(arn Arn) { +func (b *Configuration) RemoveQueueByArn(arn Arn) { var queues []QueueConfig for _, queue := range b.QueueConfigs { if queue.Queue != arn.String() { @@ -340,11 +340,11 @@ func (b *BucketNotification) RemoveQueueByArn(arn Arn) { } // RemoveQueueByArnEventsPrefixSuffix removes a queue configuration that match the exact specified ARN, events, prefix and suffix -func (b *BucketNotification) RemoveQueueByArnEventsPrefixSuffix(arn Arn, events []NotificationEventType, prefix, suffix string) error { +func (b *Configuration) RemoveQueueByArnEventsPrefixSuffix(arn Arn, events []EventType, prefix, suffix string) error { removeIndex := -1 for i, v := range b.QueueConfigs { // if it matches events and filters, mark the index for deletion - if v.Queue == arn.String() && v.NotificationConfig.Equal(events, prefix, suffix) { + if v.Queue == arn.String() && v.Config.Equal(events, prefix, suffix) { removeIndex = i break // since we have at most one matching config } @@ -353,11 +353,11 @@ func (b *BucketNotification) RemoveQueueByArnEventsPrefixSuffix(arn Arn, events b.QueueConfigs = append(b.QueueConfigs[:removeIndex], b.QueueConfigs[removeIndex+1:]...) return nil } - return ErrNoNotificationConfigMatch + return ErrNoConfigMatch } // RemoveLambdaByArn removes all lambda configurations that match the exact specified ARN -func (b *BucketNotification) RemoveLambdaByArn(arn Arn) { +func (b *Configuration) RemoveLambdaByArn(arn Arn) { var lambdas []LambdaConfig for _, lambda := range b.LambdaConfigs { if lambda.Lambda != arn.String() { @@ -368,11 +368,11 @@ func (b *BucketNotification) RemoveLambdaByArn(arn Arn) { } // RemoveLambdaByArnEventsPrefixSuffix removes a topic configuration that match the exact specified ARN, events, prefix and suffix -func (b *BucketNotification) RemoveLambdaByArnEventsPrefixSuffix(arn Arn, events []NotificationEventType, prefix, suffix string) error { +func (b *Configuration) RemoveLambdaByArnEventsPrefixSuffix(arn Arn, events []EventType, prefix, suffix string) error { removeIndex := -1 for i, v := range b.LambdaConfigs { // if it matches events and filters, mark the index for deletion - if v.Lambda == arn.String() && v.NotificationConfig.Equal(events, prefix, suffix) { + if v.Lambda == arn.String() && v.Config.Equal(events, prefix, suffix) { removeIndex = i break // since we have at most one matching config } @@ -381,5 +381,5 @@ func (b *BucketNotification) RemoveLambdaByArnEventsPrefixSuffix(arn Arn, events b.LambdaConfigs = append(b.LambdaConfigs[:removeIndex], b.LambdaConfigs[removeIndex+1:]...) return nil } - return ErrNoNotificationConfigMatch + return ErrNoConfigMatch } diff --git a/vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go b/vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go new file mode 100644 index 0000000000..11ec67982f --- /dev/null +++ b/vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go @@ -0,0 +1,360 @@ +/* + * MinIO Client (C) 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package replication + +import ( + "bytes" + "encoding/xml" + "fmt" + "strings" + "unicode/utf8" + + "github.com/rs/xid" +) + +var errInvalidFilter = fmt.Errorf("Invalid filter") + +// OptionType specifies operation to be performed on config +type OptionType string + +const ( + // AddOption specifies addition of rule to config + AddOption OptionType = "Add" + // RemoveOption specifies rule options are for removing a rule + RemoveOption OptionType = "Remove" + // ImportOption is for getting current config + ImportOption OptionType = "Import" +) + +// Options represents options to set a replication configuration rule +type Options struct { + Op OptionType + ID string + Prefix string + Disable bool + Priority int + TagString string + StorageClass string + Arn string +} + +// Tags returns a slice of tags for a rule +func (opts Options) Tags() []Tag { + var tagList []Tag + tagTokens := strings.Split(opts.TagString, "&") + for _, tok := range tagTokens { + if tok == "" { + break + } + kv := strings.SplitN(tok, "=", 2) + tagList = append(tagList, Tag{ + Key: kv[0], + Value: kv[1], + }) + } + return tagList +} + +// Config - replication configuration specified in +// https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html +type Config struct { + XMLName xml.Name `xml:"ReplicationConfiguration" json:"-"` + Rules []Rule `xml:"Rule" json:"Rules"` + // ReplicationARN is a MinIO only extension and optional for AWS + ReplicationARN string `xml:"ReplicationArn,omitempty" json:"ReplicationArn,omitempty"` +} + +// Empty returns true if config is not set +func (c *Config) Empty() bool { + return len(c.Rules) == 0 +} + +// AddRule adds a new rule to existing replication config. If a rule exists with the +// same ID, then the rule is replaced. +func (c *Config) AddRule(opts Options) error { + tags := opts.Tags() + andVal := And{ + Tags: opts.Tags(), + } + filter := Filter{Prefix: opts.Prefix} + // only a single tag is set. + if opts.Prefix == "" && len(tags) == 1 { + filter.Tag = tags[0] + } + // both prefix and tag are present + if len(andVal.Tags) > 1 || opts.Prefix != "" { + filter.And = andVal + filter.And.Prefix = opts.Prefix + filter.Prefix = "" + } + if opts.ID == "" { + opts.ID = xid.New().String() + } + status := Enabled + if opts.Disable { + status = Disabled + } + tokens := strings.Split(opts.Arn, ":") + if len(tokens) != 6 { + return fmt.Errorf("invalid format for replication Arn") + } + if c.ReplicationARN == "" { // for new configurations + c.ReplicationARN = opts.Arn + } + newRule := Rule{ + ID: opts.ID, + Priority: opts.Priority, + Status: status, + Filter: filter, + Destination: Destination{ + Bucket: fmt.Sprintf("arn:aws:s3:::%s", tokens[5]), + StorageClass: opts.StorageClass, + }, + DeleteMarkerReplication: DeleteMarkerReplication{Status: Disabled}, + } + + ruleFound := false + for i, rule := range c.Rules { + if rule.Priority == newRule.Priority && rule.ID != newRule.ID { + return fmt.Errorf("Priority must be unique. Replication configuration already has a rule with this priority") + } + if rule.Destination.Bucket != newRule.Destination.Bucket { + return fmt.Errorf("The destination bucket must be same for all rules") + } + if rule.ID != newRule.ID { + continue + } + if newRule.Status == Disabled && rule.ID == newRule.ID { + // inherit priority from existing rule, required field on server + newRule.Priority = rule.Priority + } + c.Rules[i] = newRule + ruleFound = true + break + } + // validate rule after overlaying priority for pre-existing rule being disabled. + if err := newRule.Validate(); err != nil { + return err + } + + if !ruleFound { + c.Rules = append(c.Rules, newRule) + } + return nil +} + +// RemoveRule removes a rule from replication config. +func (c *Config) RemoveRule(opts Options) error { + var newRules []Rule + for _, rule := range c.Rules { + if rule.ID != opts.ID { + newRules = append(newRules, rule) + } + } + + if len(newRules) == 0 { + return fmt.Errorf("Replication configuration should have at least one rule") + } + c.Rules = newRules + return nil + +} + +// Rule - a rule for replication configuration. +type Rule struct { + XMLName xml.Name `xml:"Rule" json:"-"` + ID string `xml:"ID,omitempty"` + Status Status `xml:"Status"` + Priority int `xml:"Priority"` + DeleteMarkerReplication DeleteMarkerReplication `xml:"DeleteMarkerReplication"` + Destination Destination `xml:"Destination"` + Filter Filter `xml:"Filter" json:"Filter"` +} + +// Validate validates the rule for correctness +func (r Rule) Validate() error { + if err := r.validateID(); err != nil { + return err + } + if err := r.validateStatus(); err != nil { + return err + } + if err := r.validateFilter(); err != nil { + return err + } + + if r.Priority <= 0 && r.Status == Enabled { + return fmt.Errorf("Priority must be set for the rule") + } + + return nil +} + +// validateID - checks if ID is valid or not. +func (r Rule) validateID() error { + // cannot be longer than 255 characters + if len(r.ID) > 255 { + return fmt.Errorf("ID must be less than 255 characters") + } + return nil +} + +// validateStatus - checks if status is valid or not. +func (r Rule) validateStatus() error { + // Status can't be empty + if len(r.Status) == 0 { + return fmt.Errorf("status cannot be empty") + } + + // Status must be one of Enabled or Disabled + if r.Status != Enabled && r.Status != Disabled { + return fmt.Errorf("status must be set to either Enabled or Disabled") + } + return nil +} + +func (r Rule) validateFilter() error { + if err := r.Filter.Validate(); err != nil { + return err + } + return nil +} + +// Prefix - a rule can either have prefix under or under +// . This method returns the prefix from the +// location where it is available +func (r Rule) Prefix() string { + if r.Filter.Prefix != "" { + return r.Filter.Prefix + } + return r.Filter.And.Prefix +} + +// Tags - a rule can either have tag under or under +// . This method returns all the tags from the +// rule in the format tag1=value1&tag2=value2 +func (r Rule) Tags() string { + if len(r.Filter.And.Tags) != 0 { + var buf bytes.Buffer + for _, t := range r.Filter.And.Tags { + if buf.Len() > 0 { + buf.WriteString("&") + } + buf.WriteString(t.String()) + } + return buf.String() + } + return "" +} + +// Filter - a filter for a replication configuration Rule. +type Filter struct { + XMLName xml.Name `xml:"Filter" json:"-"` + Prefix string `json:"Prefix,omitempty"` + And And `xml:"And,omitempty" json:"And,omitempty"` + Tag Tag `xml:"Tag,omitempty" json:"Tag,omitempty"` +} + +// Validate - validates the filter element +func (f Filter) Validate() error { + // A Filter must have exactly one of Prefix, Tag, or And specified. + if !f.And.isEmpty() { + if f.Prefix != "" { + return errInvalidFilter + } + if !f.Tag.IsEmpty() { + return errInvalidFilter + } + } + if f.Prefix != "" { + if !f.Tag.IsEmpty() { + return errInvalidFilter + } + } + if !f.Tag.IsEmpty() { + if err := f.Tag.Validate(); err != nil { + return err + } + } + return nil +} + +// Tag - a tag for a replication configuration Rule filter. +type Tag struct { + XMLName xml.Name `json:"-"` + Key string `xml:"Key,omitempty" json:"Key,omitempty"` + Value string `xml:"Value,omitempty" json:"Value,omitempty"` +} + +func (tag Tag) String() string { + return tag.Key + "=" + tag.Value +} + +// IsEmpty returns whether this tag is empty or not. +func (tag Tag) IsEmpty() bool { + return tag.Key == "" +} + +// Validate checks this tag. +func (tag Tag) Validate() error { + if len(tag.Key) == 0 || utf8.RuneCountInString(tag.Key) > 128 { + return fmt.Errorf("Invalid Tag Key") + } + + if utf8.RuneCountInString(tag.Value) > 256 { + return fmt.Errorf("Invalid Tag Value") + } + return nil +} + +// Destination - destination in ReplicationConfiguration. +type Destination struct { + XMLName xml.Name `xml:"Destination" json:"-"` + Bucket string `xml:"Bucket" json:"Bucket"` + StorageClass string `xml:"StorageClass,omitempty" json:"StorageClass,omitempty"` +} + +// And - a tag to combine a prefix and multiple tags for replication configuration rule. +type And struct { + XMLName xml.Name `xml:"And,omitempty" json:"-"` + Prefix string `xml:"Prefix,omitempty" json:"Prefix,omitempty"` + Tags []Tag `xml:"Tag,omitempty" json:"Tags,omitempty"` +} + +// isEmpty returns true if Tags field is null +func (a And) isEmpty() bool { + return len(a.Tags) == 0 && a.Prefix == "" +} + +// Status represents Enabled/Disabled status +type Status string + +// Supported status types +const ( + Enabled Status = "Enabled" + Disabled Status = "Disabled" +) + +// DeleteMarkerReplication - whether delete markers are replicated - https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html +type DeleteMarkerReplication struct { + Status Status `xml:"Status" json:"Status"` // should be set to "Disabled" by default +} + +// IsEmpty returns true if DeleteMarkerReplication is not set +func (d DeleteMarkerReplication) IsEmpty() bool { + return len(d.Status) == 0 +} diff --git a/vendor/github.com/minio/minio-go/v6/pkg/s3utils/utils.go b/vendor/github.com/minio/minio-go/v7/pkg/s3utils/utils.go similarity index 99% rename from vendor/github.com/minio/minio-go/v6/pkg/s3utils/utils.go rename to vendor/github.com/minio/minio-go/v7/pkg/s3utils/utils.go index aa96a68fbd..65f939253d 100644 --- a/vendor/github.com/minio/minio-go/v6/pkg/s3utils/utils.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/s3utils/utils.go @@ -262,6 +262,9 @@ func TagDecode(ctag string) map[string]string { // addition to the percent encoding performed by urlEncodePath() used // here, it also percent encodes '/' (forward slash) func TagEncode(tags map[string]string) string { + if tags == nil { + return "" + } values := url.Values{} for k, v := range tags { values[k] = []string{v} diff --git a/vendor/github.com/minio/minio-go/v6/pkg/set/stringset.go b/vendor/github.com/minio/minio-go/v7/pkg/set/stringset.go similarity index 100% rename from vendor/github.com/minio/minio-go/v6/pkg/set/stringset.go rename to vendor/github.com/minio/minio-go/v7/pkg/set/stringset.go diff --git a/vendor/github.com/minio/minio-go/v6/pkg/signer/request-signature-streaming.go b/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming.go similarity index 100% rename from vendor/github.com/minio/minio-go/v6/pkg/signer/request-signature-streaming.go rename to vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming.go diff --git a/vendor/github.com/minio/minio-go/v6/pkg/signer/request-signature-v2.go b/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v2.go similarity index 99% rename from vendor/github.com/minio/minio-go/v6/pkg/signer/request-signature-v2.go rename to vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v2.go index 3179d62b50..71821a26a7 100644 --- a/vendor/github.com/minio/minio-go/v6/pkg/signer/request-signature-v2.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v2.go @@ -30,7 +30,7 @@ import ( "strings" "time" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // Signature and API related constants. @@ -262,6 +262,7 @@ var resourceList = []string{ "notification", "partNumber", "policy", + "replication", "requestPayment", "response-cache-control", "response-content-disposition", diff --git a/vendor/github.com/minio/minio-go/v6/pkg/signer/request-signature-v4.go b/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v4.go similarity index 99% rename from vendor/github.com/minio/minio-go/v6/pkg/signer/request-signature-v4.go rename to vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v4.go index 0f6a4c0a6d..67572b20d2 100644 --- a/vendor/github.com/minio/minio-go/v6/pkg/signer/request-signature-v4.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v4.go @@ -26,7 +26,7 @@ import ( "strings" "time" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" ) // Signature and API related constants. diff --git a/vendor/github.com/minio/minio-go/v6/pkg/signer/utils.go b/vendor/github.com/minio/minio-go/v7/pkg/signer/utils.go similarity index 100% rename from vendor/github.com/minio/minio-go/v6/pkg/signer/utils.go rename to vendor/github.com/minio/minio-go/v7/pkg/signer/utils.go diff --git a/vendor/github.com/minio/minio-go/v7/pkg/sse/sse.go b/vendor/github.com/minio/minio-go/v7/pkg/sse/sse.go new file mode 100644 index 0000000000..b5fb9565aa --- /dev/null +++ b/vendor/github.com/minio/minio-go/v7/pkg/sse/sse.go @@ -0,0 +1,66 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sse + +import "encoding/xml" + +// ApplySSEByDefault defines default encryption configuration, KMS or SSE. To activate +// KMS, SSEAlgoritm needs to be set to "aws:kms" +// Minio currently does not support Kms. +type ApplySSEByDefault struct { + KmsMasterKeyID string `xml:"KMSMasterKeyID,omitempty"` + SSEAlgorithm string `xml:"SSEAlgorithm"` +} + +// Rule layer encapsulates default encryption configuration +type Rule struct { + Apply ApplySSEByDefault `xml:"ApplyServerSideEncryptionByDefault"` +} + +// Configuration is the default encryption configuration structure +type Configuration struct { + XMLName xml.Name `xml:"ServerSideEncryptionConfiguration"` + Rules []Rule `xml:"Rule"` +} + +// NewConfigurationSSES3 initializes a new SSE-S3 configuration +func NewConfigurationSSES3() *Configuration { + return &Configuration{ + Rules: []Rule{ + { + Apply: ApplySSEByDefault{ + SSEAlgorithm: "AES256", + }, + }, + }, + } +} + +// NewConfigurationSSEKMS initializes a new SSE-KMS configuration +func NewConfigurationSSEKMS(kmsMasterKey string) *Configuration { + return &Configuration{ + Rules: []Rule{ + { + Apply: ApplySSEByDefault{ + KmsMasterKeyID: kmsMasterKey, + SSEAlgorithm: "aws:kms", + }, + }, + }, + } +} diff --git a/vendor/github.com/minio/minio-go/v6/pkg/tags/tags.go b/vendor/github.com/minio/minio-go/v7/pkg/tags/tags.go similarity index 95% rename from vendor/github.com/minio/minio-go/v6/pkg/tags/tags.go rename to vendor/github.com/minio/minio-go/v7/pkg/tags/tags.go index 6ef5628e9c..65ba38b102 100644 --- a/vendor/github.com/minio/minio-go/v6/pkg/tags/tags.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/tags/tags.go @@ -255,6 +255,18 @@ func (tags Tags) ToMap() map[string]string { return tags.TagSet.toMap() } +// MapToObjectTags converts an input map of key and value into +// *Tags data structure with validation. +func MapToObjectTags(tagMap map[string]string) (*Tags, error) { + return NewTags(tagMap, true) +} + +// MapToBucketTags converts an input map of key and value into +// *Tags data structure with validation. +func MapToBucketTags(tagMap map[string]string) (*Tags, error) { + return NewTags(tagMap, false) +} + // NewTags creates Tags from tagMap, If isObject is set, it validates for object tags. func NewTags(tagMap map[string]string, isObject bool) (*Tags, error) { tagging := &Tags{ diff --git a/vendor/github.com/minio/minio-go/v6/post-policy.go b/vendor/github.com/minio/minio-go/v7/post-policy.go similarity index 84% rename from vendor/github.com/minio/minio-go/v6/post-policy.go rename to vendor/github.com/minio/minio-go/v7/post-policy.go index 5b84486a45..d489d981ad 100644 --- a/vendor/github.com/minio/minio-go/v6/post-policy.go +++ b/vendor/github.com/minio/minio-go/v7/post-policy.go @@ -73,7 +73,7 @@ func NewPostPolicy() *PostPolicy { // SetExpires - Sets expiration time for the new policy. func (p *PostPolicy) SetExpires(t time.Time) error { if t.IsZero() { - return ErrInvalidArgument("No expiry time set.") + return errInvalidArgument("No expiry time set.") } p.expiration = t return nil @@ -82,7 +82,7 @@ func (p *PostPolicy) SetExpires(t time.Time) error { // SetKey - Sets an object name for the policy based upload. func (p *PostPolicy) SetKey(key string) error { if strings.TrimSpace(key) == "" || key == "" { - return ErrInvalidArgument("Object name is empty.") + return errInvalidArgument("Object name is empty.") } policyCond := policyCondition{ matchType: "eq", @@ -100,7 +100,7 @@ func (p *PostPolicy) SetKey(key string) error { // can start with. func (p *PostPolicy) SetKeyStartsWith(keyStartsWith string) error { if strings.TrimSpace(keyStartsWith) == "" || keyStartsWith == "" { - return ErrInvalidArgument("Object prefix is empty.") + return errInvalidArgument("Object prefix is empty.") } policyCond := policyCondition{ matchType: "starts-with", @@ -117,7 +117,7 @@ func (p *PostPolicy) SetKeyStartsWith(keyStartsWith string) error { // SetBucket - Sets bucket at which objects will be uploaded to. func (p *PostPolicy) SetBucket(bucketName string) error { if strings.TrimSpace(bucketName) == "" || bucketName == "" { - return ErrInvalidArgument("Bucket name is empty.") + return errInvalidArgument("Bucket name is empty.") } policyCond := policyCondition{ matchType: "eq", @@ -134,7 +134,7 @@ func (p *PostPolicy) SetBucket(bucketName string) error { // SetCondition - Sets condition for credentials, date and algorithm func (p *PostPolicy) SetCondition(matchType, condition, value string) error { if strings.TrimSpace(value) == "" || value == "" { - return ErrInvalidArgument("No value specified for condition") + return errInvalidArgument("No value specified for condition") } policyCond := policyCondition{ @@ -149,14 +149,14 @@ func (p *PostPolicy) SetCondition(matchType, condition, value string) error { p.formData[condition] = value return nil } - return ErrInvalidArgument("Invalid condition in policy") + return errInvalidArgument("Invalid condition in policy") } // SetContentType - Sets content-type of the object for this policy // based upload. func (p *PostPolicy) SetContentType(contentType string) error { if strings.TrimSpace(contentType) == "" || contentType == "" { - return ErrInvalidArgument("No content type specified.") + return errInvalidArgument("No content type specified.") } policyCond := policyCondition{ matchType: "eq", @@ -174,24 +174,42 @@ func (p *PostPolicy) SetContentType(contentType string) error { // condition for all incoming uploads. func (p *PostPolicy) SetContentLengthRange(min, max int64) error { if min > max { - return ErrInvalidArgument("Minimum limit is larger than maximum limit.") + return errInvalidArgument("Minimum limit is larger than maximum limit.") } if min < 0 { - return ErrInvalidArgument("Minimum limit cannot be negative.") + return errInvalidArgument("Minimum limit cannot be negative.") } if max < 0 { - return ErrInvalidArgument("Maximum limit cannot be negative.") + return errInvalidArgument("Maximum limit cannot be negative.") } p.contentLengthRange.min = min p.contentLengthRange.max = max return nil } +// SetSuccessActionRedirect - Sets the redirect success url of the object for this policy +// based upload. +func (p *PostPolicy) SetSuccessActionRedirect(redirect string) error { + if strings.TrimSpace(redirect) == "" || redirect == "" { + return errInvalidArgument("Redirect is empty") + } + policyCond := policyCondition{ + matchType: "eq", + condition: "$success_action_redirect", + value: redirect, + } + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData["success_action_redirect"] = redirect + return nil +} + // SetSuccessStatusAction - Sets the status success code of the object for this policy // based upload. func (p *PostPolicy) SetSuccessStatusAction(status string) error { if strings.TrimSpace(status) == "" || status == "" { - return ErrInvalidArgument("Status is empty") + return errInvalidArgument("Status is empty") } policyCond := policyCondition{ matchType: "eq", @@ -209,10 +227,10 @@ func (p *PostPolicy) SetSuccessStatusAction(status string) error { // Can be retrieved through a HEAD request or an event. func (p *PostPolicy) SetUserMetadata(key string, value string) error { if strings.TrimSpace(key) == "" || key == "" { - return ErrInvalidArgument("Key is empty") + return errInvalidArgument("Key is empty") } if strings.TrimSpace(value) == "" || value == "" { - return ErrInvalidArgument("Value is empty") + return errInvalidArgument("Value is empty") } headerName := fmt.Sprintf("x-amz-meta-%s", key) policyCond := policyCondition{ @@ -231,10 +249,10 @@ func (p *PostPolicy) SetUserMetadata(key string, value string) error { // Can be retrieved through a HEAD request or an event. func (p *PostPolicy) SetUserData(key string, value string) error { if key == "" { - return ErrInvalidArgument("Key is empty") + return errInvalidArgument("Key is empty") } if value == "" { - return ErrInvalidArgument("Value is empty") + return errInvalidArgument("Value is empty") } headerName := fmt.Sprintf("x-amz-%s", key) policyCond := policyCondition{ @@ -252,7 +270,7 @@ func (p *PostPolicy) SetUserData(key string, value string) error { // addNewPolicy - internal helper to validate adding new policies. func (p *PostPolicy) addNewPolicy(policyCond policyCondition) error { if policyCond.matchType == "" || policyCond.condition == "" || policyCond.value == "" { - return ErrInvalidArgument("Policy fields are empty.") + return errInvalidArgument("Policy fields are empty.") } p.conditions = append(p.conditions, policyCond) return nil diff --git a/vendor/github.com/minio/minio-go/v6/retry-continous.go b/vendor/github.com/minio/minio-go/v7/retry-continous.go similarity index 100% rename from vendor/github.com/minio/minio-go/v6/retry-continous.go rename to vendor/github.com/minio/minio-go/v7/retry-continous.go diff --git a/vendor/github.com/minio/minio-go/v6/retry.go b/vendor/github.com/minio/minio-go/v7/retry.go similarity index 100% rename from vendor/github.com/minio/minio-go/v6/retry.go rename to vendor/github.com/minio/minio-go/v7/retry.go diff --git a/vendor/github.com/minio/minio-go/v6/s3-endpoints.go b/vendor/github.com/minio/minio-go/v7/s3-endpoints.go similarity index 100% rename from vendor/github.com/minio/minio-go/v6/s3-endpoints.go rename to vendor/github.com/minio/minio-go/v7/s3-endpoints.go diff --git a/vendor/github.com/minio/minio-go/v6/s3-error.go b/vendor/github.com/minio/minio-go/v7/s3-error.go similarity index 100% rename from vendor/github.com/minio/minio-go/v6/s3-error.go rename to vendor/github.com/minio/minio-go/v7/s3-error.go diff --git a/vendor/github.com/minio/minio-go/v6/staticcheck.conf b/vendor/github.com/minio/minio-go/v7/staticcheck.conf similarity index 100% rename from vendor/github.com/minio/minio-go/v6/staticcheck.conf rename to vendor/github.com/minio/minio-go/v7/staticcheck.conf diff --git a/vendor/github.com/minio/minio-go/v6/transport.go b/vendor/github.com/minio/minio-go/v7/transport.go similarity index 97% rename from vendor/github.com/minio/minio-go/v6/transport.go rename to vendor/github.com/minio/minio-go/v7/transport.go index 4a1e174884..d5ad15b8b4 100644 --- a/vendor/github.com/minio/minio-go/v6/transport.go +++ b/vendor/github.com/minio/minio-go/v7/transport.go @@ -41,7 +41,7 @@ func mustGetSystemCertPool() *x509.CertPool { // DefaultTransport - this default transport is similar to // http.DefaultTransport but with additional param DisableCompression // is set to true to avoid decompressing content with 'gzip' encoding. -var DefaultTransport = func(secure bool) (http.RoundTripper, error) { +var DefaultTransport = func(secure bool) (*http.Transport, error) { tr := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ diff --git a/vendor/github.com/minio/minio-go/v6/utils.go b/vendor/github.com/minio/minio-go/v7/utils.go similarity index 77% rename from vendor/github.com/minio/minio-go/v6/utils.go rename to vendor/github.com/minio/minio-go/v7/utils.go index 0214644e43..f1b711d226 100644 --- a/vendor/github.com/minio/minio-go/v6/utils.go +++ b/vendor/github.com/minio/minio-go/v7/utils.go @@ -22,6 +22,7 @@ import ( "encoding/base64" "encoding/hex" "encoding/xml" + "fmt" "hash" "io" "io/ioutil" @@ -35,7 +36,7 @@ import ( "time" md5simd "github.com/minio/md5-simd" - "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/s3utils" "github.com/minio/sha256-simd" ) @@ -44,6 +45,19 @@ func trimEtag(etag string) string { return strings.TrimSuffix(etag, "\"") } +var expirationRegex = regexp.MustCompile(`expiry-date="(.*?)", rule-id="(.*?)"`) + +func amzExpirationToExpiryDateRuleID(expiration string) (time.Time, string) { + if matches := expirationRegex.FindStringSubmatch(expiration); len(matches) == 3 { + expTime, err := time.Parse(http.TimeFormat, matches[1]) + if err != nil { + return time.Time{}, "" + } + return expTime, matches[2] + } + return time.Time{}, "" +} + // xmlDecoder provide decoded value in xml. func xmlDecoder(body io.Reader, v interface{}) error { d := xml.NewDecoder(body) @@ -75,12 +89,12 @@ func getEndpointURL(endpoint string, secure bool) (*url.URL, error) { } if !s3utils.IsValidIP(host) && !s3utils.IsValidDomain(host) { msg := "Endpoint: " + endpoint + " does not follow ip address or domain name standards." - return nil, ErrInvalidArgument(msg) + return nil, errInvalidArgument(msg) } } else { if !s3utils.IsValidIP(endpoint) && !s3utils.IsValidDomain(endpoint) { msg := "Endpoint: " + endpoint + " does not follow ip address or domain name standards." - return nil, ErrInvalidArgument(msg) + return nil, errInvalidArgument(msg) } } // If secure is false, use 'http' scheme. @@ -134,19 +148,19 @@ var ( // Verify if input endpoint URL is valid. func isValidEndpointURL(endpointURL url.URL) error { if endpointURL == sentinelURL { - return ErrInvalidArgument("Endpoint url cannot be empty.") + return errInvalidArgument("Endpoint url cannot be empty.") } if endpointURL.Path != "/" && endpointURL.Path != "" { - return ErrInvalidArgument("Endpoint url cannot have fully qualified paths.") + return errInvalidArgument("Endpoint url cannot have fully qualified paths.") } if strings.Contains(endpointURL.Host, ".s3.amazonaws.com") { if !s3utils.IsAmazonEndpoint(endpointURL) { - return ErrInvalidArgument("Amazon S3 endpoint should be 's3.amazonaws.com'.") + return errInvalidArgument("Amazon S3 endpoint should be 's3.amazonaws.com'.") } } if strings.Contains(endpointURL.Host, ".googleapis.com") { if !s3utils.IsGoogleEndpoint(endpointURL) { - return ErrInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'.") + return errInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'.") } } return nil @@ -156,10 +170,10 @@ func isValidEndpointURL(endpointURL url.URL) error { func isValidExpiry(expires time.Duration) error { expireSeconds := int64(expires / time.Second) if expireSeconds < 1 { - return ErrInvalidArgument("Expires cannot be lesser than 1 second.") + return errInvalidArgument("Expires cannot be lesser than 1 second.") } if expireSeconds > 604800 { - return ErrInvalidArgument("Expires cannot be greater than 7 days.") + return errInvalidArgument("Expires cannot be greater than 7 days.") } return nil } @@ -219,7 +233,7 @@ func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectIn // Content-Length is not valid return ObjectInfo{}, ErrorResponse{ Code: "InternalError", - Message: "Content-Length is invalid. " + reportIssue, + Message: fmt.Sprintf("Content-Length is not an integer, failed with %v", err), BucketName: bucketName, Key: objectName, RequestID: h.Get("x-amz-request-id"), @@ -234,7 +248,7 @@ func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectIn if err != nil { return ObjectInfo{}, ErrorResponse{ Code: "InternalError", - Message: "Last-Modified time format is invalid. " + reportIssue, + Message: fmt.Sprintf("Last-Modified time format is invalid, failed with %v", err), BucketName: bucketName, Key: objectName, RequestID: h.Get("x-amz-request-id"), @@ -250,9 +264,9 @@ func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectIn } expiryStr := h.Get("Expires") - var expTime time.Time - if t, err := time.Parse(http.TimeFormat, expiryStr); err == nil { - expTime = t.UTC() + var expiry time.Time + if expiryStr != "" { + expiry, _ = time.Parse(http.TimeFormat, expiryStr) } metadata := extractObjMetadata(h) @@ -264,23 +278,82 @@ func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectIn } userTags := s3utils.TagDecode(h.Get(amzTaggingHeader)) + var tagCount int + if count := h.Get(amzTaggingCount); count != "" { + tagCount, err = strconv.Atoi(count) + if err != nil { + return ObjectInfo{}, ErrorResponse{ + Code: "InternalError", + Message: fmt.Sprintf("x-amz-tagging-count is not an integer, failed with %v", err), + BucketName: bucketName, + Key: objectName, + RequestID: h.Get("x-amz-request-id"), + HostID: h.Get("x-amz-id-2"), + Region: h.Get("x-amz-bucket-region"), + } + } + } + + // extract lifecycle expiry date and rule ID + expTime, ruleID := amzExpirationToExpiryDateRuleID(h.Get(amzExpiration)) + // Save object metadata info. return ObjectInfo{ - ETag: etag, - Key: objectName, - Size: size, - LastModified: date, - ContentType: contentType, - Expires: expTime, + ETag: etag, + Key: objectName, + Size: size, + LastModified: date, + ContentType: contentType, + Expires: expiry, + VersionID: h.Get(amzVersionID), + ReplicationStatus: h.Get(amzReplicationStatus), + Expiration: expTime, + ExpirationRuleID: ruleID, // Extract only the relevant header keys describing the object. // following function filters out a list of standard set of keys // which are not part of object metadata. Metadata: metadata, UserMetadata: userMetadata, UserTags: userTags, + UserTagCount: tagCount, }, nil } +var readFull = func(r io.Reader, buf []byte) (n int, err error) { + // ReadFull reads exactly len(buf) bytes from r into buf. + // It returns the number of bytes copied and an error if + // fewer bytes were read. The error is EOF only if no bytes + // were read. If an EOF happens after reading some but not + // all the bytes, ReadFull returns ErrUnexpectedEOF. + // On return, n == len(buf) if and only if err == nil. + // If r returns an error having read at least len(buf) bytes, + // the error is dropped. + for n < len(buf) && err == nil { + var nn int + nn, err = r.Read(buf[n:]) + // Some spurious io.Reader's return + // io.ErrUnexpectedEOF when nn == 0 + // this behavior is undocumented + // so we are on purpose not using io.ReadFull + // implementation because this can lead + // to custom handling, to avoid that + // we simply modify the original io.ReadFull + // implementation to avoid this issue. + // io.ErrUnexpectedEOF with nn == 0 really + // means that io.EOF + if err == io.ErrUnexpectedEOF && nn == 0 { + err = io.EOF + } + n += nn + } + if n >= len(buf) { + err = nil + } else if n > 0 && err == io.EOF { + err = io.ErrUnexpectedEOF + } + return +} + // regCred matches credential string in HTTP header var regCred = regexp.MustCompile("Credential=([A-Z0-9]+)/") @@ -332,6 +405,7 @@ var supportedHeaders = []string{ "x-amz-metadata-directive", "x-amz-object-lock-retain-until-date", "expires", + "x-amz-replication-status", // Add more supported headers here. } diff --git a/vendor/github.com/rs/xid/.appveyor.yml b/vendor/github.com/rs/xid/.appveyor.yml new file mode 100644 index 0000000000..c73bb33baa --- /dev/null +++ b/vendor/github.com/rs/xid/.appveyor.yml @@ -0,0 +1,27 @@ +version: 1.0.0.{build} + +platform: x64 + +branches: + only: + - master + +clone_folder: c:\gopath\src\github.com\rs\xid + +environment: + GOPATH: c:\gopath + +install: + - echo %PATH% + - echo %GOPATH% + - set PATH=%GOPATH%\bin;c:\go\bin;%PATH% + - go version + - go env + - go get -t . + +build_script: + - go build + +test_script: + - go test + diff --git a/vendor/github.com/rs/xid/.travis.yml b/vendor/github.com/rs/xid/.travis.yml new file mode 100644 index 0000000000..b37da15942 --- /dev/null +++ b/vendor/github.com/rs/xid/.travis.yml @@ -0,0 +1,8 @@ +language: go +go: +- "1.9" +- "1.10" +- "master" +matrix: + allow_failures: + - go: "master" diff --git a/vendor/github.com/rs/xid/LICENSE b/vendor/github.com/rs/xid/LICENSE new file mode 100644 index 0000000000..47c5e9d2d2 --- /dev/null +++ b/vendor/github.com/rs/xid/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2015 Olivier Poitrey + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is furnished +to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/rs/xid/README.md b/vendor/github.com/rs/xid/README.md new file mode 100644 index 0000000000..1f886fd7c0 --- /dev/null +++ b/vendor/github.com/rs/xid/README.md @@ -0,0 +1,112 @@ +# Globally Unique ID Generator + +[![godoc](http://img.shields.io/badge/godoc-reference-blue.svg?style=flat)](https://godoc.org/github.com/rs/xid) [![license](http://img.shields.io/badge/license-MIT-red.svg?style=flat)](https://raw.githubusercontent.com/rs/xid/master/LICENSE) [![Build Status](https://travis-ci.org/rs/xid.svg?branch=master)](https://travis-ci.org/rs/xid) [![Coverage](http://gocover.io/_badge/github.com/rs/xid)](http://gocover.io/github.com/rs/xid) + +Package xid is a globally unique id generator library, ready to be used safely directly in your server code. + +Xid is using Mongo Object ID algorithm to generate globally unique ids with a different serialization (base64) to make it shorter when transported as a string: +https://docs.mongodb.org/manual/reference/object-id/ + +- 4-byte value representing the seconds since the Unix epoch, +- 3-byte machine identifier, +- 2-byte process id, and +- 3-byte counter, starting with a random value. + +The binary representation of the id is compatible with Mongo 12 bytes Object IDs. +The string representation is using base32 hex (w/o padding) for better space efficiency +when stored in that form (20 bytes). The hex variant of base32 is used to retain the +sortable property of the id. + +Xid doesn't use base64 because case sensitivity and the 2 non alphanum chars may be an +issue when transported as a string between various systems. Base36 wasn't retained either +because 1/ it's not standard 2/ the resulting size is not predictable (not bit aligned) +and 3/ it would not remain sortable. To validate a base32 `xid`, expect a 20 chars long, +all lowercase sequence of `a` to `v` letters and `0` to `9` numbers (`[0-9a-v]{20}`). + +UUIDs are 16 bytes (128 bits) and 36 chars as string representation. Twitter Snowflake +ids are 8 bytes (64 bits) but require machine/data-center configuration and/or central +generator servers. xid stands in between with 12 bytes (96 bits) and a more compact +URL-safe string representation (20 chars). No configuration or central generator server +is required so it can be used directly in server's code. + +| Name | Binary Size | String Size | Features +|-------------|-------------|----------------|---------------- +| [UUID] | 16 bytes | 36 chars | configuration free, not sortable +| [shortuuid] | 16 bytes | 22 chars | configuration free, not sortable +| [Snowflake] | 8 bytes | up to 20 chars | needs machin/DC configuration, needs central server, sortable +| [MongoID] | 12 bytes | 24 chars | configuration free, sortable +| xid | 12 bytes | 20 chars | configuration free, sortable + +[UUID]: https://en.wikipedia.org/wiki/Universally_unique_identifier +[shortuuid]: https://github.com/stochastic-technologies/shortuuid +[Snowflake]: https://blog.twitter.com/2010/announcing-snowflake +[MongoID]: https://docs.mongodb.org/manual/reference/object-id/ + +Features: + +- Size: 12 bytes (96 bits), smaller than UUID, larger than snowflake +- Base32 hex encoded by default (20 chars when transported as printable string, still sortable) +- Non configured, you don't need set a unique machine and/or data center id +- K-ordered +- Embedded time with 1 second precision +- Unicity guaranteed for 16,777,216 (24 bits) unique ids per second and per host/process +- Lock-free (i.e.: unlike UUIDv1 and v2) + +Best used with [zerolog](https://github.com/rs/zerolog)'s +[RequestIDHandler](https://godoc.org/github.com/rs/zerolog/hlog#RequestIDHandler). + +Notes: + +- Xid is dependent on the system time, a monotonic counter and so is not cryptographically secure. If unpredictability of IDs is important, you should not use Xids. It is worth noting that most of the other UUID like implementations are also not cryptographically secure. You shoud use libraries that rely on cryptographically secure sources (like /dev/urandom on unix, crypto/rand in golang), if you want a truly random ID generator. + +References: + +- http://www.slideshare.net/davegardnerisme/unique-id-generation-in-distributed-systems +- https://en.wikipedia.org/wiki/Universally_unique_identifier +- https://blog.twitter.com/2010/announcing-snowflake +- Python port by [Graham Abbott](https://github.com/graham): https://github.com/graham/python_xid +- Scala port by [Egor Kolotaev](https://github.com/kolotaev): https://github.com/kolotaev/ride + +## Install + + go get github.com/rs/xid + +## Usage + +```go +guid := xid.New() + +println(guid.String()) +// Output: 9m4e2mr0ui3e8a215n4g +``` + +Get `xid` embedded info: + +```go +guid.Machine() +guid.Pid() +guid.Time() +guid.Counter() +``` + +## Benchmark + +Benchmark against Go [Maxim Bublis](https://github.com/satori)'s [UUID](https://github.com/satori/go.uuid). + +``` +BenchmarkXID 20000000 91.1 ns/op 32 B/op 1 allocs/op +BenchmarkXID-2 20000000 55.9 ns/op 32 B/op 1 allocs/op +BenchmarkXID-4 50000000 32.3 ns/op 32 B/op 1 allocs/op +BenchmarkUUIDv1 10000000 204 ns/op 48 B/op 1 allocs/op +BenchmarkUUIDv1-2 10000000 160 ns/op 48 B/op 1 allocs/op +BenchmarkUUIDv1-4 10000000 195 ns/op 48 B/op 1 allocs/op +BenchmarkUUIDv4 1000000 1503 ns/op 64 B/op 2 allocs/op +BenchmarkUUIDv4-2 1000000 1427 ns/op 64 B/op 2 allocs/op +BenchmarkUUIDv4-4 1000000 1452 ns/op 64 B/op 2 allocs/op +``` + +Note: UUIDv1 requires a global lock, hence the performence degrading as we add more CPUs. + +## Licenses + +All source code is licensed under the [MIT License](https://raw.github.com/rs/xid/master/LICENSE). diff --git a/vendor/github.com/rs/xid/go.mod b/vendor/github.com/rs/xid/go.mod new file mode 100644 index 0000000000..95b8338613 --- /dev/null +++ b/vendor/github.com/rs/xid/go.mod @@ -0,0 +1 @@ +module github.com/rs/xid diff --git a/vendor/github.com/rs/xid/hostid_darwin.go b/vendor/github.com/rs/xid/hostid_darwin.go new file mode 100644 index 0000000000..08351ff72c --- /dev/null +++ b/vendor/github.com/rs/xid/hostid_darwin.go @@ -0,0 +1,9 @@ +// +build darwin + +package xid + +import "syscall" + +func readPlatformMachineID() (string, error) { + return syscall.Sysctl("kern.uuid") +} diff --git a/vendor/github.com/rs/xid/hostid_fallback.go b/vendor/github.com/rs/xid/hostid_fallback.go new file mode 100644 index 0000000000..7fbd3c004d --- /dev/null +++ b/vendor/github.com/rs/xid/hostid_fallback.go @@ -0,0 +1,9 @@ +// +build !darwin,!linux,!freebsd,!windows + +package xid + +import "errors" + +func readPlatformMachineID() (string, error) { + return "", errors.New("not implemented") +} diff --git a/vendor/github.com/rs/xid/hostid_freebsd.go b/vendor/github.com/rs/xid/hostid_freebsd.go new file mode 100644 index 0000000000..be25a039e9 --- /dev/null +++ b/vendor/github.com/rs/xid/hostid_freebsd.go @@ -0,0 +1,9 @@ +// +build freebsd + +package xid + +import "syscall" + +func readPlatformMachineID() (string, error) { + return syscall.Sysctl("kern.hostuuid") +} diff --git a/vendor/github.com/rs/xid/hostid_linux.go b/vendor/github.com/rs/xid/hostid_linux.go new file mode 100644 index 0000000000..7d0c4a9ec4 --- /dev/null +++ b/vendor/github.com/rs/xid/hostid_linux.go @@ -0,0 +1,10 @@ +// +build linux + +package xid + +import "io/ioutil" + +func readPlatformMachineID() (string, error) { + b, err := ioutil.ReadFile("/sys/class/dmi/id/product_uuid") + return string(b), err +} diff --git a/vendor/github.com/rs/xid/hostid_windows.go b/vendor/github.com/rs/xid/hostid_windows.go new file mode 100644 index 0000000000..ec2593ee31 --- /dev/null +++ b/vendor/github.com/rs/xid/hostid_windows.go @@ -0,0 +1,38 @@ +// +build windows + +package xid + +import ( + "fmt" + "syscall" + "unsafe" +) + +func readPlatformMachineID() (string, error) { + // source: https://github.com/shirou/gopsutil/blob/master/host/host_syscall.go + var h syscall.Handle + err := syscall.RegOpenKeyEx(syscall.HKEY_LOCAL_MACHINE, syscall.StringToUTF16Ptr(`SOFTWARE\Microsoft\Cryptography`), 0, syscall.KEY_READ|syscall.KEY_WOW64_64KEY, &h) + if err != nil { + return "", err + } + defer syscall.RegCloseKey(h) + + const syscallRegBufLen = 74 // len(`{`) + len(`abcdefgh-1234-456789012-123345456671` * 2) + len(`}`) // 2 == bytes/UTF16 + const uuidLen = 36 + + var regBuf [syscallRegBufLen]uint16 + bufLen := uint32(syscallRegBufLen) + var valType uint32 + err = syscall.RegQueryValueEx(h, syscall.StringToUTF16Ptr(`MachineGuid`), nil, &valType, (*byte)(unsafe.Pointer(®Buf[0])), &bufLen) + if err != nil { + return "", err + } + + hostID := syscall.UTF16ToString(regBuf[:]) + hostIDLen := len(hostID) + if hostIDLen != uuidLen { + return "", fmt.Errorf("HostID incorrect: %q\n", hostID) + } + + return hostID, nil +} diff --git a/vendor/github.com/rs/xid/id.go b/vendor/github.com/rs/xid/id.go new file mode 100644 index 0000000000..466faf2628 --- /dev/null +++ b/vendor/github.com/rs/xid/id.go @@ -0,0 +1,365 @@ +// Package xid is a globally unique id generator suited for web scale +// +// Xid is using Mongo Object ID algorithm to generate globally unique ids: +// https://docs.mongodb.org/manual/reference/object-id/ +// +// - 4-byte value representing the seconds since the Unix epoch, +// - 3-byte machine identifier, +// - 2-byte process id, and +// - 3-byte counter, starting with a random value. +// +// The binary representation of the id is compatible with Mongo 12 bytes Object IDs. +// The string representation is using base32 hex (w/o padding) for better space efficiency +// when stored in that form (20 bytes). The hex variant of base32 is used to retain the +// sortable property of the id. +// +// Xid doesn't use base64 because case sensitivity and the 2 non alphanum chars may be an +// issue when transported as a string between various systems. Base36 wasn't retained either +// because 1/ it's not standard 2/ the resulting size is not predictable (not bit aligned) +// and 3/ it would not remain sortable. To validate a base32 `xid`, expect a 20 chars long, +// all lowercase sequence of `a` to `v` letters and `0` to `9` numbers (`[0-9a-v]{20}`). +// +// UUID is 16 bytes (128 bits), snowflake is 8 bytes (64 bits), xid stands in between +// with 12 bytes with a more compact string representation ready for the web and no +// required configuration or central generation server. +// +// Features: +// +// - Size: 12 bytes (96 bits), smaller than UUID, larger than snowflake +// - Base32 hex encoded by default (16 bytes storage when transported as printable string) +// - Non configured, you don't need set a unique machine and/or data center id +// - K-ordered +// - Embedded time with 1 second precision +// - Unicity guaranteed for 16,777,216 (24 bits) unique ids per second and per host/process +// +// Best used with xlog's RequestIDHandler (https://godoc.org/github.com/rs/xlog#RequestIDHandler). +// +// References: +// +// - http://www.slideshare.net/davegardnerisme/unique-id-generation-in-distributed-systems +// - https://en.wikipedia.org/wiki/Universally_unique_identifier +// - https://blog.twitter.com/2010/announcing-snowflake +package xid + +import ( + "bytes" + "crypto/md5" + "crypto/rand" + "database/sql/driver" + "encoding/binary" + "errors" + "fmt" + "hash/crc32" + "io/ioutil" + "os" + "sort" + "sync/atomic" + "time" +) + +// Code inspired from mgo/bson ObjectId + +// ID represents a unique request id +type ID [rawLen]byte + +const ( + encodedLen = 20 // string encoded len + rawLen = 12 // binary raw len + + // encoding stores a custom version of the base32 encoding with lower case + // letters. + encoding = "0123456789abcdefghijklmnopqrstuv" +) + +var ( + // ErrInvalidID is returned when trying to unmarshal an invalid ID + ErrInvalidID = errors.New("xid: invalid ID") + + // objectIDCounter is atomically incremented when generating a new ObjectId + // using NewObjectId() function. It's used as a counter part of an id. + // This id is initialized with a random value. + objectIDCounter = randInt() + + // machineId stores machine id generated once and used in subsequent calls + // to NewObjectId function. + machineID = readMachineID() + + // pid stores the current process id + pid = os.Getpid() + + nilID ID + + // dec is the decoding map for base32 encoding + dec [256]byte +) + +func init() { + for i := 0; i < len(dec); i++ { + dec[i] = 0xFF + } + for i := 0; i < len(encoding); i++ { + dec[encoding[i]] = byte(i) + } + + // If /proc/self/cpuset exists and is not /, we can assume that we are in a + // form of container and use the content of cpuset xor-ed with the PID in + // order get a reasonable machine global unique PID. + b, err := ioutil.ReadFile("/proc/self/cpuset") + if err == nil && len(b) > 1 { + pid ^= int(crc32.ChecksumIEEE(b)) + } +} + +// readMachineId generates machine id and puts it into the machineId global +// variable. If this function fails to get the hostname, it will cause +// a runtime error. +func readMachineID() []byte { + id := make([]byte, 3) + hid, err := readPlatformMachineID() + if err != nil || len(hid) == 0 { + hid, err = os.Hostname() + } + if err == nil && len(hid) != 0 { + hw := md5.New() + hw.Write([]byte(hid)) + copy(id, hw.Sum(nil)) + } else { + // Fallback to rand number if machine id can't be gathered + if _, randErr := rand.Reader.Read(id); randErr != nil { + panic(fmt.Errorf("xid: cannot get hostname nor generate a random number: %v; %v", err, randErr)) + } + } + return id +} + +// randInt generates a random uint32 +func randInt() uint32 { + b := make([]byte, 3) + if _, err := rand.Reader.Read(b); err != nil { + panic(fmt.Errorf("xid: cannot generate random number: %v;", err)) + } + return uint32(b[0])<<16 | uint32(b[1])<<8 | uint32(b[2]) +} + +// New generates a globally unique ID +func New() ID { + return NewWithTime(time.Now()) +} + +// NewWithTime generates a globally unique ID with the passed in time +func NewWithTime(t time.Time) ID { + var id ID + // Timestamp, 4 bytes, big endian + binary.BigEndian.PutUint32(id[:], uint32(t.Unix())) + // Machine, first 3 bytes of md5(hostname) + id[4] = machineID[0] + id[5] = machineID[1] + id[6] = machineID[2] + // Pid, 2 bytes, specs don't specify endianness, but we use big endian. + id[7] = byte(pid >> 8) + id[8] = byte(pid) + // Increment, 3 bytes, big endian + i := atomic.AddUint32(&objectIDCounter, 1) + id[9] = byte(i >> 16) + id[10] = byte(i >> 8) + id[11] = byte(i) + return id +} + +// FromString reads an ID from its string representation +func FromString(id string) (ID, error) { + i := &ID{} + err := i.UnmarshalText([]byte(id)) + return *i, err +} + +// String returns a base32 hex lowercased with no padding representation of the id (char set is 0-9, a-v). +func (id ID) String() string { + text := make([]byte, encodedLen) + encode(text, id[:]) + return string(text) +} + +// MarshalText implements encoding/text TextMarshaler interface +func (id ID) MarshalText() ([]byte, error) { + text := make([]byte, encodedLen) + encode(text, id[:]) + return text, nil +} + +// MarshalJSON implements encoding/json Marshaler interface +func (id ID) MarshalJSON() ([]byte, error) { + if id.IsNil() { + return []byte("null"), nil + } + text, err := id.MarshalText() + return []byte(`"` + string(text) + `"`), err +} + +// encode by unrolling the stdlib base32 algorithm + removing all safe checks +func encode(dst, id []byte) { + dst[0] = encoding[id[0]>>3] + dst[1] = encoding[(id[1]>>6)&0x1F|(id[0]<<2)&0x1F] + dst[2] = encoding[(id[1]>>1)&0x1F] + dst[3] = encoding[(id[2]>>4)&0x1F|(id[1]<<4)&0x1F] + dst[4] = encoding[id[3]>>7|(id[2]<<1)&0x1F] + dst[5] = encoding[(id[3]>>2)&0x1F] + dst[6] = encoding[id[4]>>5|(id[3]<<3)&0x1F] + dst[7] = encoding[id[4]&0x1F] + dst[8] = encoding[id[5]>>3] + dst[9] = encoding[(id[6]>>6)&0x1F|(id[5]<<2)&0x1F] + dst[10] = encoding[(id[6]>>1)&0x1F] + dst[11] = encoding[(id[7]>>4)&0x1F|(id[6]<<4)&0x1F] + dst[12] = encoding[id[8]>>7|(id[7]<<1)&0x1F] + dst[13] = encoding[(id[8]>>2)&0x1F] + dst[14] = encoding[(id[9]>>5)|(id[8]<<3)&0x1F] + dst[15] = encoding[id[9]&0x1F] + dst[16] = encoding[id[10]>>3] + dst[17] = encoding[(id[11]>>6)&0x1F|(id[10]<<2)&0x1F] + dst[18] = encoding[(id[11]>>1)&0x1F] + dst[19] = encoding[(id[11]<<4)&0x1F] +} + +// UnmarshalText implements encoding/text TextUnmarshaler interface +func (id *ID) UnmarshalText(text []byte) error { + if len(text) != encodedLen { + return ErrInvalidID + } + for _, c := range text { + if dec[c] == 0xFF { + return ErrInvalidID + } + } + decode(id, text) + return nil +} + +// UnmarshalJSON implements encoding/json Unmarshaler interface +func (id *ID) UnmarshalJSON(b []byte) error { + s := string(b) + if s == "null" { + *id = nilID + return nil + } + return id.UnmarshalText(b[1 : len(b)-1]) +} + +// decode by unrolling the stdlib base32 algorithm + removing all safe checks +func decode(id *ID, src []byte) { + id[0] = dec[src[0]]<<3 | dec[src[1]]>>2 + id[1] = dec[src[1]]<<6 | dec[src[2]]<<1 | dec[src[3]]>>4 + id[2] = dec[src[3]]<<4 | dec[src[4]]>>1 + id[3] = dec[src[4]]<<7 | dec[src[5]]<<2 | dec[src[6]]>>3 + id[4] = dec[src[6]]<<5 | dec[src[7]] + id[5] = dec[src[8]]<<3 | dec[src[9]]>>2 + id[6] = dec[src[9]]<<6 | dec[src[10]]<<1 | dec[src[11]]>>4 + id[7] = dec[src[11]]<<4 | dec[src[12]]>>1 + id[8] = dec[src[12]]<<7 | dec[src[13]]<<2 | dec[src[14]]>>3 + id[9] = dec[src[14]]<<5 | dec[src[15]] + id[10] = dec[src[16]]<<3 | dec[src[17]]>>2 + id[11] = dec[src[17]]<<6 | dec[src[18]]<<1 | dec[src[19]]>>4 +} + +// Time returns the timestamp part of the id. +// It's a runtime error to call this method with an invalid id. +func (id ID) Time() time.Time { + // First 4 bytes of ObjectId is 32-bit big-endian seconds from epoch. + secs := int64(binary.BigEndian.Uint32(id[0:4])) + return time.Unix(secs, 0) +} + +// Machine returns the 3-byte machine id part of the id. +// It's a runtime error to call this method with an invalid id. +func (id ID) Machine() []byte { + return id[4:7] +} + +// Pid returns the process id part of the id. +// It's a runtime error to call this method with an invalid id. +func (id ID) Pid() uint16 { + return binary.BigEndian.Uint16(id[7:9]) +} + +// Counter returns the incrementing value part of the id. +// It's a runtime error to call this method with an invalid id. +func (id ID) Counter() int32 { + b := id[9:12] + // Counter is stored as big-endian 3-byte value + return int32(uint32(b[0])<<16 | uint32(b[1])<<8 | uint32(b[2])) +} + +// Value implements the driver.Valuer interface. +func (id ID) Value() (driver.Value, error) { + if id.IsNil() { + return nil, nil + } + b, err := id.MarshalText() + return string(b), err +} + +// Scan implements the sql.Scanner interface. +func (id *ID) Scan(value interface{}) (err error) { + switch val := value.(type) { + case string: + return id.UnmarshalText([]byte(val)) + case []byte: + return id.UnmarshalText(val) + case nil: + *id = nilID + return nil + default: + return fmt.Errorf("xid: scanning unsupported type: %T", value) + } +} + +// IsNil Returns true if this is a "nil" ID +func (id ID) IsNil() bool { + return id == nilID +} + +// NilID returns a zero value for `xid.ID`. +func NilID() ID { + return nilID +} + +// Bytes returns the byte array representation of `ID` +func (id ID) Bytes() []byte { + return id[:] +} + +// FromBytes convert the byte array representation of `ID` back to `ID` +func FromBytes(b []byte) (ID, error) { + var id ID + if len(b) != rawLen { + return id, ErrInvalidID + } + copy(id[:], b) + return id, nil +} + +// Compare returns an integer comparing two IDs. It behaves just like `bytes.Compare`. +// The result will be 0 if two IDs are identical, -1 if current id is less than the other one, +// and 1 if current id is greater than the other. +func (id ID) Compare(other ID) int { + return bytes.Compare(id[:], other[:]) +} + +type sorter []ID + +func (s sorter) Len() int { + return len(s) +} + +func (s sorter) Less(i, j int) bool { + return s[i].Compare(s[j]) < 0 +} + +func (s sorter) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +// Sort sorts an array of IDs inplace. +// It works by wrapping `[]ID` and use `sort.Sort`. +func Sort(ids []ID) { + sort.Sort(sorter(ids)) +} diff --git a/vendor/golang.org/x/sys/cpu/cpu.go b/vendor/golang.org/x/sys/cpu/cpu.go index e44deb7574..5cce25ed9b 100644 --- a/vendor/golang.org/x/sys/cpu/cpu.go +++ b/vendor/golang.org/x/sys/cpu/cpu.go @@ -6,6 +6,11 @@ // various CPU architectures. package cpu +import ( + "os" + "strings" +) + // Initialized reports whether the CPU features were initialized. // // For some GOOS/GOARCH combinations initialization of the CPU features depends @@ -169,3 +174,94 @@ var S390X struct { HasVXE bool // vector-enhancements facility 1 _ CacheLinePad } + +func init() { + archInit() + initOptions() + processOptions() +} + +// options contains the cpu debug options that can be used in GODEBUG. +// Options are arch dependent and are added by the arch specific initOptions functions. +// Features that are mandatory for the specific GOARCH should have the Required field set +// (e.g. SSE2 on amd64). +var options []option + +// Option names should be lower case. e.g. avx instead of AVX. +type option struct { + Name string + Feature *bool + Specified bool // whether feature value was specified in GODEBUG + Enable bool // whether feature should be enabled + Required bool // whether feature is mandatory and can not be disabled +} + +func processOptions() { + env := os.Getenv("GODEBUG") +field: + for env != "" { + field := "" + i := strings.IndexByte(env, ',') + if i < 0 { + field, env = env, "" + } else { + field, env = env[:i], env[i+1:] + } + if len(field) < 4 || field[:4] != "cpu." { + continue + } + i = strings.IndexByte(field, '=') + if i < 0 { + print("GODEBUG sys/cpu: no value specified for \"", field, "\"\n") + continue + } + key, value := field[4:i], field[i+1:] // e.g. "SSE2", "on" + + var enable bool + switch value { + case "on": + enable = true + case "off": + enable = false + default: + print("GODEBUG sys/cpu: value \"", value, "\" not supported for cpu option \"", key, "\"\n") + continue field + } + + if key == "all" { + for i := range options { + options[i].Specified = true + options[i].Enable = enable || options[i].Required + } + continue field + } + + for i := range options { + if options[i].Name == key { + options[i].Specified = true + options[i].Enable = enable + continue field + } + } + + print("GODEBUG sys/cpu: unknown cpu feature \"", key, "\"\n") + } + + for _, o := range options { + if !o.Specified { + continue + } + + if o.Enable && !*o.Feature { + print("GODEBUG sys/cpu: can not enable \"", o.Name, "\", missing CPU support\n") + continue + } + + if !o.Enable && o.Required { + print("GODEBUG sys/cpu: can not disable \"", o.Name, "\", required CPU feature\n") + continue + } + + *o.Feature = o.Enable + } +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_aix.go b/vendor/golang.org/x/sys/cpu/cpu_aix.go index da29896687..464a209cf5 100644 --- a/vendor/golang.org/x/sys/cpu/cpu_aix.go +++ b/vendor/golang.org/x/sys/cpu/cpu_aix.go @@ -6,8 +6,6 @@ package cpu -const cacheLineSize = 128 - const ( // getsystemcfg constants _SC_IMPL = 2 @@ -15,7 +13,7 @@ const ( _IMPL_POWER9 = 0x20000 ) -func init() { +func archInit() { impl := getsystemcfg(_SC_IMPL) if impl&_IMPL_POWER8 != 0 { PPC64.IsPOWER8 = true diff --git a/vendor/golang.org/x/sys/cpu/cpu_arm.go b/vendor/golang.org/x/sys/cpu/cpu_arm.go index 981af6818c..301b752e9c 100644 --- a/vendor/golang.org/x/sys/cpu/cpu_arm.go +++ b/vendor/golang.org/x/sys/cpu/cpu_arm.go @@ -38,3 +38,36 @@ const ( hwcap2_SHA2 = 1 << 3 hwcap2_CRC32 = 1 << 4 ) + +func initOptions() { + options = []option{ + {Name: "pmull", Feature: &ARM.HasPMULL}, + {Name: "sha1", Feature: &ARM.HasSHA1}, + {Name: "sha2", Feature: &ARM.HasSHA2}, + {Name: "swp", Feature: &ARM.HasSWP}, + {Name: "thumb", Feature: &ARM.HasTHUMB}, + {Name: "thumbee", Feature: &ARM.HasTHUMBEE}, + {Name: "tls", Feature: &ARM.HasTLS}, + {Name: "vfp", Feature: &ARM.HasVFP}, + {Name: "vfpd32", Feature: &ARM.HasVFPD32}, + {Name: "vfpv3", Feature: &ARM.HasVFPv3}, + {Name: "vfpv3d16", Feature: &ARM.HasVFPv3D16}, + {Name: "vfpv4", Feature: &ARM.HasVFPv4}, + {Name: "half", Feature: &ARM.HasHALF}, + {Name: "26bit", Feature: &ARM.Has26BIT}, + {Name: "fastmul", Feature: &ARM.HasFASTMUL}, + {Name: "fpa", Feature: &ARM.HasFPA}, + {Name: "edsp", Feature: &ARM.HasEDSP}, + {Name: "java", Feature: &ARM.HasJAVA}, + {Name: "iwmmxt", Feature: &ARM.HasIWMMXT}, + {Name: "crunch", Feature: &ARM.HasCRUNCH}, + {Name: "neon", Feature: &ARM.HasNEON}, + {Name: "idivt", Feature: &ARM.HasIDIVT}, + {Name: "idiva", Feature: &ARM.HasIDIVA}, + {Name: "lpae", Feature: &ARM.HasLPAE}, + {Name: "evtstrm", Feature: &ARM.HasEVTSTRM}, + {Name: "aes", Feature: &ARM.HasAES}, + {Name: "crc32", Feature: &ARM.HasCRC32}, + } + +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_arm64.go b/vendor/golang.org/x/sys/cpu/cpu_arm64.go index 7bcb36c7bb..2d90024387 100644 --- a/vendor/golang.org/x/sys/cpu/cpu_arm64.go +++ b/vendor/golang.org/x/sys/cpu/cpu_arm64.go @@ -8,7 +8,36 @@ import "runtime" const cacheLineSize = 64 -func init() { +func initOptions() { + options = []option{ + {Name: "fp", Feature: &ARM64.HasFP}, + {Name: "asimd", Feature: &ARM64.HasASIMD}, + {Name: "evstrm", Feature: &ARM64.HasEVTSTRM}, + {Name: "aes", Feature: &ARM64.HasAES}, + {Name: "fphp", Feature: &ARM64.HasFPHP}, + {Name: "jscvt", Feature: &ARM64.HasJSCVT}, + {Name: "lrcpc", Feature: &ARM64.HasLRCPC}, + {Name: "pmull", Feature: &ARM64.HasPMULL}, + {Name: "sha1", Feature: &ARM64.HasSHA1}, + {Name: "sha2", Feature: &ARM64.HasSHA2}, + {Name: "sha3", Feature: &ARM64.HasSHA3}, + {Name: "sha512", Feature: &ARM64.HasSHA512}, + {Name: "sm3", Feature: &ARM64.HasSM3}, + {Name: "sm4", Feature: &ARM64.HasSM4}, + {Name: "sve", Feature: &ARM64.HasSVE}, + {Name: "crc32", Feature: &ARM64.HasCRC32}, + {Name: "atomics", Feature: &ARM64.HasATOMICS}, + {Name: "asimdhp", Feature: &ARM64.HasASIMDHP}, + {Name: "cpuid", Feature: &ARM64.HasCPUID}, + {Name: "asimrdm", Feature: &ARM64.HasASIMDRDM}, + {Name: "fcma", Feature: &ARM64.HasFCMA}, + {Name: "dcpop", Feature: &ARM64.HasDCPOP}, + {Name: "asimddp", Feature: &ARM64.HasASIMDDP}, + {Name: "asimdfhm", Feature: &ARM64.HasASIMDFHM}, + } +} + +func archInit() { switch runtime.GOOS { case "android", "darwin", "netbsd": // Android and iOS don't seem to allow reading these registers. diff --git a/vendor/golang.org/x/sys/cpu/cpu_linux.go b/vendor/golang.org/x/sys/cpu/cpu_linux.go index fe139182c8..6fc874f7fe 100644 --- a/vendor/golang.org/x/sys/cpu/cpu_linux.go +++ b/vendor/golang.org/x/sys/cpu/cpu_linux.go @@ -6,7 +6,7 @@ package cpu -func init() { +func archInit() { if err := readHWCAP(); err != nil { return } diff --git a/vendor/golang.org/x/sys/cpu/cpu_linux_ppc64x.go b/vendor/golang.org/x/sys/cpu/cpu_linux_ppc64x.go index 6c8d975d40..99f8a6399e 100644 --- a/vendor/golang.org/x/sys/cpu/cpu_linux_ppc64x.go +++ b/vendor/golang.org/x/sys/cpu/cpu_linux_ppc64x.go @@ -7,8 +7,6 @@ package cpu -const cacheLineSize = 128 - // HWCAP/HWCAP2 bits. These are exposed by the kernel. const ( // ISA Level diff --git a/vendor/golang.org/x/sys/cpu/cpu_linux_s390x.go b/vendor/golang.org/x/sys/cpu/cpu_linux_s390x.go index d579eaef40..b88d6b8f66 100644 --- a/vendor/golang.org/x/sys/cpu/cpu_linux_s390x.go +++ b/vendor/golang.org/x/sys/cpu/cpu_linux_s390x.go @@ -4,8 +4,6 @@ package cpu -const cacheLineSize = 256 - const ( // bit mask values from /usr/include/bits/hwcap.h hwcap_ZARCH = 2 diff --git a/vendor/golang.org/x/sys/cpu/cpu_mips64x.go b/vendor/golang.org/x/sys/cpu/cpu_mips64x.go index 6165f12124..57b5b677de 100644 --- a/vendor/golang.org/x/sys/cpu/cpu_mips64x.go +++ b/vendor/golang.org/x/sys/cpu/cpu_mips64x.go @@ -7,3 +7,9 @@ package cpu const cacheLineSize = 32 + +func initOptions() { + options = []option{ + {Name: "msa", Feature: &MIPS64X.HasMSA}, + } +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_mipsx.go b/vendor/golang.org/x/sys/cpu/cpu_mipsx.go index 1269eee88d..cfc1946b7b 100644 --- a/vendor/golang.org/x/sys/cpu/cpu_mipsx.go +++ b/vendor/golang.org/x/sys/cpu/cpu_mipsx.go @@ -7,3 +7,5 @@ package cpu const cacheLineSize = 32 + +func initOptions() {} diff --git a/vendor/golang.org/x/sys/cpu/cpu_other_arm.go b/vendor/golang.org/x/sys/cpu/cpu_other_arm.go new file mode 100644 index 0000000000..b412efc1bd --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_other_arm.go @@ -0,0 +1,9 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !linux,arm + +package cpu + +func archInit() {} diff --git a/vendor/golang.org/x/sys/cpu/cpu_ppc64x.go b/vendor/golang.org/x/sys/cpu/cpu_ppc64x.go new file mode 100644 index 0000000000..d28d675b5f --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_ppc64x.go @@ -0,0 +1,16 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ppc64 ppc64le + +package cpu + +const cacheLineSize = 128 + +func initOptions() { + options = []option{ + {Name: "darn", Feature: &PPC64.HasDARN}, + {Name: "scv", Feature: &PPC64.HasSCV}, + } +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_riscv64.go b/vendor/golang.org/x/sys/cpu/cpu_riscv64.go index efe2b7a847..8b08de341b 100644 --- a/vendor/golang.org/x/sys/cpu/cpu_riscv64.go +++ b/vendor/golang.org/x/sys/cpu/cpu_riscv64.go @@ -7,3 +7,5 @@ package cpu const cacheLineSize = 32 + +func initOptions() {} diff --git a/vendor/golang.org/x/sys/cpu/cpu_s390x.go b/vendor/golang.org/x/sys/cpu/cpu_s390x.go new file mode 100644 index 0000000000..544cd621ce --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_s390x.go @@ -0,0 +1,30 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const cacheLineSize = 256 + +func initOptions() { + options = []option{ + {Name: "zarch", Feature: &S390X.HasZARCH}, + {Name: "stfle", Feature: &S390X.HasSTFLE}, + {Name: "ldisp", Feature: &S390X.HasLDISP}, + {Name: "eimm", Feature: &S390X.HasEIMM}, + {Name: "dfp", Feature: &S390X.HasDFP}, + {Name: "etf3eh", Feature: &S390X.HasETF3EH}, + {Name: "msa", Feature: &S390X.HasMSA}, + {Name: "aes", Feature: &S390X.HasAES}, + {Name: "aescbc", Feature: &S390X.HasAESCBC}, + {Name: "aesctr", Feature: &S390X.HasAESCTR}, + {Name: "aesgcm", Feature: &S390X.HasAESGCM}, + {Name: "ghash", Feature: &S390X.HasGHASH}, + {Name: "sha1", Feature: &S390X.HasSHA1}, + {Name: "sha256", Feature: &S390X.HasSHA256}, + {Name: "sha3", Feature: &S390X.HasSHA3}, + {Name: "sha512", Feature: &S390X.HasSHA512}, + {Name: "vx", Feature: &S390X.HasVX}, + {Name: "vxe", Feature: &S390X.HasVXE}, + } +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_wasm.go b/vendor/golang.org/x/sys/cpu/cpu_wasm.go index 8681e876a9..5382f2a227 100644 --- a/vendor/golang.org/x/sys/cpu/cpu_wasm.go +++ b/vendor/golang.org/x/sys/cpu/cpu_wasm.go @@ -11,3 +11,7 @@ package cpu // rules are good enough. const cacheLineSize = 0 + +func initOptions() {} + +func archInit() {} diff --git a/vendor/golang.org/x/sys/cpu/cpu_x86.go b/vendor/golang.org/x/sys/cpu/cpu_x86.go index d70d317f5a..2ad039d40e 100644 --- a/vendor/golang.org/x/sys/cpu/cpu_x86.go +++ b/vendor/golang.org/x/sys/cpu/cpu_x86.go @@ -6,9 +6,37 @@ package cpu +import "runtime" + const cacheLineSize = 64 -func init() { +func initOptions() { + options = []option{ + {Name: "adx", Feature: &X86.HasADX}, + {Name: "aes", Feature: &X86.HasAES}, + {Name: "avx", Feature: &X86.HasAVX}, + {Name: "avx2", Feature: &X86.HasAVX2}, + {Name: "bmi1", Feature: &X86.HasBMI1}, + {Name: "bmi2", Feature: &X86.HasBMI2}, + {Name: "erms", Feature: &X86.HasERMS}, + {Name: "fma", Feature: &X86.HasFMA}, + {Name: "osxsave", Feature: &X86.HasOSXSAVE}, + {Name: "pclmulqdq", Feature: &X86.HasPCLMULQDQ}, + {Name: "popcnt", Feature: &X86.HasPOPCNT}, + {Name: "rdrand", Feature: &X86.HasRDRAND}, + {Name: "rdseed", Feature: &X86.HasRDSEED}, + {Name: "sse3", Feature: &X86.HasSSE3}, + {Name: "sse41", Feature: &X86.HasSSE41}, + {Name: "sse42", Feature: &X86.HasSSE42}, + {Name: "ssse3", Feature: &X86.HasSSSE3}, + + // These capabilities should always be enabled on amd64: + {Name: "sse2", Feature: &X86.HasSSE2, Required: runtime.GOARCH == "amd64"}, + } +} + +func archInit() { + Initialized = true maxID, _, _, _ := cpuid(0, 0) @@ -52,6 +80,7 @@ func init() { X86.HasERMS = isSet(9, ebx7) X86.HasRDSEED = isSet(18, ebx7) X86.HasADX = isSet(19, ebx7) + } func isSet(bitpos uint, value uint32) bool { diff --git a/vendor/golang.org/x/sys/unix/mkerrors.sh b/vendor/golang.org/x/sys/unix/mkerrors.sh index 08f8230d6d..53a2493126 100644 --- a/vendor/golang.org/x/sys/unix/mkerrors.sh +++ b/vendor/golang.org/x/sys/unix/mkerrors.sh @@ -107,6 +107,7 @@ includes_FreeBSD=' #include #include #include +#include #include #include #include @@ -297,6 +298,7 @@ includes_NetBSD=' #include #include #include +#include #include #include #include @@ -325,6 +327,7 @@ includes_OpenBSD=' #include #include #include +#include #include #include #include @@ -507,6 +510,8 @@ ccflags="$@" $2 ~ /^(CLOCK|TIMER)_/ || $2 ~ /^CAN_/ || $2 ~ /^CAP_/ || + $2 ~ /^CP_/ || + $2 ~ /^CPUSTATES$/ || $2 ~ /^ALG_/ || $2 ~ /^FS_(POLICY_FLAGS|KEY_DESC|ENCRYPTION_MODE|[A-Z0-9_]+_KEY_SIZE)/ || $2 ~ /^FS_IOC_.*(ENCRYPTION|VERITY|[GS]ETFLAGS)/ || diff --git a/vendor/golang.org/x/sys/unix/syscall_bsd.go b/vendor/golang.org/x/sys/unix/syscall_bsd.go index 68605db624..60bbe10adf 100644 --- a/vendor/golang.org/x/sys/unix/syscall_bsd.go +++ b/vendor/golang.org/x/sys/unix/syscall_bsd.go @@ -527,6 +527,23 @@ func SysctlClockinfo(name string) (*Clockinfo, error) { return &ci, nil } +func SysctlTimeval(name string) (*Timeval, error) { + mib, err := sysctlmib(name) + if err != nil { + return nil, err + } + + var tv Timeval + n := uintptr(unsafe.Sizeof(tv)) + if err := sysctl(mib, (*byte)(unsafe.Pointer(&tv)), &n, nil, 0); err != nil { + return nil, err + } + if n != unsafe.Sizeof(tv) { + return nil, EIO + } + return &tv, nil +} + //sys utimes(path string, timeval *[2]Timeval) (err error) func Utimes(path string, tv []Timeval) error { diff --git a/vendor/golang.org/x/sys/unix/syscall_linux.go b/vendor/golang.org/x/sys/unix/syscall_linux.go index e50e4cb276..ab16736894 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux.go @@ -1965,10 +1965,15 @@ func isGroupMember(gid int) bool { } //sys faccessat(dirfd int, path string, mode uint32) (err error) +//sys Faccessat2(dirfd int, path string, mode uint32, flags int) (err error) func Faccessat(dirfd int, path string, mode uint32, flags int) (err error) { - if flags & ^(AT_SYMLINK_NOFOLLOW|AT_EACCESS) != 0 { - return EINVAL + if flags == 0 { + return faccessat(dirfd, path, mode) + } + + if err := Faccessat2(dirfd, path, mode, flags); err != ENOSYS { + return err } // The Linux kernel faccessat system call does not take any flags. @@ -1977,8 +1982,8 @@ func Faccessat(dirfd int, path string, mode uint32, flags int) (err error) { // Because people naturally expect syscall.Faccessat to act // like C faccessat, we do the same. - if flags == 0 { - return faccessat(dirfd, path, mode) + if flags & ^(AT_SYMLINK_NOFOLLOW|AT_EACCESS) != 0 { + return EINVAL } var st Stat_t @@ -2122,6 +2127,18 @@ func Klogset(typ int, arg int) (err error) { return nil } +// RemoteIovec is Iovec with the pointer replaced with an integer. +// It is used for ProcessVMReadv and ProcessVMWritev, where the pointer +// refers to a location in a different process' address space, which +// would confuse the Go garbage collector. +type RemoteIovec struct { + Base uintptr + Len int +} + +//sys ProcessVMReadv(pid int, localIov []Iovec, remoteIov []RemoteIovec, flags uint) (n int, err error) = SYS_PROCESS_VM_READV +//sys ProcessVMWritev(pid int, localIov []Iovec, remoteIov []RemoteIovec, flags uint) (n int, err error) = SYS_PROCESS_VM_WRITEV + /* * Unimplemented */ diff --git a/vendor/golang.org/x/sys/unix/zerrors_freebsd_386.go b/vendor/golang.org/x/sys/unix/zerrors_freebsd_386.go index 8482458734..3689c80848 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_freebsd_386.go +++ b/vendor/golang.org/x/sys/unix/zerrors_freebsd_386.go @@ -339,6 +339,12 @@ const ( CLOCK_UPTIME_FAST = 0x8 CLOCK_UPTIME_PRECISE = 0x7 CLOCK_VIRTUAL = 0x1 + CPUSTATES = 0x5 + CP_IDLE = 0x4 + CP_INTR = 0x3 + CP_NICE = 0x1 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x30000 CS5 = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_freebsd_amd64.go b/vendor/golang.org/x/sys/unix/zerrors_freebsd_amd64.go index 4acd101c3e..b8f7c3c930 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_freebsd_amd64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_freebsd_amd64.go @@ -339,6 +339,12 @@ const ( CLOCK_UPTIME_FAST = 0x8 CLOCK_UPTIME_PRECISE = 0x7 CLOCK_VIRTUAL = 0x1 + CPUSTATES = 0x5 + CP_IDLE = 0x4 + CP_INTR = 0x3 + CP_NICE = 0x1 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x30000 CS5 = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_freebsd_arm.go b/vendor/golang.org/x/sys/unix/zerrors_freebsd_arm.go index e4719873b9..be14bb1a4c 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_freebsd_arm.go +++ b/vendor/golang.org/x/sys/unix/zerrors_freebsd_arm.go @@ -339,6 +339,12 @@ const ( CLOCK_UPTIME_FAST = 0x8 CLOCK_UPTIME_PRECISE = 0x7 CLOCK_VIRTUAL = 0x1 + CPUSTATES = 0x5 + CP_IDLE = 0x4 + CP_INTR = 0x3 + CP_NICE = 0x1 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x30000 CS5 = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_freebsd_arm64.go b/vendor/golang.org/x/sys/unix/zerrors_freebsd_arm64.go index 5e49769d96..7ce9c0081a 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_freebsd_arm64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_freebsd_arm64.go @@ -339,6 +339,12 @@ const ( CLOCK_UPTIME_FAST = 0x8 CLOCK_UPTIME_PRECISE = 0x7 CLOCK_VIRTUAL = 0x1 + CPUSTATES = 0x5 + CP_IDLE = 0x4 + CP_INTR = 0x3 + CP_NICE = 0x1 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x30000 CS5 = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux.go b/vendor/golang.org/x/sys/unix/zerrors_linux.go index f8bd50c11b..d91382078e 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux.go @@ -265,6 +265,7 @@ const ( CAP_AUDIT_READ = 0x25 CAP_AUDIT_WRITE = 0x1d CAP_BLOCK_SUSPEND = 0x24 + CAP_BPF = 0x27 CAP_CHOWN = 0x0 CAP_DAC_OVERRIDE = 0x1 CAP_DAC_READ_SEARCH = 0x2 @@ -273,7 +274,7 @@ const ( CAP_IPC_LOCK = 0xe CAP_IPC_OWNER = 0xf CAP_KILL = 0x5 - CAP_LAST_CAP = 0x25 + CAP_LAST_CAP = 0x27 CAP_LEASE = 0x1c CAP_LINUX_IMMUTABLE = 0x9 CAP_MAC_ADMIN = 0x21 @@ -283,6 +284,7 @@ const ( CAP_NET_BIND_SERVICE = 0xa CAP_NET_BROADCAST = 0xb CAP_NET_RAW = 0xd + CAP_PERFMON = 0x26 CAP_SETFCAP = 0x1f CAP_SETGID = 0x6 CAP_SETPCAP = 0x8 @@ -372,6 +374,7 @@ const ( DEVLINK_GENL_NAME = "devlink" DEVLINK_GENL_VERSION = 0x1 DEVLINK_SB_THRESHOLD_TO_ALPHA_MAX = 0x14 + DEVMEM_MAGIC = 0x454d444d DEVPTS_SUPER_MAGIC = 0x1cd1 DMA_BUF_MAGIC = 0x444d4142 DT_BLK = 0x6 @@ -475,6 +478,7 @@ const ( ETH_P_MOBITEX = 0x15 ETH_P_MPLS_MC = 0x8848 ETH_P_MPLS_UC = 0x8847 + ETH_P_MRP = 0x88e3 ETH_P_MVRP = 0x88f5 ETH_P_NCSI = 0x88f8 ETH_P_NSH = 0x894f @@ -602,8 +606,9 @@ const ( FSCRYPT_POLICY_FLAGS_PAD_4 = 0x0 FSCRYPT_POLICY_FLAGS_PAD_8 = 0x1 FSCRYPT_POLICY_FLAGS_PAD_MASK = 0x3 - FSCRYPT_POLICY_FLAGS_VALID = 0xf + FSCRYPT_POLICY_FLAGS_VALID = 0x1f FSCRYPT_POLICY_FLAG_DIRECT_KEY = 0x4 + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32 = 0x10 FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 = 0x8 FSCRYPT_POLICY_V1 = 0x0 FSCRYPT_POLICY_V2 = 0x2 @@ -632,7 +637,7 @@ const ( FS_POLICY_FLAGS_PAD_4 = 0x0 FS_POLICY_FLAGS_PAD_8 = 0x1 FS_POLICY_FLAGS_PAD_MASK = 0x3 - FS_POLICY_FLAGS_VALID = 0xf + FS_POLICY_FLAGS_VALID = 0x1f FS_VERITY_FL = 0x100000 FS_VERITY_HASH_ALG_SHA256 = 0x1 FS_VERITY_HASH_ALG_SHA512 = 0x2 @@ -1016,6 +1021,7 @@ const ( KEYCTL_CAPS0_PERSISTENT_KEYRINGS = 0x2 KEYCTL_CAPS0_PUBLIC_KEY = 0x8 KEYCTL_CAPS0_RESTRICT_KEYRING = 0x40 + KEYCTL_CAPS1_NOTIFICATIONS = 0x4 KEYCTL_CAPS1_NS_KEYRING_NAME = 0x1 KEYCTL_CAPS1_NS_KEY_TAG = 0x2 KEYCTL_CHOWN = 0x4 @@ -1053,6 +1059,7 @@ const ( KEYCTL_SUPPORTS_VERIFY = 0x8 KEYCTL_UNLINK = 0x9 KEYCTL_UPDATE = 0x2 + KEYCTL_WATCH_KEY = 0x20 KEY_REQKEY_DEFL_DEFAULT = 0x0 KEY_REQKEY_DEFL_GROUP_KEYRING = 0x6 KEY_REQKEY_DEFL_NO_CHANGE = -0x1 @@ -1096,6 +1103,8 @@ const ( LOOP_SET_FD = 0x4c00 LOOP_SET_STATUS = 0x4c02 LOOP_SET_STATUS64 = 0x4c04 + LOOP_SET_STATUS_CLEARABLE_FLAGS = 0x4 + LOOP_SET_STATUS_SETTABLE_FLAGS = 0xc LO_KEY_SIZE = 0x20 LO_NAME_SIZE = 0x40 MADV_COLD = 0x14 @@ -1992,8 +2001,10 @@ const ( STATX_ATTR_APPEND = 0x20 STATX_ATTR_AUTOMOUNT = 0x1000 STATX_ATTR_COMPRESSED = 0x4 + STATX_ATTR_DAX = 0x2000 STATX_ATTR_ENCRYPTED = 0x800 STATX_ATTR_IMMUTABLE = 0x10 + STATX_ATTR_MOUNT_ROOT = 0x2000 STATX_ATTR_NODUMP = 0x40 STATX_ATTR_VERITY = 0x100000 STATX_BASIC_STATS = 0x7ff @@ -2002,6 +2013,7 @@ const ( STATX_CTIME = 0x80 STATX_GID = 0x10 STATX_INO = 0x100 + STATX_MNT_ID = 0x1000 STATX_MODE = 0x2 STATX_MTIME = 0x40 STATX_NLINK = 0x4 diff --git a/vendor/golang.org/x/sys/unix/zerrors_netbsd_386.go b/vendor/golang.org/x/sys/unix/zerrors_netbsd_386.go index 96b9b8ab30..20f3a5799f 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_netbsd_386.go +++ b/vendor/golang.org/x/sys/unix/zerrors_netbsd_386.go @@ -158,6 +158,12 @@ const ( CLONE_SIGHAND = 0x800 CLONE_VFORK = 0x4000 CLONE_VM = 0x100 + CPUSTATES = 0x5 + CP_IDLE = 0x4 + CP_INTR = 0x3 + CP_NICE = 0x1 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x10000 CS5 = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_netbsd_amd64.go b/vendor/golang.org/x/sys/unix/zerrors_netbsd_amd64.go index ed522a84e8..90b8fcd29c 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_netbsd_amd64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_netbsd_amd64.go @@ -158,6 +158,12 @@ const ( CLONE_SIGHAND = 0x800 CLONE_VFORK = 0x4000 CLONE_VM = 0x100 + CPUSTATES = 0x5 + CP_IDLE = 0x4 + CP_INTR = 0x3 + CP_NICE = 0x1 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x10000 CS5 = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_netbsd_arm.go b/vendor/golang.org/x/sys/unix/zerrors_netbsd_arm.go index c8d36fe998..c5c03993b6 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_netbsd_arm.go +++ b/vendor/golang.org/x/sys/unix/zerrors_netbsd_arm.go @@ -150,6 +150,12 @@ const ( BRKINT = 0x2 CFLUSH = 0xf CLOCAL = 0x8000 + CPUSTATES = 0x5 + CP_IDLE = 0x4 + CP_INTR = 0x3 + CP_NICE = 0x1 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x10000 CS5 = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_netbsd_arm64.go b/vendor/golang.org/x/sys/unix/zerrors_netbsd_arm64.go index f1c146a74c..14dd3c1d1e 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_netbsd_arm64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_netbsd_arm64.go @@ -158,6 +158,12 @@ const ( CLONE_SIGHAND = 0x800 CLONE_VFORK = 0x4000 CLONE_VM = 0x100 + CPUSTATES = 0x5 + CP_IDLE = 0x4 + CP_INTR = 0x3 + CP_NICE = 0x1 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x10000 CS5 = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_openbsd_386.go b/vendor/golang.org/x/sys/unix/zerrors_openbsd_386.go index 5402bd55ce..c865a10df4 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_openbsd_386.go +++ b/vendor/golang.org/x/sys/unix/zerrors_openbsd_386.go @@ -146,6 +146,13 @@ const ( BRKINT = 0x2 CFLUSH = 0xf CLOCAL = 0x8000 + CPUSTATES = 0x6 + CP_IDLE = 0x5 + CP_INTR = 0x4 + CP_NICE = 0x1 + CP_SPIN = 0x3 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x10000 CS5 = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_openbsd_amd64.go b/vendor/golang.org/x/sys/unix/zerrors_openbsd_amd64.go index ffaf2d2f9f..9db6b2fb6e 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_openbsd_amd64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_openbsd_amd64.go @@ -153,6 +153,13 @@ const ( CLOCK_REALTIME = 0x0 CLOCK_THREAD_CPUTIME_ID = 0x4 CLOCK_UPTIME = 0x5 + CPUSTATES = 0x6 + CP_IDLE = 0x5 + CP_INTR = 0x4 + CP_NICE = 0x1 + CP_SPIN = 0x3 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x10000 CS5 = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_openbsd_arm.go b/vendor/golang.org/x/sys/unix/zerrors_openbsd_arm.go index 7aa796a642..7072526a64 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_openbsd_arm.go +++ b/vendor/golang.org/x/sys/unix/zerrors_openbsd_arm.go @@ -146,6 +146,13 @@ const ( BRKINT = 0x2 CFLUSH = 0xf CLOCAL = 0x8000 + CPUSTATES = 0x6 + CP_IDLE = 0x5 + CP_INTR = 0x4 + CP_NICE = 0x1 + CP_SPIN = 0x3 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x10000 CS5 = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_openbsd_arm64.go b/vendor/golang.org/x/sys/unix/zerrors_openbsd_arm64.go index 1792d3f13e..ac5efbe5ac 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_openbsd_arm64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_openbsd_arm64.go @@ -156,6 +156,13 @@ const ( CLOCK_REALTIME = 0x0 CLOCK_THREAD_CPUTIME_ID = 0x4 CLOCK_UPTIME = 0x5 + CPUSTATES = 0x6 + CP_IDLE = 0x5 + CP_INTR = 0x4 + CP_NICE = 0x1 + CP_SPIN = 0x3 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x10000 CS5 = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux.go b/vendor/golang.org/x/sys/unix/zsyscall_linux.go index df217825f0..4eec7a7953 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux.go @@ -1821,6 +1821,21 @@ func faccessat(dirfd int, path string, mode uint32) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func Faccessat2(dirfd int, path string, mode uint32, flags int) (err error) { + var _p0 *byte + _p0, err = BytePtrFromString(path) + if err != nil { + return + } + _, _, e1 := Syscall6(SYS_FACCESSAT2, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(flags), 0, 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func nameToHandleAt(dirFD int, pathname string, fh *fileHandle, mountID *_C_int, flags int) (err error) { var _p0 *byte _p0, err = BytePtrFromString(pathname) @@ -1847,6 +1862,52 @@ func openByHandleAt(mountFD int, fh *fileHandle, flags int) (fd int, err error) // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ProcessVMReadv(pid int, localIov []Iovec, remoteIov []RemoteIovec, flags uint) (n int, err error) { + var _p0 unsafe.Pointer + if len(localIov) > 0 { + _p0 = unsafe.Pointer(&localIov[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + var _p1 unsafe.Pointer + if len(remoteIov) > 0 { + _p1 = unsafe.Pointer(&remoteIov[0]) + } else { + _p1 = unsafe.Pointer(&_zero) + } + r0, _, e1 := Syscall6(SYS_PROCESS_VM_READV, uintptr(pid), uintptr(_p0), uintptr(len(localIov)), uintptr(_p1), uintptr(len(remoteIov)), uintptr(flags)) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func ProcessVMWritev(pid int, localIov []Iovec, remoteIov []RemoteIovec, flags uint) (n int, err error) { + var _p0 unsafe.Pointer + if len(localIov) > 0 { + _p0 = unsafe.Pointer(&localIov[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + var _p1 unsafe.Pointer + if len(remoteIov) > 0 { + _p1 = unsafe.Pointer(&remoteIov[0]) + } else { + _p1 = unsafe.Pointer(&_zero) + } + r0, _, e1 := Syscall6(SYS_PROCESS_VM_WRITEV, uintptr(pid), uintptr(_p0), uintptr(len(localIov)), uintptr(_p1), uintptr(len(remoteIov)), uintptr(flags)) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func pipe2(p *[2]_C_int, flags int) (err error) { _, _, e1 := RawSyscall(SYS_PIPE2, uintptr(unsafe.Pointer(p)), uintptr(flags), 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_386.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_386.go index 54559a8956..a597e061ca 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_386.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_386.go @@ -433,4 +433,5 @@ const ( SYS_CLONE3 = 435 SYS_OPENAT2 = 437 SYS_PIDFD_GETFD = 438 + SYS_FACCESSAT2 = 439 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go index 054a741b7f..8c102e55a1 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go @@ -355,4 +355,5 @@ const ( SYS_CLONE3 = 435 SYS_OPENAT2 = 437 SYS_PIDFD_GETFD = 438 + SYS_FACCESSAT2 = 439 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go index 307f2ba12e..98f9b68fb9 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go @@ -397,4 +397,5 @@ const ( SYS_CLONE3 = 435 SYS_OPENAT2 = 437 SYS_PIDFD_GETFD = 438 + SYS_FACCESSAT2 = 439 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go index e9404dd545..4dabc33fbc 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go @@ -300,4 +300,5 @@ const ( SYS_CLONE3 = 435 SYS_OPENAT2 = 437 SYS_PIDFD_GETFD = 438 + SYS_FACCESSAT2 = 439 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go index 68bb6d29b8..d5724e5964 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go @@ -418,4 +418,5 @@ const ( SYS_CLONE3 = 4435 SYS_OPENAT2 = 4437 SYS_PIDFD_GETFD = 4438 + SYS_FACCESSAT2 = 4439 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go index 4e5251185f..c1d824a4f3 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go @@ -348,4 +348,5 @@ const ( SYS_CLONE3 = 5435 SYS_OPENAT2 = 5437 SYS_PIDFD_GETFD = 5438 + SYS_FACCESSAT2 = 5439 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go index 4d9aa3003b..598dd5d637 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go @@ -348,4 +348,5 @@ const ( SYS_CLONE3 = 5435 SYS_OPENAT2 = 5437 SYS_PIDFD_GETFD = 5438 + SYS_FACCESSAT2 = 5439 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go index 64af0707d5..c36782d08e 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go @@ -418,4 +418,5 @@ const ( SYS_CLONE3 = 4435 SYS_OPENAT2 = 4437 SYS_PIDFD_GETFD = 4438 + SYS_FACCESSAT2 = 4439 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go index cc3c067ba3..9287538d36 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go @@ -397,4 +397,5 @@ const ( SYS_CLONE3 = 435 SYS_OPENAT2 = 437 SYS_PIDFD_GETFD = 438 + SYS_FACCESSAT2 = 439 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go index 4050ff9836..4dafad8352 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go @@ -397,4 +397,5 @@ const ( SYS_CLONE3 = 435 SYS_OPENAT2 = 437 SYS_PIDFD_GETFD = 438 + SYS_FACCESSAT2 = 439 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_riscv64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_riscv64.go index 529abb6a7f..6642cfccdf 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_riscv64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_riscv64.go @@ -299,4 +299,5 @@ const ( SYS_CLONE3 = 435 SYS_OPENAT2 = 437 SYS_PIDFD_GETFD = 438 + SYS_FACCESSAT2 = 439 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go index 2766500109..23367b9467 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go @@ -362,4 +362,5 @@ const ( SYS_CLONE3 = 435 SYS_OPENAT2 = 437 SYS_PIDFD_GETFD = 438 + SYS_FACCESSAT2 = 439 ) diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_sparc64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_sparc64.go index 4dc82bb249..083aa0204e 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_sparc64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_sparc64.go @@ -376,4 +376,5 @@ const ( SYS_PIDFD_OPEN = 434 SYS_OPENAT2 = 437 SYS_PIDFD_GETFD = 438 + SYS_FACCESSAT2 = 439 ) diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux.go b/vendor/golang.org/x/sys/unix/ztypes_linux.go index 27d67ac8f5..11b15ef785 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux.go @@ -67,7 +67,9 @@ type Statx_t struct { Rdev_minor uint32 Dev_major uint32 Dev_minor uint32 - _ [14]uint64 + Mnt_id uint64 + _ uint64 + _ [12]uint64 } type Fsid struct { diff --git a/vendor/modules.txt b/vendor/modules.txt index 772ff6e5f2..c4c95cb102 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -157,7 +157,6 @@ github.com/golang/snappy # github.com/google/btree v1.0.0 github.com/google/btree # github.com/google/uuid v1.1.1 -## explicit github.com/google/uuid # github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 github.com/gopherjs/gopherjs/js @@ -245,8 +244,7 @@ github.com/jmoiron/sqlx/reflectx github.com/jonboulle/clockwork # github.com/json-iterator/go v1.1.10 github.com/json-iterator/go -# github.com/klauspost/cpuid v1.3.0 -## explicit +# github.com/klauspost/cpuid v1.3.1 github.com/klauspost/cpuid # github.com/konsorten/go-windows-terminal-sequences v1.0.3 github.com/konsorten/go-windows-terminal-sequences @@ -316,15 +314,19 @@ github.com/matttproud/golang_protobuf_extensions/pbutil github.com/miekg/dns # github.com/minio/md5-simd v1.1.0 github.com/minio/md5-simd -# github.com/minio/minio-go/v6 v6.0.57 +# github.com/minio/minio-go/v7 v7.0.3 ## explicit -github.com/minio/minio-go/v6 -github.com/minio/minio-go/v6/pkg/credentials -github.com/minio/minio-go/v6/pkg/encrypt -github.com/minio/minio-go/v6/pkg/s3utils -github.com/minio/minio-go/v6/pkg/set -github.com/minio/minio-go/v6/pkg/signer -github.com/minio/minio-go/v6/pkg/tags +github.com/minio/minio-go/v7 +github.com/minio/minio-go/v7/pkg/credentials +github.com/minio/minio-go/v7/pkg/encrypt +github.com/minio/minio-go/v7/pkg/lifecycle +github.com/minio/minio-go/v7/pkg/notification +github.com/minio/minio-go/v7/pkg/replication +github.com/minio/minio-go/v7/pkg/s3utils +github.com/minio/minio-go/v7/pkg/set +github.com/minio/minio-go/v7/pkg/signer +github.com/minio/minio-go/v7/pkg/sse +github.com/minio/minio-go/v7/pkg/tags # github.com/minio/sha256-simd v0.1.1 github.com/minio/sha256-simd # github.com/mitchellh/go-homedir v1.1.0 @@ -407,6 +409,8 @@ github.com/prometheus/procfs/internal/util # github.com/rs/cors v1.7.0 ## explicit github.com/rs/cors +# github.com/rs/xid v1.2.1 +github.com/rs/xid # github.com/rudderlabs/analytics-go v3.2.1+incompatible ## explicit github.com/rudderlabs/analytics-go @@ -535,7 +539,7 @@ go.uber.org/zap/internal/bufferpool go.uber.org/zap/internal/color go.uber.org/zap/internal/exit go.uber.org/zap/zapcore -# golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 +# golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de ## explicit golang.org/x/crypto/acme golang.org/x/crypto/acme/autocert @@ -572,7 +576,7 @@ golang.org/x/image/webp ## explicit golang.org/x/mod/module golang.org/x/mod/semver -# golang.org/x/net v0.0.0-20200625001655-4c5254603344 +# golang.org/x/net v0.0.0-20200707034311-ab3426394381 ## explicit golang.org/x/net/bpf golang.org/x/net/context @@ -590,7 +594,7 @@ golang.org/x/net/ipv4 golang.org/x/net/ipv6 golang.org/x/net/publicsuffix golang.org/x/net/trace -# golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae +# golang.org/x/sys v0.0.0-20200805065543-0cf7623e9dbd ## explicit golang.org/x/sys/cpu golang.org/x/sys/internal/unsafeheader @@ -738,7 +742,6 @@ google.golang.org/protobuf/types/known/timestamppb ## explicit gopkg.in/alexcesaro/quotedprintable.v3 # gopkg.in/ini.v1 v1.57.0 -## explicit gopkg.in/ini.v1 # gopkg.in/mail.v2 v2.3.1 ## explicit