From adc841144135707f580d790283884d603975df94 Mon Sep 17 00:00:00 2001
From: cc <12904584+shvc@users.noreply.github.com>
Date: Thu, 7 Sep 2023 10:34:05 +0800
Subject: [PATCH] mpu concurency (#31)

---
 go.mod   | 6 +++---
 go.sum   | 8 ++++----
 main.go  | 4 +++-
 s3cli.go | 3 ++-
 4 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/go.mod b/go.mod
index 2a7c462..0988fb5 100644
--- a/go.mod
+++ b/go.mod
@@ -1,10 +1,10 @@
 module s3cli
 
-go 1.20
+go 1.21
 
 require (
-	github.com/aws/aws-sdk-go v1.44.307
-	github.com/johannesboyne/gofakes3 v0.0.0-20230506070712-04da935ef877
+	github.com/aws/aws-sdk-go v1.45.2
+	github.com/johannesboyne/gofakes3 v0.0.0-20230905042043-42c772b34232
 	github.com/spf13/cobra v1.7.0
 )
 
diff --git a/go.sum b/go.sum
index 9fa2601..13bf6b2 100644
--- a/go.sum
+++ b/go.sum
@@ -1,6 +1,6 @@
 github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
-github.com/aws/aws-sdk-go v1.44.307 h1:2R0/EPgpZcFSUwZhYImq/srjaOrOfLv5MNRzrFyAM38=
-github.com/aws/aws-sdk-go v1.44.307/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go v1.45.2 h1:hTong9YUklQKqzrGk3WnKABReb5R8GjbG4Y6dEQfjnk=
+github.com/aws/aws-sdk-go v1.45.2/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -10,8 +10,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
-github.com/johannesboyne/gofakes3 v0.0.0-20230506070712-04da935ef877 h1:O7syWuYGzre3s73s+NkgB8e0ZvsIVhT/zxNU7V1gHK8=
-github.com/johannesboyne/gofakes3 v0.0.0-20230506070712-04da935ef877/go.mod h1:AxgWC4DDX54O2WDoQO1Ceabtn6IbktjU/7bigor+66g=
+github.com/johannesboyne/gofakes3 v0.0.0-20230905042043-42c772b34232 h1:bLR4m7ytWFI7DNjaaucEs/7cSgLmc4fIXNt0utD+yZ4=
+github.com/johannesboyne/gofakes3 v0.0.0-20230905042043-42c772b34232/go.mod h1:AxgWC4DDX54O2WDoQO1Ceabtn6IbktjU/7bigor+66g=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
diff --git a/main.go b/main.go
index 36dbccb..a6fdc9f 100644
--- a/main.go
+++ b/main.go
@@ -1006,6 +1006,7 @@ EnvVar:
 	}
 	rootCmd.AddCommand(mpuCompleteCmd)
 
+	var mpuConcurrency = s3manager.DefaultUploadConcurrency
 	mpuCmd := &cobra.Command{
 		Use:   "mpu <bucket[/key]> [file]",
 		Short: "mpu Object(mpu-create, mpu-upload and mpu-complete)",
@@ -1046,13 +1047,14 @@ EnvVar:
 				key = filepath.Base(args[1])
 			}
 
-			err = sc.mpu(ctx, bucket, key, objectContentType, partSize<<20, fd, metadata)
+			err = sc.mpu(ctx, bucket, key, objectContentType, partSize<<20, mpuConcurrency, fd, metadata)
 
 			return sc.errorHandler(err)
 		},
 	}
 	mpuCmd.Flags().StringVar(&objectContentType, "content-type", "", "Object content-type(auto detect if not specified)")
 	mpuCmd.Flags().Int64("part-size", s3manager.MinUploadPartSize>>20, "MPU part-size in MB")
+	mpuCmd.Flags().IntVar(&mpuConcurrency, "concurrency", mpuConcurrency, "MPU concurrency num")
 	mpuCmd.Flags().StringArrayVar(&objectMetadata, "md", nil, "Object user metadata(format Key:Value)")
 	rootCmd.AddCommand(mpuCmd)
 
diff --git a/s3cli.go b/s3cli.go
index 4bb9505..f752a67 100644
--- a/s3cli.go
+++ b/s3cli.go
@@ -1703,9 +1703,10 @@ func (sc *S3Cli) mpuComplete(ctx context.Context, bucket, key, uid string, etags
 	return err
 }
 
-func (sc *S3Cli) mpu(ctx context.Context, bucket, key, contentType string, partSize int64, r io.Reader, metadata map[string]*string) error {
+func (sc *S3Cli) mpu(ctx context.Context, bucket, key, contentType string, partSize int64, concurrency int, r io.Reader, metadata map[string]*string) error {
 	uploader := s3manager.NewUploaderWithClient(sc.Client, func(u *s3manager.Uploader) {
 		u.PartSize = partSize
+		u.Concurrency = concurrency
 	})
 
 	mi := &s3manager.UploadInput{