From e17cdabea2dd5523ef6a6104b474fb58296a32cb Mon Sep 17 00:00:00 2001 From: Smith Cruise Date: Sat, 8 Jun 2024 21:44:58 +0800 Subject: [PATCH 1/5] Support external-id in assume role Signed-off-by: Smith Cruise --- .../src/main/java/org/apache/hadoop/fs/s3a/Constants.java | 5 +++++ .../hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 185389739cbad..ac2c64300d2e7 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -94,6 +94,11 @@ private Constants() { public static final String ASSUMED_ROLE_ARN = "fs.s3a.assumed.role.arn"; + /** + * external id for assume role request: {@value}. + */ + public static final String ASSUMED_ROLE_EXTERNAL_ID = "fs.s3a.assumed.role.external.id"; + /** * Session name for the assumed role, must be valid characters according * to the AWS APIs: {@value}. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java index c2ac8fe4c8197..ce20684feca83 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java @@ -125,6 +125,7 @@ public AssumedRoleCredentialProvider(@Nullable URI fsUri, Configuration conf) duration = conf.getTimeDuration(ASSUMED_ROLE_SESSION_DURATION, ASSUMED_ROLE_SESSION_DURATION_DEFAULT, TimeUnit.SECONDS); String policy = conf.getTrimmed(ASSUMED_ROLE_POLICY, ""); + String externalId = conf.getTrimmed(ASSUMED_ROLE_EXTERNAL_ID, ""); LOG.debug("{}", this); @@ -132,6 +133,10 @@ public AssumedRoleCredentialProvider(@Nullable URI fsUri, Configuration conf) AssumeRoleRequest.builder().roleArn(arn).roleSessionName(sessionName) .durationSeconds((int) duration); + if (StringUtils.isNotEmpty(externalId)) { + requestBuilder.externalId(externalId); + } + if (StringUtils.isNotEmpty(policy)) { LOG.debug("Scope down policy {}", policy); requestBuilder.policy(policy); From d443ac7d2bd02230b9db3ad1491d4eb0ef8c0673 Mon Sep 17 00:00:00 2001 From: Smith Cruise Date: Fri, 30 Aug 2024 13:08:18 +0800 Subject: [PATCH 2/5] update doc Signed-off-by: Smith Cruise --- .../hadoop-common/src/main/resources/core-default.xml | 8 ++++++++ .../src/site/markdown/tools/hadoop-aws/assumed_roles.md | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index bd91de0f080fd..05944df65ce50 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1457,6 +1457,14 @@ + + fs.s3a.assumed.role.external.id + + + External id for assumed role, it's an optional configuration. + + + fs.s3a.assumed.role.policy diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md index 065a757f21704..df0ffb6e88ff8 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md @@ -153,6 +153,14 @@ Here are the full set of configuration options. + + fs.s3a.assumed.role.external.id + + + External id for assumed role, it's an optional configuration. + + + fs.s3a.assumed.role.policy From b4a6ba5553d8db7801f8298188adc1d66d889745 Mon Sep 17 00:00:00 2001 From: Smith Cruise Date: Fri, 30 Aug 2024 21:58:06 +0800 Subject: [PATCH 3/5] update doc Signed-off-by: Smith Cruise --- .../hadoop-common/src/main/resources/core-default.xml | 8 -------- .../src/site/markdown/tools/hadoop-aws/assumed_roles.md | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 05944df65ce50..bd91de0f080fd 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1457,14 +1457,6 @@ - - fs.s3a.assumed.role.external.id - - - External id for assumed role, it's an optional configuration. - - - fs.s3a.assumed.role.policy diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md index df0ffb6e88ff8..17d1feac0a687 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md @@ -157,7 +157,7 @@ Here are the full set of configuration options. fs.s3a.assumed.role.external.id - External id for assumed role, it's an optional configuration. + External id for assumed role, it's an optional configuration. How to Use External ID When Granting Access to Your AWS Resources From d68a856ed7e96ca014e12d9080131ed72016375e Mon Sep 17 00:00:00 2001 From: Smith Cruise Date: Sat, 31 Aug 2024 10:08:42 +0800 Subject: [PATCH 4/5] update doc Signed-off-by: Smith Cruise --- .../src/site/markdown/tools/hadoop-aws/assumed_roles.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md index 17d1feac0a687..54b1493993af4 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md @@ -157,7 +157,7 @@ Here are the full set of configuration options. fs.s3a.assumed.role.external.id - External id for assumed role, it's an optional configuration. How to Use External ID When Granting Access to Your AWS Resources + External id for assumed role, it's an optional configuration. "https://aws.amazon.com/cn/blogs/security/how-to-use-external-id-when-granting-access-to-your-aws-resources/" From d75fb47ce3a40b0d70059da5f015760c0e62eb09 Mon Sep 17 00:00:00 2001 From: Smith Cruise Date: Tue, 10 Sep 2024 15:43:15 +0800 Subject: [PATCH 5/5] improve Signed-off-by: Smith Cruise --- .../src/site/markdown/tools/hadoop-aws/assumed_roles.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md index 54b1493993af4..ba1bc4b362c47 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md @@ -155,7 +155,7 @@ Here are the full set of configuration options. fs.s3a.assumed.role.external.id - + arbitrary value, specific by user in AWS console External id for assumed role, it's an optional configuration. "https://aws.amazon.com/cn/blogs/security/how-to-use-external-id-when-granting-access-to-your-aws-resources/"