From 4f98399ff6bcfd023e607cb6db5b92955362095a Mon Sep 17 00:00:00 2001 From: vnarayanan Date: Tue, 4 Feb 2025 17:56:50 -0800 Subject: [PATCH] Add ProfileAWSCredentialsProvider test --- .../src/main/resources/core-default.xml | 2 +- .../fs/s3a/ProfileAWSCredentialsProvider.java | 46 --------------- .../auth/CredentialProviderListFactory.java | 1 - .../auth/ProfileAWSCredentialsProvider.java | 56 +++++++++++++++++++ .../fs/s3a/TestS3AAWSCredentialsProvider.java | 35 +++++++++--- 5 files changed, 85 insertions(+), 55 deletions(-) delete mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProfileAWSCredentialsProvider.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 8c17cb8c93ef2..117ec1bab0b4e 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1431,7 +1431,6 @@ org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider, software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider, org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider, - org.apache.hadoop.fs.s3a.ProfileAWSCredentialsProvider Comma-separated class names of credential provider classes which implement @@ -1443,6 +1442,7 @@ token binding it may be used to communicate wih the STS endpoint to request session/role credentials. + org.apache.hadoop.fs.s3a.auth.ProfileAWSCredentialsProvider is also supported, but is not enabled by default. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProfileAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProfileAWSCredentialsProvider.java deleted file mode 100644 index 9efb68788ed29..0000000000000 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProfileAWSCredentialsProvider.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.apache.hadoop.fs.s3a; - -import org.apache.commons.lang3.SystemUtils; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.conf.Configuration; -import software.amazon.awssdk.auth.credentials.AwsCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; -import software.amazon.awssdk.profiles.ProfileFile; - -import java.net.URI; -import java.nio.file.FileSystems; -import java.nio.file.Path; - -@InterfaceAudience.Public -@InterfaceStability.Stable -public class ProfileAWSCredentialsProvider implements AwsCredentialsProvider { - public static final String NAME - = "org.apache.hadoop.fs.s3a.ProfileAWSCredentialsProvider"; - - private ProfileCredentialsProvider pcp; - - private static Path getCredentialsPath() { - String credentialsFile = SystemUtils.getEnvironmentVariable("AWS_SHARED_CREDENTIALS_FILE", null); - Path path = (credentialsFile == null) ? - FileSystems.getDefault().getPath(SystemUtils.getUserHome().getPath(),".aws","credentials") - : FileSystems.getDefault().getPath(credentialsFile); - return path; - } - - public ProfileAWSCredentialsProvider(URI uri, Configuration conf) { - ProfileCredentialsProvider.Builder builder = ProfileCredentialsProvider.builder(); - builder.profileName("default").profileFile(ProfileFile.builder().content(getCredentialsPath()).type(ProfileFile.Type.CREDENTIALS).build()); - pcp = builder.build(); - } - - public ProfileAWSCredentialsProvider(Configuration conf) { - ProfileCredentialsProvider.Builder builder = ProfileCredentialsProvider.builder(); - builder.profileName("default").profileFile(ProfileFile.builder().content(getCredentialsPath()).build()); - pcp = builder.build(); - } - public AwsCredentials resolveCredentials() { - return pcp.resolveCredentials(); - } -} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java index 387f0ab46767c..116349c49eac1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java @@ -44,7 +44,6 @@ import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.S3AUtils; import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider; -import org.apache.hadoop.fs.s3a.ProfileAWSCredentialsProvider; import org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider; import org.apache.hadoop.fs.s3a.adapter.AwsV1BindingSupport; import org.apache.hadoop.fs.s3a.impl.InstantiationIOException; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java new file mode 100644 index 0000000000000..acdad9fc52480 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/ProfileAWSCredentialsProvider.java @@ -0,0 +1,56 @@ +package org.apache.hadoop.fs.s3a.auth; + +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.profiles.ProfileFile; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +import org.apache.commons.lang3.SystemUtils; +import org.apache.hadoop.conf.Configuration; + +import java.net.URI; +import java.nio.file.FileSystems; +import java.nio.file.Path; + +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class ProfileAWSCredentialsProvider extends AbstractAWSCredentialProvider { + public static final String NAME + = "org.apache.hadoop.fs.s3a.auth.ProfileAWSCredentialsProvider"; + public static final String PROFILE_FILE = "fs.s3a.auth.profile.file"; + public static final String PROFILE_NAME = "fs.s3a.auth.profile.name"; + + private final ProfileCredentialsProvider pcp; + + private static Path getCredentialsPath(Configuration conf) { + String credentialsFile = conf.get(PROFILE_FILE, null); + if (credentialsFile == null) { + credentialsFile = SystemUtils.getEnvironmentVariable("AWS_SHARED_CREDENTIALS_FILE", null); + } + Path path = (credentialsFile == null) ? + FileSystems.getDefault().getPath(SystemUtils.getUserHome().getPath(),".aws","credentials") + : FileSystems.getDefault().getPath(credentialsFile); + return path; + } + + private static String getCredentialsName(Configuration conf) { + String profileName = conf.get(PROFILE_NAME, null); + if (profileName == null) { + profileName = SystemUtils.getEnvironmentVariable("AWS_PROFILE", "default"); + } + return profileName; + } + + public ProfileAWSCredentialsProvider(URI uri, Configuration conf) { + super(uri, conf); + ProfileCredentialsProvider.Builder builder = ProfileCredentialsProvider.builder(); + builder.profileName(getCredentialsName(conf)).profileFile(ProfileFile.builder().content(getCredentialsPath(conf)).type(ProfileFile.Type.CREDENTIALS).build()); + pcp = builder.build(); + } + + public AwsCredentials resolveCredentials() { + return pcp.resolveCredentials(); + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index d51bc954a6329..d26c1a7e2de33 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -18,8 +18,7 @@ package org.apache.hadoop.fs.s3a; -import java.io.IOException; -import java.io.InterruptedIOException; +import java.io.*; import java.net.URI; import java.nio.file.AccessDeniedException; import java.util.ArrayList; @@ -35,6 +34,7 @@ import java.util.stream.Collectors; import javax.annotation.Nullable; +import org.apache.hadoop.fs.s3a.auth.*; import org.assertj.core.api.Assertions; import org.junit.Test; import org.slf4j.Logger; @@ -47,11 +47,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.s3a.auth.AbstractSessionCredentialsProvider; -import org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider; -import org.apache.hadoop.fs.s3a.auth.CredentialProviderListFactory; -import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider; -import org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException; import org.apache.hadoop.fs.s3a.auth.delegation.CountInvocationsProvider; import org.apache.hadoop.fs.s3a.impl.InstantiationIOException; import org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils; @@ -139,6 +134,32 @@ public void testInstantiationChain() throws Throwable { assertCredentialProviders(expectedClasses, list); } + @Test + public void testProfileAWSCredentialsProvider() throws Throwable { + Configuration conf = new Configuration(false); + conf.set(AWS_CREDENTIALS_PROVIDER, ProfileAWSCredentialsProvider.NAME); + try (FileWriter fileWriter = new FileWriter("testcred"); BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) { + bufferedWriter.write("[default]\n" + + "aws_access_key_id = defaultaccesskeyid\n" + + "aws_secret_access_key = defaultsecretkeyid\n"); + bufferedWriter.write("[nondefault]\n" + + "aws_access_key_id = nondefaultaccesskeyid\n" + + "aws_secret_access_key = nondefaultsecretkeyid\n"); + } + conf.set(ProfileAWSCredentialsProvider.PROFILE_FILE, "testcred"); + URI testUri = new URI("s3a://bucket1"); + AWSCredentialProviderList list = createAWSCredentialProviderList(testUri, conf); + assertCredentialProviders(Collections.singletonList(ProfileAWSCredentialsProvider.class), list); + AwsCredentials credentials = list.resolveCredentials(); + assertEquals("defaultaccesskeyid", credentials.accessKeyId()); + assertEquals("defaultsecretkeyid", credentials.secretAccessKey()); + conf.set(ProfileAWSCredentialsProvider.PROFILE_NAME, "nondefault"); + list = createAWSCredentialProviderList(testUri, conf); + credentials = list.resolveCredentials(); + assertEquals("nondefaultaccesskeyid", credentials.accessKeyId()); + assertEquals("nondefaultsecretkeyid", credentials.secretAccessKey()); + } + @Test public void testDefaultChain() throws Exception { URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2");