Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
HADOOP-19197. S3A: Support AWS KMS Encryption Context
Browse files Browse the repository at this point in the history
Add the property fs.s3a.encryption.context that allow users to specify the AWS KMS Encryption Context to be used in S3A.

The value of the encryption context is a key/value string that will be Base64 encoded and set in the parameter ssekmsEncryptionContext from the S3 client.

Contributed by Raphael Azzolini
raphaelazzolini committed Jun 8, 2024
1 parent 01d257d commit 71ffcd5
Showing 18 changed files with 464 additions and 27 deletions.
Original file line number Diff line number Diff line change
@@ -1022,6 +1022,7 @@ public class CommonConfigurationKeysPublic {
"fs.s3a.*.server-side-encryption.key",
"fs.s3a.encryption.algorithm",
"fs.s3a.encryption.key",
"fs.s3a.encryption.context",
"fs.azure\\.account.key.*",
"credential$",
"oauth.*secret",
Original file line number Diff line number Diff line change
@@ -742,6 +742,7 @@
fs.s3a.*.server-side-encryption.key
fs.s3a.encryption.algorithm
fs.s3a.encryption.key
fs.s3a.encryption.context
fs.s3a.secret.key
fs.s3a.*.secret.key
fs.s3a.session.key
@@ -1760,6 +1761,15 @@
</description>
</property>

<property>
<name>fs.s3a.encryption.context</name>
<description>Specific encryption context to use if fs.s3a.encryption.algorithm
has been set to 'SSE-KMS' or 'DSSE-KMS'. The value of this property is a set
of non-secret comma-separated key-value pairs of additional contextual
information about the data that are separated by equal operator (=).
</description>
</property>

<property>
<name>fs.s3a.signing-algorithm</name>
<description>Override the default signing algorithm so legacy
Original file line number Diff line number Diff line change
@@ -736,6 +736,16 @@ private Constants() {
public static final String S3_ENCRYPTION_KEY =
"fs.s3a.encryption.key";

/**
* Set S3-SSE encryption context.
* The value of this property is a set of non-secret comma-separated key-value pairs
* of additional contextual information about the data that are separated by equal
* operator (=).
* value:{@value}
*/
public static final String S3_ENCRYPTION_CONTEXT =
"fs.s3a.encryption.context";

/**
* List of custom Signers. The signer class will be loaded, and the signer
* name will be associated with this signer class in the S3 SDK.
Original file line number Diff line number Diff line change
@@ -18,6 +18,7 @@

package org.apache.hadoop.fs.s3a;

import com.fasterxml.jackson.databind.ObjectMapper;
import software.amazon.awssdk.awscore.exception.AwsServiceException;
import software.amazon.awssdk.core.exception.AbortedException;
import software.amazon.awssdk.core.exception.ApiCallAttemptTimeoutException;
@@ -27,6 +28,7 @@
import software.amazon.awssdk.services.s3.model.S3Exception;
import software.amazon.awssdk.services.s3.model.S3Object;

import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -62,6 +64,7 @@
import java.lang.reflect.Modifier;
import java.net.SocketTimeoutException;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.AccessDeniedException;
import java.util.ArrayList;
import java.util.Collection;
@@ -124,6 +127,7 @@ public final class S3AUtils {
S3AEncryptionMethods.SSE_S3.getMethod()
+ " is enabled but an encryption key was set in "
+ Constants.S3_ENCRYPTION_KEY;

public static final String EOF_MESSAGE_IN_XML_PARSER
= "Failed to sanitize XML document destined for handler class";

@@ -1402,6 +1406,79 @@ public static String getS3EncryptionKey(
}
}

/**
* Get any SSE context, without propagating exceptions from
* JCEKs files.
* @param bucket bucket to query for
* @param conf configuration to examine
* @return the encryption context value or ""
* @throws IllegalArgumentException bad arguments.
*/
public static String getS3EncryptionContext(
String bucket,
Configuration conf) {
try {
return getEncryptionContextValue(bucket, conf);
} catch (IOException e) {
// never going to happen, but to make sure, covert to
// runtime exception
throw new UncheckedIOException(e);
}
}

/**
* Get any SSE context from a configuration/credential provider.
* This includes converting the values to a base64-encoded UTF-8 string
* holding JSON with the encryption context key-value pairs
* @param bucket bucket to query for
* @param conf configuration to examine
* @param propagateExceptions should IO exceptions be rethrown?
* @return the Base64 encryption context or ""
* @throws IllegalArgumentException bad arguments.
* @throws IOException if propagateExceptions==true and reading a JCEKS file raised an IOE
*/
public static String getS3EncryptionContextBase64Encoded(
String bucket,
Configuration conf,
boolean propagateExceptions) throws IOException {
try {
final String encryptionContextValue = getEncryptionContextValue(bucket, conf);
if (StringUtils.isBlank(encryptionContextValue)) {
return "";
}
final Map<String, String> encryptionContextMap = getTrimmedStringCollectionSplitByEquals(
encryptionContextValue);
if (encryptionContextMap.isEmpty()) {
return "";
}
final String encryptionContextJson = new ObjectMapper().writeValueAsString(
encryptionContextMap);
return Base64.encodeBase64String(encryptionContextJson.getBytes(StandardCharsets.UTF_8));
} catch (IOException e) {
if (propagateExceptions) {
throw e;
}
LOG.warn("Cannot retrieve {} for bucket {}",
S3_ENCRYPTION_CONTEXT, bucket, e);
return "";
}
}

private static String getEncryptionContextValue(String bucket, Configuration conf)
throws IOException {
// look up the per-bucket value of the encryption context
String encryptionContext = lookupBucketSecret(bucket, conf, S3_ENCRYPTION_CONTEXT);
if (encryptionContext == null) {
// look up the global value of the encryption context
encryptionContext = lookupPassword(null, conf, S3_ENCRYPTION_CONTEXT);
}
if (encryptionContext == null) {
// no encryption context, return ""
return "";
}
return encryptionContext;
}

/**
* Get the server-side encryption or client side encryption algorithm.
* This includes validation of the configuration, checking the state of
@@ -1493,7 +1570,10 @@ public static EncryptionSecrets buildEncryptionSecrets(String bucket,
LOG.debug("Data is unencrypted");
break;
}
return new EncryptionSecrets(encryptionMethod, encryptionKey);

String encryptionContext = getS3EncryptionContextBase64Encoded(bucket, conf,
encryptionMethod.requiresSecret());
return new EncryptionSecrets(encryptionMethod, encryptionKey, encryptionContext);
}

/**
@@ -1686,6 +1766,21 @@ public static Map<String, String> getTrimmedStringCollectionSplitByEquals(
final Configuration configuration,
final String name) {
String valueString = configuration.get(name);
return getTrimmedStringCollectionSplitByEquals(valueString);
}

/**
* Get the equal op (=) delimited key-value pairs of the <code>name</code> property as
* a collection of pair of <code>String</code>s, trimmed of the leading and trailing whitespace
* after delimiting the <code>name</code> by comma and new line separator.
* If no such property is specified then empty <code>Map</code> is returned.
*
* @param valueString the string containing the key-value pairs.
* @return property value as a <code>Map</code> of <code>String</code>s, or empty
* <code>Map</code>.
*/
private static Map<String, String> getTrimmedStringCollectionSplitByEquals(
final String valueString) {
if (null == valueString) {
return new HashMap<>();
}
Original file line number Diff line number Diff line change
@@ -61,4 +61,21 @@ public static Optional<String> getSSEAwsKMSKey(final EncryptionSecrets secrets)
return Optional.empty();
}
}

/**
* Gets the SSE-KMS context if present, else don't set it in the S3 request.
*
* @param secrets source of the encryption secrets.
* @return an optional AWS KMS encryption context to attach to a request.
*/
public static Optional<String> getSSEAwsKMSEncryptionContext(final EncryptionSecrets secrets) {
if ((secrets.getEncryptionMethod() == S3AEncryptionMethods.SSE_KMS
|| secrets.getEncryptionMethod() == S3AEncryptionMethods.DSSE_KMS)
&& secrets.hasEncryptionKey()
&& secrets.hasEncryptionContext()) {
return Optional.of(secrets.getEncryptionContext());
} else {
return Optional.empty();
}
}
}
Original file line number Diff line number Diff line change
@@ -67,6 +67,11 @@ public class EncryptionSecrets implements Writable, Serializable {
*/
private String encryptionKey = "";

/**
* Encryption context: base64-encoded UTF-8 string.
*/
private String encryptionContext = "";

/**
* This field isn't serialized/marshalled; it is rebuilt from the
* encryptionAlgorithm field.
@@ -84,23 +89,28 @@ public EncryptionSecrets() {
* Create a pair of secrets.
* @param encryptionAlgorithm algorithm enumeration.
* @param encryptionKey key/key reference.
* @param encryptionContext base64-encoded string with the encryption context key-value pairs.
* @throws IOException failure to initialize.
*/
public EncryptionSecrets(final S3AEncryptionMethods encryptionAlgorithm,
final String encryptionKey) throws IOException {
this(encryptionAlgorithm.getMethod(), encryptionKey);
final String encryptionKey,
final String encryptionContext) throws IOException {
this(encryptionAlgorithm.getMethod(), encryptionKey, encryptionContext);
}

/**
* Create a pair of secrets.
* @param encryptionAlgorithm algorithm name
* @param encryptionKey key/key reference.
* @param encryptionContext base64-encoded string with the encryption context key-value pairs.
* @throws IOException failure to initialize.
*/
public EncryptionSecrets(final String encryptionAlgorithm,
final String encryptionKey) throws IOException {
final String encryptionKey,
final String encryptionContext) throws IOException {
this.encryptionAlgorithm = encryptionAlgorithm;
this.encryptionKey = encryptionKey;
this.encryptionContext = encryptionContext;
init();
}

@@ -114,6 +124,7 @@ public void write(final DataOutput out) throws IOException {
new LongWritable(serialVersionUID).write(out);
Text.writeString(out, encryptionAlgorithm);
Text.writeString(out, encryptionKey);
Text.writeString(out, encryptionContext);
}

/**
@@ -132,6 +143,7 @@ public void readFields(final DataInput in) throws IOException {
}
encryptionAlgorithm = Text.readString(in, MAX_SECRET_LENGTH);
encryptionKey = Text.readString(in, MAX_SECRET_LENGTH);
encryptionContext = Text.readString(in);
init();
}

@@ -164,6 +176,10 @@ public String getEncryptionKey() {
return encryptionKey;
}

public String getEncryptionContext() {
return encryptionContext;
}

/**
* Does this instance have encryption options?
* That is: is the algorithm non-null.
@@ -181,6 +197,14 @@ public boolean hasEncryptionKey() {
return StringUtils.isNotEmpty(encryptionKey);
}

/**
* Does this instance have an encryption context?
* @return true if there's an encryption context.
*/
public boolean hasEncryptionContext() {
return StringUtils.isNotEmpty(encryptionContext);
}

@Override
public boolean equals(final Object o) {
if (this == o) {
@@ -191,12 +215,13 @@ public boolean equals(final Object o) {
}
final EncryptionSecrets that = (EncryptionSecrets) o;
return Objects.equals(encryptionAlgorithm, that.encryptionAlgorithm)
&& Objects.equals(encryptionKey, that.encryptionKey);
&& Objects.equals(encryptionKey, that.encryptionKey)
&& Objects.equals(encryptionContext, that.encryptionContext);
}

@Override
public int hashCode() {
return Objects.hash(encryptionAlgorithm, encryptionKey);
return Objects.hash(encryptionAlgorithm, encryptionKey, encryptionContext);
}

/**
Original file line number Diff line number Diff line change
@@ -282,11 +282,15 @@ protected void copyEncryptionParameters(HeadObjectResponse srcom,
// Set the KMS key if present, else S3 uses AWS managed key.
EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets)
.ifPresent(copyObjectRequestBuilder::ssekmsKeyId);
EncryptionSecretOperations.getSSEAwsKMSEncryptionContext(encryptionSecrets)
.ifPresent(copyObjectRequestBuilder::ssekmsEncryptionContext);
break;
case DSSE_KMS:
copyObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE);
EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets)
.ifPresent(copyObjectRequestBuilder::ssekmsKeyId);
EncryptionSecretOperations.getSSEAwsKMSEncryptionContext(encryptionSecrets)
.ifPresent(copyObjectRequestBuilder::ssekmsEncryptionContext);
break;
case SSE_C:
EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets)
@@ -371,11 +375,15 @@ private void putEncryptionParameters(PutObjectRequest.Builder putObjectRequestBu
// Set the KMS key if present, else S3 uses AWS managed key.
EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets)
.ifPresent(putObjectRequestBuilder::ssekmsKeyId);
EncryptionSecretOperations.getSSEAwsKMSEncryptionContext(encryptionSecrets)
.ifPresent(putObjectRequestBuilder::ssekmsEncryptionContext);
break;
case DSSE_KMS:
putObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE);
EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets)
.ifPresent(putObjectRequestBuilder::ssekmsKeyId);
EncryptionSecretOperations.getSSEAwsKMSEncryptionContext(encryptionSecrets)
.ifPresent(putObjectRequestBuilder::ssekmsEncryptionContext);
break;
case SSE_C:
EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets)
@@ -447,11 +455,15 @@ private void multipartUploadEncryptionParameters(
// Set the KMS key if present, else S3 uses AWS managed key.
EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets)
.ifPresent(mpuRequestBuilder::ssekmsKeyId);
EncryptionSecretOperations.getSSEAwsKMSEncryptionContext(encryptionSecrets)
.ifPresent(mpuRequestBuilder::ssekmsEncryptionContext);
break;
case DSSE_KMS:
mpuRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE);
EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets)
.ifPresent(mpuRequestBuilder::ssekmsKeyId);
EncryptionSecretOperations.getSSEAwsKMSEncryptionContext(encryptionSecrets)
.ifPresent(mpuRequestBuilder::ssekmsEncryptionContext);
break;
case SSE_C:
EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets)
Loading

0 comments on commit 71ffcd5

Please sign in to comment.