Allow S3 configuration for S3 buckets using environment variables.

Signed-off-by: Pascal Spörri <[email protected]>
IBM · May 22, 2024 · 3d6d096 · 3d6d096
1 parent 73420fd
commit 3d6d096
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 12 deletions.
diff --git a/src/main/java/com/ibm/geds/hdfs/Constants.java b/src/main/java/com/ibm/geds/hdfs/Constants.java
@@ -17,4 +17,5 @@ public class Constants {
     public static final String AVAILABLE_LOCAL_STORAGE = "available_local_storage";
     public static final String AVAILABLE_LOCAL_MEMORY = "available_local_memory";
     public static final String IO_THREAD_POOL_SIZE = "io_thread_pool_size";
+    public static final String CONFIGURE_S3_USING_ENV = "configure_s3_using_env";
 }
diff --git a/src/main/java/com/ibm/geds/hdfs/GEDSInstance.java b/src/main/java/com/ibm/geds/hdfs/GEDSInstance.java
@@ -11,6 +11,7 @@
 import org.apache.hadoop.conf.Configuration;
 
 public class GEDSInstance {
+    private static boolean configureS3usingEnv;
     private static GEDS instance;
     private static GEDSConfig instanceConfig;
 
@@ -96,6 +97,9 @@ public synchronized static GEDSConfig getConfig(Configuration conf) {
             if (io_thread_pool_size != 0) {
                 instanceConfig.set(Constants.IO_THREAD_POOL_SIZE, io_thread_pool_size);
             }
+
+            configureS3usingEnv = conf.getLong(Constants.GEDS_PREFIX + Constants.CONFIGURE_S3_USING_ENV,
+                    getEnvLong("GEDS_CONFIGURE_S3_USING_ENV", 0)) == 1;
         }
         return instanceConfig;
     }
@@ -108,18 +112,36 @@ public synchronized static GEDS initialize(String bucket, Configuration conf) {
         } catch (Exception e) {
             // Always initialize bucket.
         }
-        String bucketAccessKey = conf.get(Constants.GEDS_PREFIX + bucket + ".accessKey");
-        String bucketSecretKey = conf.get(Constants.GEDS_PREFIX + bucket + ".secretKey");
-        String bucketEndpoint = conf.get(Constants.GEDS_PREFIX + bucket + ".endpoint");
-
-        boolean hasAccessKey = bucketAccessKey != null;
-        boolean hasSecretKey = bucketSecretKey != null;
-        boolean hasBucketEndpoint = bucketEndpoint != null;
-        if (hasAccessKey && hasSecretKey && hasBucketEndpoint) {
-            geds.registerObjectStoreConfig(bucket, bucketEndpoint, bucketAccessKey, bucketSecretKey);
-        } else if (hasAccessKey || hasSecretKey || hasBucketEndpoint) {
-            throw new RuntimeException("Bucket " + bucket
-                    + " has either an accessKey, secretKey or an endpoint registered. To map the bucket to S3 all variables need to be configured.");
+        if (configureS3usingEnv) {
+            String accessKey = System.getenv("AWS_ACCESS_KEY_ID");
+            String secretKey = System.getenv("AWS_SECRET_ACCESS_KEY");
+            String endpoint = System.getenv("AWS_ENDPOINT_URL");
+
+            boolean hasAccessKey = accessKey != null;
+            boolean hasSecretKey = secretKey != null;
+            boolean hasEndpoint = endpoint != null;
+            if (hasAccessKey && hasSecretKey && hasEndpoint) {
+                geds.registerObjectStoreConfig(bucket, endpoint, accessKey, secretKey);
+            } else if (hasAccessKey || hasSecretKey || hasEndpoint) {
+                throw new RuntimeException(Constants.CONFIGURE_S3_USING_ENV
+                        + "is enabled and either AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY or AWS_ENDPOINT_URL is set.");
+            }
+        } else {
+            String bucketAccessKey = conf.get(Constants.GEDS_PREFIX + bucket + ".accessKey",
+                    getEnv("AWS_ACCESS_KEY_ID"));
+            String bucketSecretKey = conf.get(Constants.GEDS_PREFIX + bucket + ".secretKey",
+                    getEnv("AWS_ACCESS_KEY_ID"));
+            String bucketEndpoint = conf.get(Constants.GEDS_PREFIX + bucket + ".endpoint", getEnv("AWS_ENDPOINT_URL"));
+
+            boolean hasAccessKey = bucketAccessKey != null;
+            boolean hasSecretKey = bucketSecretKey != null;
+            boolean hasBucketEndpoint = bucketEndpoint != null;
+            if (hasAccessKey && hasSecretKey && hasBucketEndpoint) {
+                geds.registerObjectStoreConfig(bucket, bucketEndpoint, bucketAccessKey, bucketSecretKey);
+            } else if (hasAccessKey || hasSecretKey || hasBucketEndpoint) {
+                throw new RuntimeException("Bucket " + bucket
+                        + " has either an accessKey, secretKey or an endpoint registered. To map the bucket to S3 all variables need to be configured.");
+            }
         }
         return geds;
     }