Merge pull request #5 from pnadolny13/fix_parquet_region

Add parquet region
crowemi · Apr 3, 2023 · b4b5d46 · b4b5d46
2 parents 6dcf59f + 5eba4e8
commit b4b5d46
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 17 deletions.
diff --git a/target_s3/formats/format_parquet.py b/target_s3/formats/format_parquet.py
@@ -8,29 +8,19 @@
 class FormatParquet(FormatBase):
     def __init__(self, config, context) -> None:
         super().__init__(config, context, "parquet")
-        self.create_filesystem(
-            config.get("aws_profile_name", None),
-            config.get("aws_region", None),
-        )
+        self.create_filesystem()
 
     def create_filesystem(
         self,
-        aws_profile: str = None,
-        aws_region: str = None,
     ) -> None:
         """Creates a pyarrow FileSystem object for accessing S3."""
-        aws_region = self.aws_region if aws_region is None else aws_region
         try:
-            if self.session:
-                self.file_system = fs.S3FileSystem(
-                    access_key=self.session.get_credentials().access_key,
-                    secret_key=self.session.get_credentials().secret_key,
-                    session_token=self.session.get_credentials().token,
-                )
-            else:
-                self.file_system = fs.S3FileSystem(
-                    region=aws_region,
-                )
+            self.file_system = fs.S3FileSystem(
+                access_key=self.session.get_credentials().access_key,
+                secret_key=self.session.get_credentials().secret_key,
+                session_token=self.session.get_credentials().token,
+                region=self.session.region_name,
+            )
         except Exception as e:
             self.logger.error("Failed to create parquet file system.")
             self.logger.error(e)

diff --git a/target_s3/target.py b/target_s3/target.py
@@ -65,29 +65,34 @@ class Targets3(Target):
             "append_date_to_prefix",
             th.BooleanType,
             description="A flag to append the date to the key prefix.",
+            default=True,
         ),
         th.Property(
             "append_date_to_prefix_grain",
             th.StringType,
             description="The grain of the date to append to the prefix.",
             allowed_values=DATE_GRAIN.keys(),
+            default="day",
         ),
         th.Property(
             "append_date_to_filename",
             th.BooleanType,
             description="A flag to append the date to the key filename.",
+            default=True,
         ),
         th.Property(
             "append_date_to_filename_grain",
             th.StringType,
             description="The grain of the date to append to the filename.",
             allowed_values=DATE_GRAIN.keys(),
+            default="day",
         ),
         th.Property(
             "format_type",
             th.StringType,
             description="The format of the storage object.",
             allowed_values=sinks.FORMAT_TYPE.keys(),
+            required=True,
         ),
         th.Property(
             "flatten_records",