diff --git a/target_s3/formats/format_parquet.py b/target_s3/formats/format_parquet.py index 6d9eae2..4f99eb2 100644 --- a/target_s3/formats/format_parquet.py +++ b/target_s3/formats/format_parquet.py @@ -8,29 +8,19 @@ class FormatParquet(FormatBase): def __init__(self, config, context) -> None: super().__init__(config, context, "parquet") - self.create_filesystem( - config.get("aws_profile_name", None), - config.get("aws_region", None), - ) + self.create_filesystem() def create_filesystem( self, - aws_profile: str = None, - aws_region: str = None, ) -> None: """Creates a pyarrow FileSystem object for accessing S3.""" - aws_region = self.aws_region if aws_region is None else aws_region try: - if self.session: - self.file_system = fs.S3FileSystem( - access_key=self.session.get_credentials().access_key, - secret_key=self.session.get_credentials().secret_key, - session_token=self.session.get_credentials().token, - ) - else: - self.file_system = fs.S3FileSystem( - region=aws_region, - ) + self.file_system = fs.S3FileSystem( + access_key=self.session.get_credentials().access_key, + secret_key=self.session.get_credentials().secret_key, + session_token=self.session.get_credentials().token, + region=self.session.region_name, + ) except Exception as e: self.logger.error("Failed to create parquet file system.") self.logger.error(e) diff --git a/target_s3/target.py b/target_s3/target.py index 83e6421..9d26148 100644 --- a/target_s3/target.py +++ b/target_s3/target.py @@ -65,29 +65,34 @@ class Targets3(Target): "append_date_to_prefix", th.BooleanType, description="A flag to append the date to the key prefix.", + default=True, ), th.Property( "append_date_to_prefix_grain", th.StringType, description="The grain of the date to append to the prefix.", allowed_values=DATE_GRAIN.keys(), + default="day", ), th.Property( "append_date_to_filename", th.BooleanType, description="A flag to append the date to the key filename.", + default=True, ), th.Property( "append_date_to_filename_grain", th.StringType, description="The grain of the date to append to the filename.", allowed_values=DATE_GRAIN.keys(), + default="day", ), th.Property( "format_type", th.StringType, description="The format of the storage object.", allowed_values=sinks.FORMAT_TYPE.keys(), + required=True, ), th.Property( "flatten_records",