diff --git a/README.md b/README.md index f4a5d34..1bfae33 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ Build with the [Meltano Target SDK](https://sdk.meltano.com). "stream_name_path_override": "StreamName", "include_process_date": true, "append_date_to_prefix": false, + "partition_name_enabled": false, "append_date_to_prefix_grain": "day", "append_date_to_filename": true, "append_date_to_filename_grain": "microsecond", diff --git a/sample-config.json b/sample-config.json index fef1e08..adffd74 100644 --- a/sample-config.json +++ b/sample-config.json @@ -23,6 +23,7 @@ "stream_name_path_override": "StreamName", "include_process_date": true, "append_date_to_prefix": false, + "partition_name_enabled": false, "append_date_to_prefix_grain": "day", "append_date_to_filename": true, "append_date_to_filename_grain": "microsecond", diff --git a/target_s3/formats/format_base.py b/target_s3/formats/format_base.py index 58d37d2..7545b27 100644 --- a/target_s3/formats/format_base.py +++ b/target_s3/formats/format_base.py @@ -115,22 +115,25 @@ def create_key(self) -> str: file_name = "" if self.config["append_date_to_prefix"]: grain = DATE_GRAIN[self.config["append_date_to_prefix_grain"].lower()] - folder_path += self.create_folder_structure(batch_start, grain) + partition_name_enabled = False + if self.config["partition_name_enabled"]: + partition_name_enabled = self.config["partition_name_enabled"] + folder_path += self.create_folder_structure(batch_start, grain, partition_name_enabled) if self.config["append_date_to_filename"]: grain = DATE_GRAIN[self.config["append_date_to_filename_grain"].lower()] file_name += f"{self.create_file_structure(batch_start, grain)}" return f"{folder_path}{file_name}" - def create_folder_structure(self, batch_start: datetime, grain: int) -> str: + def create_folder_structure(self, batch_start: datetime, grain: int, partition_name_enabled: bool) -> str: ret = "" - ret += f"{batch_start.year}/" if grain <= 7 else "" - ret += f"{batch_start.month:02}/" if grain <= 6 else "" - ret += f"{batch_start.day:02}/" if grain <= 5 else "" - ret += f"{batch_start.hour:02}/" if grain <= 4 else "" - ret += f"{batch_start.minute:02}/" if grain <= 3 else "" - ret += f"{batch_start.second:02}/" if grain <= 4 else "" - ret += f"{batch_start.microsecond}/" if grain <= 1 else "" + ret += f"{'year=' if partition_name_enabled else ''}{batch_start.year}/" if grain <= 7 else "" + ret += f"{'month=' if partition_name_enabled else ''}{batch_start.month:02}/" if grain <= 6 else "" + ret += f"{'day=' if partition_name_enabled else ''}{batch_start.day:02}/" if grain <= 5 else "" + ret += f"{'hour=' if partition_name_enabled else ''}{batch_start.hour:02}/" if grain <= 4 else "" + ret += f"{'minute=' if partition_name_enabled else ''}{batch_start.minute:02}/" if grain <= 3 else "" + ret += f"{'second=' if partition_name_enabled else ''}{batch_start.second:02}/" if grain <= 4 else "" + ret += f"{'microsecond=' if partition_name_enabled else ''}{batch_start.microsecond}/" if grain <= 1 else "" return ret def create_file_structure(self, batch_start: datetime, grain: int) -> str: diff --git a/target_s3/target.py b/target_s3/target.py index cb42d75..7e3bc61 100644 --- a/target_s3/target.py +++ b/target_s3/target.py @@ -130,6 +130,12 @@ class Targets3(Target): description="A flag to append the date to the key prefix.", default=True, ), + th.Property( + "partition_name_enabled", + th.BooleanType, + description="A flag (only works if append_date_to_prefix is enabled) to have partitioning name formatted e.g. 'year=2023/month=01/day=01'.", + default=False, + ), th.Property( "append_date_to_prefix_grain", th.StringType,