Skip to content

Commit

Permalink
fix(databricks): fix time format for use in file name
Browse files Browse the repository at this point in the history
  • Loading branch information
pbullhove committed Oct 4, 2024
1 parent 26adaad commit a7dbf41
Showing 1 changed file with 6 additions and 3 deletions.
9 changes: 6 additions & 3 deletions databricks/Havvarsel - Ingest as Bronze.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,17 @@ def fetch_ocean_temperature_preditions(depth_index, lat, lon):
# COMMAND ----------

import requests
from pyspark.sql.functions import lit, current_timestamp
from pyspark.sql.functions import lit, current_timestamp, date_format
from datetime import datetime
from helpers.adls_utils import save_df_as_delta, get_adls_folder_path, connect_to_adls
connect_to_adls()

fetch_time = current_timestamp()
bronze_df_file_name = f"bronze/hav_temperature_projection_{fetch_time}" # have a new bronze for each fetch date
fetch_time = datetime.utcnow()
formatted_time = fetch_time.strftime("%Y-%m-%dT%H%M%S")

bronze_df_file_name = f"bronze/hav_temperature_projection_{formatted_time}" # have a new bronze for each fetch date

print(bronze_df_file_name)

dbutils.fs.rm(f"{get_adls_folder_path()}/{bronze_df_file_name}", recurse=True) # delete old in order to remove duplicates

Expand Down

0 comments on commit a7dbf41

Please sign in to comment.