From fd5a0f86dbc5731e897d5e8fc4ecd1fef18bc71b Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Wed, 3 Jan 2024 08:38:28 -0600 Subject: [PATCH 1/4] 20-minutely schedule for VZ OCR ETL --- dags/vz_cr3_extract_ocr_narrative.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dags/vz_cr3_extract_ocr_narrative.py b/dags/vz_cr3_extract_ocr_narrative.py index fcd18ea3..2936eab1 100644 --- a/dags/vz_cr3_extract_ocr_narrative.py +++ b/dags/vz_cr3_extract_ocr_narrative.py @@ -44,7 +44,9 @@ dag_id=f"vz_cr3_ocr_narrative_extract_{DEPLOYMENT_ENVIRONMENT}", default_args=default_args, # Every 5 minutes, at 8A, 9A, and 10A - schedule_interval="*/5 8-10 * * *" if DEPLOYMENT_ENVIRONMENT == "production" else None, + schedule_interval="*/20 * * * *" + if DEPLOYMENT_ENVIRONMENT == "production" + else None, tags=["repo:atd-vz-data", "vision-zero"], catchup=False, ) as dag: From f4cadbba3cbf555556920dfe82b60d691d77e5c9 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Wed, 3 Jan 2024 08:39:10 -0600 Subject: [PATCH 2/4] 20 minute dag execution timeout --- dags/vz_cr3_extract_ocr_narrative.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/vz_cr3_extract_ocr_narrative.py b/dags/vz_cr3_extract_ocr_narrative.py index 2936eab1..af44492b 100644 --- a/dags/vz_cr3_extract_ocr_narrative.py +++ b/dags/vz_cr3_extract_ocr_narrative.py @@ -17,7 +17,7 @@ "email_on_failure": False, "email_on_retry": False, "retries": 0, - "execution_timeout": duration(minutes=5), + "execution_timeout": duration(minutes=20), "on_failure_callback": task_fail_slack_alert, } From a4c42bf4c20a5d3d9be79753c010e05f1235b930 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Wed, 3 Jan 2024 08:39:20 -0600 Subject: [PATCH 3/4] lint with black --- dags/vz_cr3_extract_ocr_narrative.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dags/vz_cr3_extract_ocr_narrative.py b/dags/vz_cr3_extract_ocr_narrative.py index af44492b..bc5ba35d 100644 --- a/dags/vz_cr3_extract_ocr_narrative.py +++ b/dags/vz_cr3_extract_ocr_narrative.py @@ -29,7 +29,7 @@ "HASURA_ADMIN_KEY": { "opitem": "Vision Zero CRIS Import", "opfield": "production.GraphQL Endpoint key", - }, + }, "AWS_ACCESS_KEY_ID": { "opitem": "Vision Zero CRIS Import", "opfield": "production.AWS Access key", @@ -37,7 +37,7 @@ "AWS_SECRET_ACCESS_KEY": { "opitem": "Vision Zero CRIS Import", "opfield": "production.AWS Secret key", - } + }, } with DAG( @@ -50,16 +50,18 @@ tags=["repo:atd-vz-data", "vision-zero"], catchup=False, ) as dag: + @task( task_id="get_env_vars", execution_timeout=duration(seconds=30), ) def get_env_vars(): from utils.onepassword import load_dict + return load_dict(REQUIRED_SECRETS) env_vars = get_env_vars() - + DockerOperator( task_id="ocr_narrative_extract", image="atddocker/atd-vz-cr3-extract:production", From da545f9a52630a645d64240eb29887b29f6ef0d9 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Wed, 3 Jan 2024 09:02:10 -0600 Subject: [PATCH 4/4] Being particular in how this is lined up --- dags/vz_cr3_extract_ocr_narrative.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dags/vz_cr3_extract_ocr_narrative.py b/dags/vz_cr3_extract_ocr_narrative.py index bc5ba35d..642b2968 100644 --- a/dags/vz_cr3_extract_ocr_narrative.py +++ b/dags/vz_cr3_extract_ocr_narrative.py @@ -40,16 +40,15 @@ }, } +# fmt: off with DAG( dag_id=f"vz_cr3_ocr_narrative_extract_{DEPLOYMENT_ENVIRONMENT}", default_args=default_args, - # Every 5 minutes, at 8A, 9A, and 10A - schedule_interval="*/20 * * * *" - if DEPLOYMENT_ENVIRONMENT == "production" - else None, + schedule_interval="*/20 * * * *" if DEPLOYMENT_ENVIRONMENT == "production" else None, tags=["repo:atd-vz-data", "vision-zero"], catchup=False, ) as dag: +# fmt: on @task( task_id="get_env_vars",