From f6b9bf92b38d5055e61ecb7d75a0f35e36aae0a2 Mon Sep 17 00:00:00 2001 From: DonHaul Date: Mon, 29 Jul 2024 15:22:07 +0200 Subject: [PATCH] restart actions: added test for airflow utils --- .../backoffice/workflows/airflow_utils.py | 13 +- ...AirflowUtils.test_delete_workflow_dag.yaml | 112 ++++++++ ...tAirflowUtils.test_find_executed_dags.yaml | 183 +++++++++++++ ...TestAirflowUtils.test_find_failed_dag.yaml | 184 +++++++++++++ ...irflowUtils.test_restart_failed_tasks.yaml | 230 ++++++++++++++++ ...rflowUtils.test_restart_workflow_dags.yaml | 257 ++++++++++++++++++ ...AirflowUtils.test_trigger_airflow_dag.yaml | 76 ++++++ .../workflows/tests/test_airflow_utils.py | 60 ++++ 8 files changed, 1108 insertions(+), 7 deletions(-) create mode 100644 backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_delete_workflow_dag.yaml create mode 100644 backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_find_executed_dags.yaml create mode 100644 backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_find_failed_dag.yaml create mode 100644 backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_restart_failed_tasks.yaml create mode 100644 backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_restart_workflow_dags.yaml create mode 100644 backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_trigger_airflow_dag.yaml create mode 100644 backoffice/backoffice/workflows/tests/test_airflow_utils.py diff --git a/backoffice/backoffice/workflows/airflow_utils.py b/backoffice/backoffice/workflows/airflow_utils.py index df34899d..d97ef4b4 100644 --- a/backoffice/backoffice/workflows/airflow_utils.py +++ b/backoffice/backoffice/workflows/airflow_utils.py @@ -2,7 +2,7 @@ from os import environ import requests -from django.http import JsonResponse +from django.http import HttpResponse, JsonResponse from requests.exceptions import RequestException from rest_framework import status @@ -57,7 +57,6 @@ def restart_failed_tasks(workflow): """ dag_id = find_failed_dag(workflow) - # assumes current task is one of the failed tasks data = { "dry_run": False, @@ -94,7 +93,7 @@ def find_executed_dags(workflow): headers=AIRFLOW_HEADERS, ) if response.status_code == status.HTTP_200_OK: - executed_dags_for_workflow[dag_id] = response.content + executed_dags_for_workflow[dag_id] = response.json() return executed_dags_for_workflow @@ -109,7 +108,7 @@ def find_failed_dag(workflow): executed_dags_for_workflow = find_executed_dags(workflow) for dag, dag_data in executed_dags_for_workflow.items(): - if dag_data["status"] == "failed": + if dag_data["state"] == "failed": return dag @@ -125,12 +124,12 @@ def delete_workflow_dag(dag_id, workflow): headers=AIRFLOW_HEADERS, ) response.raise_for_status() + return HttpResponse() except RequestException: - data = {"error": response.json()} - return JsonResponse(data, status=status.HTTP_424_FAILED_DEPENDENCY) + return HttpResponse(status=status.HTTP_424_FAILED_DEPENDENCY) -def restart_workflow_dags(workflow, params): +def restart_workflow_dags(workflow, params=None): """Restarts dags of a given workflow. :param workflow: workflow whoose dags should be restarted diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_delete_workflow_dag.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_delete_workflow_dag.yaml new file mode 100644 index 00000000..9c540819 --- /dev/null +++ b/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_delete_workflow_dag.yaml @@ -0,0 +1,112 @@ +interactions: +- request: + body: '{"dag_run_id": "6569b2e7-0ed2-4bcd-91f4-31f2f45f8acf", "conf": {"workflow_id": + "6569b2e7-0ed2-4bcd-91f4-31f2f45f8acf"}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '119' + Content-Type: + - application/json + method: POST + uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"6569b2e7-0ed2-4bcd-91f4-31f2f45f8acf\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"6569b2e7-0ed2-4bcd-91f4-31f2f45f8acf\",\n \"data_interval_end\": \"2024-07-29T13:59:32.188032+00:00\",\n + \ \"data_interval_start\": \"2024-07-29T13:59:32.188032+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-07-29T13:59:32.188032+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-29T13:59:32.188032+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 29 Jul 2024 13:59:32 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + Content-Type: + - application/json + method: DELETE + uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/6569b2e7-0ed2-4bcd-91f4-31f2f45f8acf + response: + body: + string: '' + headers: + Connection: + - close + Content-Type: + - application/json + Date: + - Mon, 29 Jul 2024 13:59:32 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 204 + message: NO CONTENT +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + Content-Type: + - application/json + method: DELETE + uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/6569b2e7-0ed2-4bcd-91f4-31f2f45f8acf + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_initialization_dag' + and DagRun ID: '6569b2e7-0ed2-4bcd-91f4-31f2f45f8acf' not found\",\n \"status\": + 404,\n \"title\": \"Not Found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '293' + Content-Type: + - application/problem+json + Date: + - Mon, 29 Jul 2024 13:59:32 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +version: 1 diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_find_executed_dags.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_find_executed_dags.yaml new file mode 100644 index 00000000..ab25e868 --- /dev/null +++ b/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_find_executed_dags.yaml @@ -0,0 +1,183 @@ +interactions: +- request: + body: '{"dag_run_id": "41f9047d-f7a4-48cc-b138-3e2fa9e19802", "conf": {"workflow_id": + "41f9047d-f7a4-48cc-b138-3e2fa9e19802"}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '119' + Content-Type: + - application/json + method: POST + uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"41f9047d-f7a4-48cc-b138-3e2fa9e19802\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"41f9047d-f7a4-48cc-b138-3e2fa9e19802\",\n \"data_interval_end\": \"2024-07-29T13:59:32.491551+00:00\",\n + \ \"data_interval_start\": \"2024-07-29T13:59:32.491551+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-07-29T13:59:32.491551+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-29T13:59:32.491551+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 29 Jul 2024 13:59:32 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Type: + - application/json + method: GET + uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/41f9047d-f7a4-48cc-b138-3e2fa9e19802 + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"41f9047d-f7a4-48cc-b138-3e2fa9e19802\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"41f9047d-f7a4-48cc-b138-3e2fa9e19802\",\n \"data_interval_end\": \"2024-07-29T13:59:32.491551+00:00\",\n + \ \"data_interval_start\": \"2024-07-29T13:59:32.491551+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-07-29T13:59:32.491551+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-29T13:59:32.491551+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 29 Jul 2024 13:59:32 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Type: + - application/json + method: GET + uri: http://host.docker.internal:8080/api/v1/dags/author_create_approved_dag/dagRuns/41f9047d-f7a4-48cc-b138-3e2fa9e19802 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' + and DagRun ID: '41f9047d-f7a4-48cc-b138-3e2fa9e19802' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 29 Jul 2024 13:59:32 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Type: + - application/json + method: GET + uri: http://host.docker.internal:8080/api/v1/dags/author_create_rejected_dag/dagRuns/41f9047d-f7a4-48cc-b138-3e2fa9e19802 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' + and DagRun ID: '41f9047d-f7a4-48cc-b138-3e2fa9e19802' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 29 Jul 2024 13:59:32 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + Content-Type: + - application/json + method: DELETE + uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/41f9047d-f7a4-48cc-b138-3e2fa9e19802 + response: + body: + string: '' + headers: + Connection: + - close + Content-Type: + - application/json + Date: + - Mon, 29 Jul 2024 13:59:32 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 204 + message: NO CONTENT +version: 1 diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_find_failed_dag.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_find_failed_dag.yaml new file mode 100644 index 00000000..9d5153b7 --- /dev/null +++ b/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_find_failed_dag.yaml @@ -0,0 +1,184 @@ +interactions: +- request: + body: '{"dag_run_id": "d100b8c4-6099-4f8b-b8fc-12f33538e14e", "conf": {"workflow_id": + "d100b8c4-6099-4f8b-b8fc-12f33538e14e"}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '119' + Content-Type: + - application/json + method: POST + uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"d100b8c4-6099-4f8b-b8fc-12f33538e14e\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"d100b8c4-6099-4f8b-b8fc-12f33538e14e\",\n \"data_interval_end\": \"2024-07-29T13:59:32.933753+00:00\",\n + \ \"data_interval_start\": \"2024-07-29T13:59:32.933753+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-07-29T13:59:32.933753+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-29T13:59:32.933753+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 29 Jul 2024 13:59:32 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Type: + - application/json + method: GET + uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/d100b8c4-6099-4f8b-b8fc-12f33538e14e + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"d100b8c4-6099-4f8b-b8fc-12f33538e14e\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"d100b8c4-6099-4f8b-b8fc-12f33538e14e\",\n \"data_interval_end\": \"2024-07-29T13:59:32.933753+00:00\",\n + \ \"data_interval_start\": \"2024-07-29T13:59:32.933753+00:00\",\n \"end_date\": + \"2024-07-29T13:59:50.949911+00:00\",\n \"execution_date\": \"2024-07-29T13:59:32.933753+00:00\",\n + \ \"external_trigger\": true,\n \"last_scheduling_decision\": \"2024-07-29T13:59:50.947511+00:00\",\n + \ \"logical_date\": \"2024-07-29T13:59:32.933753+00:00\",\n \"note\": null,\n + \ \"run_type\": \"manual\",\n \"start_date\": \"2024-07-29T13:59:33.453020+00:00\",\n + \ \"state\": \"failed\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '669' + Content-Type: + - application/json + Date: + - Mon, 29 Jul 2024 13:59:53 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Type: + - application/json + method: GET + uri: http://host.docker.internal:8080/api/v1/dags/author_create_approved_dag/dagRuns/d100b8c4-6099-4f8b-b8fc-12f33538e14e + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' + and DagRun ID: 'd100b8c4-6099-4f8b-b8fc-12f33538e14e' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 29 Jul 2024 13:59:53 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Type: + - application/json + method: GET + uri: http://host.docker.internal:8080/api/v1/dags/author_create_rejected_dag/dagRuns/d100b8c4-6099-4f8b-b8fc-12f33538e14e + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' + and DagRun ID: 'd100b8c4-6099-4f8b-b8fc-12f33538e14e' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 29 Jul 2024 13:59:53 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + Content-Type: + - application/json + method: DELETE + uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/d100b8c4-6099-4f8b-b8fc-12f33538e14e + response: + body: + string: '' + headers: + Connection: + - close + Content-Type: + - application/json + Date: + - Mon, 29 Jul 2024 13:59:53 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 204 + message: NO CONTENT +version: 1 diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_restart_failed_tasks.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_restart_failed_tasks.yaml new file mode 100644 index 00000000..84f42536 --- /dev/null +++ b/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_restart_failed_tasks.yaml @@ -0,0 +1,230 @@ +interactions: +- request: + body: '{"dag_run_id": "e0250817-ca9c-451c-82d6-d5ab4e07ce4a", "conf": {"workflow_id": + "e0250817-ca9c-451c-82d6-d5ab4e07ce4a"}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '119' + Content-Type: + - application/json + method: POST + uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"e0250817-ca9c-451c-82d6-d5ab4e07ce4a\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"e0250817-ca9c-451c-82d6-d5ab4e07ce4a\",\n \"data_interval_end\": \"2024-07-29T13:59:53.376034+00:00\",\n + \ \"data_interval_start\": \"2024-07-29T13:59:53.376034+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-07-29T13:59:53.376034+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-29T13:59:53.376034+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 29 Jul 2024 13:59:53 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Type: + - application/json + method: GET + uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/e0250817-ca9c-451c-82d6-d5ab4e07ce4a + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"e0250817-ca9c-451c-82d6-d5ab4e07ce4a\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"e0250817-ca9c-451c-82d6-d5ab4e07ce4a\",\n \"data_interval_end\": \"2024-07-29T13:59:53.376034+00:00\",\n + \ \"data_interval_start\": \"2024-07-29T13:59:53.376034+00:00\",\n \"end_date\": + \"2024-07-29T14:00:11.782158+00:00\",\n \"execution_date\": \"2024-07-29T13:59:53.376034+00:00\",\n + \ \"external_trigger\": true,\n \"last_scheduling_decision\": \"2024-07-29T14:00:11.781108+00:00\",\n + \ \"logical_date\": \"2024-07-29T13:59:53.376034+00:00\",\n \"note\": null,\n + \ \"run_type\": \"manual\",\n \"start_date\": \"2024-07-29T13:59:53.746711+00:00\",\n + \ \"state\": \"failed\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '669' + Content-Type: + - application/json + Date: + - Mon, 29 Jul 2024 14:00:13 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Type: + - application/json + method: GET + uri: http://host.docker.internal:8080/api/v1/dags/author_create_approved_dag/dagRuns/e0250817-ca9c-451c-82d6-d5ab4e07ce4a + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' + and DagRun ID: 'e0250817-ca9c-451c-82d6-d5ab4e07ce4a' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 29 Jul 2024 14:00:13 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Type: + - application/json + method: GET + uri: http://host.docker.internal:8080/api/v1/dags/author_create_rejected_dag/dagRuns/e0250817-ca9c-451c-82d6-d5ab4e07ce4a + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' + and DagRun ID: 'e0250817-ca9c-451c-82d6-d5ab4e07ce4a' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 29 Jul 2024 14:00:13 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: '{"dry_run": false, "dag_run_id": "e0250817-ca9c-451c-82d6-d5ab4e07ce4a", + "reset_dag_runs": false, "only_failed": true}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '118' + Content-Type: + - application/json + method: POST + uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/clearTaskInstances + response: + body: + string: "{\n \"task_instances\": [\n {\n \"dag_id\": \"author_create_initialization_dag\",\n + \ \"dag_run_id\": \"e0250817-ca9c-451c-82d6-d5ab4e07ce4a\",\n \"execution_date\": + \"2024-07-29T13:59:53.376034+00:00\",\n \"task_id\": \"set_schema\"\n + \ },\n {\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"e0250817-ca9c-451c-82d6-d5ab4e07ce4a\",\n \"execution_date\": \"2024-07-29T13:59:53.376034+00:00\",\n + \ \"task_id\": \"set_workflow_status_to_running\"\n },\n {\n \"dag_id\": + \"author_create_initialization_dag\",\n \"dag_run_id\": \"e0250817-ca9c-451c-82d6-d5ab4e07ce4a\",\n + \ \"execution_date\": \"2024-07-29T13:59:53.376034+00:00\",\n \"task_id\": + \"create_author_create_user_ticket\"\n },\n {\n \"dag_id\": \"author_create_initialization_dag\",\n + \ \"dag_run_id\": \"e0250817-ca9c-451c-82d6-d5ab4e07ce4a\",\n \"execution_date\": + \"2024-07-29T13:59:53.376034+00:00\",\n \"task_id\": \"set_author_create_workflow_status_to_approval\"\n + \ }\n ]\n}\n" + headers: + Connection: + - close + Content-Length: + - '966' + Content-Type: + - application/json + Date: + - Mon, 29 Jul 2024 14:00:13 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + Content-Type: + - application/json + method: DELETE + uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/e0250817-ca9c-451c-82d6-d5ab4e07ce4a + response: + body: + string: '' + headers: + Connection: + - close + Content-Type: + - application/json + Date: + - Mon, 29 Jul 2024 14:00:13 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 204 + message: NO CONTENT +version: 1 diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_restart_workflow_dags.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_restart_workflow_dags.yaml new file mode 100644 index 00000000..4039fbb1 --- /dev/null +++ b/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_restart_workflow_dags.yaml @@ -0,0 +1,257 @@ +interactions: +- request: + body: '{"dag_run_id": "f2a695e7-d915-46f9-9a5d-2ce688ca7c22", "conf": {"workflow_id": + "f2a695e7-d915-46f9-9a5d-2ce688ca7c22"}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '119' + Content-Type: + - application/json + method: POST + uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"f2a695e7-d915-46f9-9a5d-2ce688ca7c22\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"f2a695e7-d915-46f9-9a5d-2ce688ca7c22\",\n \"data_interval_end\": \"2024-07-29T14:00:14.071608+00:00\",\n + \ \"data_interval_start\": \"2024-07-29T14:00:14.071608+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-07-29T14:00:14.071608+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-29T14:00:14.071608+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 29 Jul 2024 14:00:14 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Type: + - application/json + method: GET + uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/f2a695e7-d915-46f9-9a5d-2ce688ca7c22 + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"f2a695e7-d915-46f9-9a5d-2ce688ca7c22\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"f2a695e7-d915-46f9-9a5d-2ce688ca7c22\",\n \"data_interval_end\": \"2024-07-29T14:00:14.071608+00:00\",\n + \ \"data_interval_start\": \"2024-07-29T14:00:14.071608+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-07-29T14:00:14.071608+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-29T14:00:14.071608+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 29 Jul 2024 14:00:14 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Type: + - application/json + method: GET + uri: http://host.docker.internal:8080/api/v1/dags/author_create_approved_dag/dagRuns/f2a695e7-d915-46f9-9a5d-2ce688ca7c22 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag' + and DagRun ID: 'f2a695e7-d915-46f9-9a5d-2ce688ca7c22' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 29 Jul 2024 14:00:14 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Type: + - application/json + method: GET + uri: http://host.docker.internal:8080/api/v1/dags/author_create_rejected_dag/dagRuns/f2a695e7-d915-46f9-9a5d-2ce688ca7c22 + response: + body: + string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag' + and DagRun ID: 'f2a695e7-d915-46f9-9a5d-2ce688ca7c22' not found\",\n \"status\": + 404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '294' + Content-Type: + - application/problem+json + Date: + - Mon, 29 Jul 2024 14:00:14 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 404 + message: NOT FOUND +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + Content-Type: + - application/json + method: DELETE + uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/f2a695e7-d915-46f9-9a5d-2ce688ca7c22 + response: + body: + string: '' + headers: + Connection: + - close + Content-Type: + - application/json + Date: + - Mon, 29 Jul 2024 14:00:14 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 204 + message: NO CONTENT +- request: + body: '{"dag_run_id": "f2a695e7-d915-46f9-9a5d-2ce688ca7c22", "conf": {"workflow_id": + "f2a695e7-d915-46f9-9a5d-2ce688ca7c22"}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '119' + Content-Type: + - application/json + method: POST + uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"f2a695e7-d915-46f9-9a5d-2ce688ca7c22\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"f2a695e7-d915-46f9-9a5d-2ce688ca7c22\",\n \"data_interval_end\": \"2024-07-29T14:00:14.383997+00:00\",\n + \ \"data_interval_start\": \"2024-07-29T14:00:14.383997+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-07-29T14:00:14.383997+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-29T14:00:14.383997+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 29 Jul 2024 14:00:14 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + Content-Type: + - application/json + method: DELETE + uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/f2a695e7-d915-46f9-9a5d-2ce688ca7c22 + response: + body: + string: '' + headers: + Connection: + - close + Content-Type: + - application/json + Date: + - Mon, 29 Jul 2024 14:00:14 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 204 + message: NO CONTENT +version: 1 diff --git a/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_trigger_airflow_dag.yaml b/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_trigger_airflow_dag.yaml new file mode 100644 index 00000000..8379efa6 --- /dev/null +++ b/backoffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_trigger_airflow_dag.yaml @@ -0,0 +1,76 @@ +interactions: +- request: + body: '{"dag_run_id": "7156f917-a901-465e-9782-e205999caf8e", "conf": {"workflow_id": + "7156f917-a901-465e-9782-e205999caf8e"}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '119' + Content-Type: + - application/json + method: POST + uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns + response: + body: + string: "{\n \"conf\": {\n \"workflow_id\": \"7156f917-a901-465e-9782-e205999caf8e\"\n + \ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\": + \"7156f917-a901-465e-9782-e205999caf8e\",\n \"data_interval_end\": \"2024-07-29T14:00:14.606223+00:00\",\n + \ \"data_interval_start\": \"2024-07-29T14:00:14.606223+00:00\",\n \"end_date\": + null,\n \"execution_date\": \"2024-07-29T14:00:14.606223+00:00\",\n \"external_trigger\": + true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-29T14:00:14.606223+00:00\",\n + \ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n + \ \"state\": \"queued\"\n}\n" + headers: + Connection: + - close + Content-Length: + - '579' + Content-Type: + - application/json + Date: + - Mon, 29 Jul 2024 14:00:14 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + Content-Type: + - application/json + method: DELETE + uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/7156f917-a901-465e-9782-e205999caf8e + response: + body: + string: '' + headers: + Connection: + - close + Content-Type: + - application/json + Date: + - Mon, 29 Jul 2024 14:00:14 GMT + Server: + - gunicorn + X-Robots-Tag: + - noindex, nofollow + status: + code: 204 + message: NO CONTENT +version: 1 diff --git a/backoffice/backoffice/workflows/tests/test_airflow_utils.py b/backoffice/backoffice/workflows/tests/test_airflow_utils.py new file mode 100644 index 00000000..5b84d025 --- /dev/null +++ b/backoffice/backoffice/workflows/tests/test_airflow_utils.py @@ -0,0 +1,60 @@ +import time + +import pytest +from django.apps import apps +from django.test import TransactionTestCase + +from backoffice.workflows import airflow_utils +from backoffice.workflows.constants import WORKFLOW_DAGS, StatusChoices, WorkflowType + +Workflow = apps.get_model(app_label="workflows", model_name="Workflow") + + +class TestAirflowUtils(TransactionTestCase): + def setUp(self): + self.dag_id = WORKFLOW_DAGS[WorkflowType.AUTHOR_CREATE][0] + self.workflow = Workflow.objects.create( + data={}, + status=StatusChoices.APPROVAL, + core=True, + is_update=False, + workflow_type=WorkflowType.AUTHOR_CREATE, + ) + self.response = airflow_utils.trigger_airflow_dag( + self.dag_id, str(self.workflow.id) + ) + + def tearDown(self) -> None: + airflow_utils.delete_workflow_dag(self.dag_id, self.workflow) + + @pytest.mark.vcr() + def test_trigger_airflow_dag(self): + self.assertEqual(self.response.status_code, 200) + + @pytest.mark.vcr() + def test_restart_failed_tasks(self): + time.sleep(20) # wait for dag to fail + response = airflow_utils.restart_failed_tasks(self.workflow) + self.assertEqual(response.status_code, 200) + + @pytest.mark.vcr() + def test_find_executed_dags(self): + executed_dags_for_workflow = airflow_utils.find_executed_dags(self.workflow) + + self.assertIn(self.dag_id, executed_dags_for_workflow) + + @pytest.mark.vcr() + def test_find_failed_dag(self): + time.sleep(20) # wait for dag to fail + failed_dag = airflow_utils.find_failed_dag(self.workflow) + self.assertEqual(self.dag_id, failed_dag) + + @pytest.mark.vcr() + def test_delete_workflow_dag(self): + response = airflow_utils.delete_workflow_dag(self.dag_id, self.workflow) + self.assertEqual(response.status_code, 200) + + @pytest.mark.vcr() + def test_restart_workflow_dags(self): + response = airflow_utils.restart_workflow_dags(self.workflow) + self.assertEqual(response.status_code, 200)