Skip to content
This repository has been archived by the owner on Nov 21, 2024. It is now read-only.

Commit

Permalink
restart actions: added test for airflow utils
Browse files Browse the repository at this point in the history
  • Loading branch information
@DonHaul
DonHaul committed Jul 29, 2024
1 parent 80e05d3 commit f6b9bf9
Show file tree
Hide file tree
Showing 8 changed files with 1,108 additions and 7 deletions.
13 changes: 6 additions & 7 deletions backoffice/backoffice/workflows/airflow_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
from os import environ

import requests
from django.http import JsonResponse
from django.http import HttpResponse, JsonResponse
from requests.exceptions import RequestException
from rest_framework import status

Expand Down Expand Up @@ -57,7 +57,6 @@ def restart_failed_tasks(workflow):
"""

dag_id = find_failed_dag(workflow)

# assumes current task is one of the failed tasks
data = {
"dry_run": False,
Expand Down Expand Up @@ -94,7 +93,7 @@ def find_executed_dags(workflow):
headers=AIRFLOW_HEADERS,
)
if response.status_code == status.HTTP_200_OK:
executed_dags_for_workflow[dag_id] = response.content
executed_dags_for_workflow[dag_id] = response.json()

return executed_dags_for_workflow

Expand All @@ -109,7 +108,7 @@ def find_failed_dag(workflow):
executed_dags_for_workflow = find_executed_dags(workflow)

for dag, dag_data in executed_dags_for_workflow.items():
if dag_data["status"] == "failed":
if dag_data["state"] == "failed":
return dag


Expand All @@ -125,12 +124,12 @@ def delete_workflow_dag(dag_id, workflow):
headers=AIRFLOW_HEADERS,
)
response.raise_for_status()
return HttpResponse()
except RequestException:
data = {"error": response.json()}
return JsonResponse(data, status=status.HTTP_424_FAILED_DEPENDENCY)
return HttpResponse(status=status.HTTP_424_FAILED_DEPENDENCY)


def restart_workflow_dags(workflow, params):
def restart_workflow_dags(workflow, params=None):
"""Restarts dags of a given workflow.
:param workflow: workflow whoose dags should be restarted
Expand Down
112 changes: 112 additions & 0 deletions ...ffice/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_delete_workflow_dag.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
interactions:
- request:
body: '{"dag_run_id": "6569b2e7-0ed2-4bcd-91f4-31f2f45f8acf", "conf": {"workflow_id":
"6569b2e7-0ed2-4bcd-91f4-31f2f45f8acf"}}'
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '119'
Content-Type:
- application/json
method: POST
uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns
response:
body:
string: "{\n \"conf\": {\n \"workflow_id\": \"6569b2e7-0ed2-4bcd-91f4-31f2f45f8acf\"\n
\ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\":
\"6569b2e7-0ed2-4bcd-91f4-31f2f45f8acf\",\n \"data_interval_end\": \"2024-07-29T13:59:32.188032+00:00\",\n
\ \"data_interval_start\": \"2024-07-29T13:59:32.188032+00:00\",\n \"end_date\":
null,\n \"execution_date\": \"2024-07-29T13:59:32.188032+00:00\",\n \"external_trigger\":
true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-29T13:59:32.188032+00:00\",\n
\ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n
\ \"state\": \"queued\"\n}\n"
headers:
Connection:
- close
Content-Length:
- '579'
Content-Type:
- application/json
Date:
- Mon, 29 Jul 2024 13:59:32 GMT
Server:
- gunicorn
X-Robots-Tag:
- noindex, nofollow
status:
code: 200
message: OK
- request:
body: null
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '0'
Content-Type:
- application/json
method: DELETE
uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/6569b2e7-0ed2-4bcd-91f4-31f2f45f8acf
response:
body:
string: ''
headers:
Connection:
- close
Content-Type:
- application/json
Date:
- Mon, 29 Jul 2024 13:59:32 GMT
Server:
- gunicorn
X-Robots-Tag:
- noindex, nofollow
status:
code: 204
message: NO CONTENT
- request:
body: null
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '0'
Content-Type:
- application/json
method: DELETE
uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/6569b2e7-0ed2-4bcd-91f4-31f2f45f8acf
response:
body:
string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_initialization_dag'
and DagRun ID: '6569b2e7-0ed2-4bcd-91f4-31f2f45f8acf' not found\",\n \"status\":
404,\n \"title\": \"Not Found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n"
headers:
Connection:
- close
Content-Length:
- '293'
Content-Type:
- application/problem+json
Date:
- Mon, 29 Jul 2024 13:59:32 GMT
Server:
- gunicorn
X-Robots-Tag:
- noindex, nofollow
status:
code: 404
message: NOT FOUND
version: 1
183 changes: 183 additions & 0 deletions ...office/backoffice/workflows/tests/cassettes/TestAirflowUtils.test_find_executed_dags.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,183 @@
interactions:
- request:
body: '{"dag_run_id": "41f9047d-f7a4-48cc-b138-3e2fa9e19802", "conf": {"workflow_id":
"41f9047d-f7a4-48cc-b138-3e2fa9e19802"}}'
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '119'
Content-Type:
- application/json
method: POST
uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns
response:
body:
string: "{\n \"conf\": {\n \"workflow_id\": \"41f9047d-f7a4-48cc-b138-3e2fa9e19802\"\n
\ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\":
\"41f9047d-f7a4-48cc-b138-3e2fa9e19802\",\n \"data_interval_end\": \"2024-07-29T13:59:32.491551+00:00\",\n
\ \"data_interval_start\": \"2024-07-29T13:59:32.491551+00:00\",\n \"end_date\":
null,\n \"execution_date\": \"2024-07-29T13:59:32.491551+00:00\",\n \"external_trigger\":
true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-29T13:59:32.491551+00:00\",\n
\ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n
\ \"state\": \"queued\"\n}\n"
headers:
Connection:
- close
Content-Length:
- '579'
Content-Type:
- application/json
Date:
- Mon, 29 Jul 2024 13:59:32 GMT
Server:
- gunicorn
X-Robots-Tag:
- noindex, nofollow
status:
code: 200
message: OK
- request:
body: null
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Type:
- application/json
method: GET
uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/41f9047d-f7a4-48cc-b138-3e2fa9e19802
response:
body:
string: "{\n \"conf\": {\n \"workflow_id\": \"41f9047d-f7a4-48cc-b138-3e2fa9e19802\"\n
\ },\n \"dag_id\": \"author_create_initialization_dag\",\n \"dag_run_id\":
\"41f9047d-f7a4-48cc-b138-3e2fa9e19802\",\n \"data_interval_end\": \"2024-07-29T13:59:32.491551+00:00\",\n
\ \"data_interval_start\": \"2024-07-29T13:59:32.491551+00:00\",\n \"end_date\":
null,\n \"execution_date\": \"2024-07-29T13:59:32.491551+00:00\",\n \"external_trigger\":
true,\n \"last_scheduling_decision\": null,\n \"logical_date\": \"2024-07-29T13:59:32.491551+00:00\",\n
\ \"note\": null,\n \"run_type\": \"manual\",\n \"start_date\": null,\n
\ \"state\": \"queued\"\n}\n"
headers:
Connection:
- close
Content-Length:
- '579'
Content-Type:
- application/json
Date:
- Mon, 29 Jul 2024 13:59:32 GMT
Server:
- gunicorn
X-Robots-Tag:
- noindex, nofollow
status:
code: 200
message: OK
- request:
body: null
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Type:
- application/json
method: GET
uri: http://host.docker.internal:8080/api/v1/dags/author_create_approved_dag/dagRuns/41f9047d-f7a4-48cc-b138-3e2fa9e19802
response:
body:
string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_approved_dag'
and DagRun ID: '41f9047d-f7a4-48cc-b138-3e2fa9e19802' not found\",\n \"status\":
404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n"
headers:
Connection:
- close
Content-Length:
- '294'
Content-Type:
- application/problem+json
Date:
- Mon, 29 Jul 2024 13:59:32 GMT
Server:
- gunicorn
X-Robots-Tag:
- noindex, nofollow
status:
code: 404
message: NOT FOUND
- request:
body: null
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Type:
- application/json
method: GET
uri: http://host.docker.internal:8080/api/v1/dags/author_create_rejected_dag/dagRuns/41f9047d-f7a4-48cc-b138-3e2fa9e19802
response:
body:
string: "{\n \"detail\": \"DAGRun with DAG ID: 'author_create_rejected_dag'
and DagRun ID: '41f9047d-f7a4-48cc-b138-3e2fa9e19802' not found\",\n \"status\":
404,\n \"title\": \"DAGRun not found\",\n \"type\": \"https://airflow.apache.org/docs/apache-airflow/2.8.3/stable-rest-api-ref.html#section/Errors/NotFound\"\n}\n"
headers:
Connection:
- close
Content-Length:
- '294'
Content-Type:
- application/problem+json
Date:
- Mon, 29 Jul 2024 13:59:32 GMT
Server:
- gunicorn
X-Robots-Tag:
- noindex, nofollow
status:
code: 404
message: NOT FOUND
- request:
body: null
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '0'
Content-Type:
- application/json
method: DELETE
uri: http://host.docker.internal:8080/api/v1/dags/author_create_initialization_dag/dagRuns/41f9047d-f7a4-48cc-b138-3e2fa9e19802
response:
body:
string: ''
headers:
Connection:
- close
Content-Type:
- application/json
Date:
- Mon, 29 Jul 2024 13:59:32 GMT
Server:
- gunicorn
X-Robots-Tag:
- noindex, nofollow
status:
code: 204
message: NO CONTENT
version: 1
Loading

0 comments on commit f6b9bf9

Please sign in to comment.