diff --git a/import_export_extensions/admin/model_admins/export_job_admin.py b/import_export_extensions/admin/model_admins/export_job_admin.py index 5b4ff14..3efb78f 100644 --- a/import_export_extensions/admin/model_admins/export_job_admin.py +++ b/import_export_extensions/admin/model_admins/export_job_admin.py @@ -1,4 +1,6 @@ +import http + from django.contrib import admin, messages from django.core.handlers.wsgi import WSGIRequest from django.db.models import QuerySet @@ -92,7 +94,10 @@ def export_job_progress_view( id=job_id, ) except self.export_job_model.DoesNotExist as error: - return JsonResponse(dict(validation_error=error.args[0])) + return JsonResponse( + dict(validation_error=error.args[0]), + status=http.HTTPStatus.NOT_FOUND, + ) response_data = dict(status=job.export_status.title()) @@ -102,16 +107,17 @@ def export_job_progress_view( percent = 0 total = 0 current = 0 - info = job.progress["info"] + job_progress = job.progress + progress_info = job_progress["info"] - if info and info["total"]: - percent = int(100 / info["total"] * info["current"]) - total = info["total"] - current = info["current"] + if progress_info and progress_info["total"]: + total = progress_info["total"] + current = progress_info["current"] + percent = int(100 / total * current) response_data.update( dict( - state=job.progress["state"], + state=job_progress["state"], percent=percent, total=total, current=current, @@ -125,7 +131,9 @@ def get_readonly_fields(self, request, obj=None): Some fields are editable for new ExportJob. """ - readonly_fields = [ + base_readonly_fields = super().get_readonly_fields(request, obj) + readonly_fields = ( + *base_readonly_fields, "export_status", "traceback", "file_format_path", @@ -134,15 +142,10 @@ def get_readonly_fields(self, request, obj=None): "export_finished", "error_message", "_model", - ] - if obj: - readonly_fields.extend( - [ - "resource_path", - "data_file", - "resource_kwargs", - ], - ) + "resource_path", + "data_file", + "resource_kwargs", + ) return readonly_fields diff --git a/import_export_extensions/models/core.py b/import_export_extensions/models/core.py index 2278556..bab72e3 100644 --- a/import_export_extensions/models/core.py +++ b/import_export_extensions/models/core.py @@ -55,7 +55,17 @@ class Meta: class TaskStateInfo(typing.TypedDict): - """Class representing task state dict.""" + """Class representing task state dict. + + Possible states: + 1. PENDING + 2. STARTED + 3. SUCCESS + 4. EXPORTING - custom status that also set export info + + https://docs.celeryproject.org/en/latest/userguide/tasks.html#states + + """ state: str info: dict[str, int] | None diff --git a/import_export_extensions/models/export_job.py b/import_export_extensions/models/export_job.py index dafdbeb..fbb9789 100644 --- a/import_export_extensions/models/export_job.py +++ b/import_export_extensions/models/export_job.py @@ -157,41 +157,14 @@ def export_filename(self) -> str: @property def progress(self) -> TaskStateInfo | None: - """Return dict with parsing state. - - Example for sync mode:: - - { - 'state': 'EXPORTING', - 'info': None - } - - Example for celery (celery) mode:: - - { - 'state': 'EXPORTING', - 'info': {'current': 15, 'total': 100} - } - - Possible states: - 1. PENDING - 2. STARTED - 3. SUCCESS - 4. EXPORTING - custom status that also set export info - - https://docs.celeryproject.org/en/latest/userguide/tasks.html#states - - """ - if self.export_status not in (self.ExportStatus.EXPORTING,): - return None - - if not self.export_task_id or current_app.conf.task_always_eager: - return dict( - state=self.export_status.upper(), - info=None, - ) - - return self._get_task_state(self.export_task_id) + """Return dict with export state.""" + if ( + self.export_task_id + and self.export_status == self.ExportStatus.EXPORTING + ): + return self._get_task_state(self.export_task_id) + + return None def _check_export_status_correctness( self, diff --git a/test_project/tests/integration_tests/test_admin/test_export.py b/test_project/tests/integration_tests/test_admin/test_export.py index a8379d7..106aeb9 100644 --- a/test_project/tests/integration_tests/test_admin/test_export.py +++ b/test_project/tests/integration_tests/test_admin/test_export.py @@ -7,8 +7,11 @@ from rest_framework import status import pytest +import pytest_mock +from celery import states from import_export_extensions.models import ExportJob +from test_project.fake_app.factories import ArtistExportJobFactory @pytest.mark.usefixtures("existing_artist") @@ -49,3 +52,303 @@ def test_export_using_admin_model(client: Client, superuser: User): assert ExportJob.objects.exists() export_job = ExportJob.objects.first() assert export_job.export_status == ExportJob.ExportStatus.EXPORTED + + +@pytest.mark.django_db(transaction=True) +def test_export_progress_during_export( + client: Client, + superuser: User, + mocker: pytest_mock.MockerFixture, +): + """Test export job admin progress page during export.""" + client.force_login(superuser) + + # Prepare data to imitate intermediate task state + fake_progress_info = { + "current": 2, + "total": 3, + } + mocker.patch( + "celery.result.AsyncResult.info", + new=fake_progress_info, + ) + expected_percent = int( + fake_progress_info["current"] / fake_progress_info["total"] * 100, + ) + + artist_export_job = ArtistExportJobFactory() + artist_export_job.export_status = ExportJob.ExportStatus.EXPORTING + artist_export_job.save() + + response = client.post( + path=reverse( + "admin:export_job_progress", + kwargs={"job_id": artist_export_job.pk}, + ), + ) + assert response.status_code == status.HTTP_200_OK + json_data = response.json() + + assert json_data == { + "status": ExportJob.ExportStatus.EXPORTING.title(), + "state": "SUCCESS", + "percent": expected_percent, + **fake_progress_info, + } + + +@pytest.mark.django_db(transaction=True) +def test_export_progress_after_complete_export( + client: Client, + superuser: User, +): + """Test export job admin progress page after complete export.""" + client.force_login(superuser) + + artist_export_job = ArtistExportJobFactory() + artist_export_job.refresh_from_db() + + response = client.post( + path=reverse( + "admin:export_job_progress", + kwargs={"job_id": artist_export_job.pk}, + ), + ) + assert response.status_code == status.HTTP_200_OK + assert response.json() == { + "status": artist_export_job.export_status.title(), + } + + +@pytest.mark.django_db(transaction=True) +def test_export_progress_with_deleted_export_job( + client: Client, + superuser: User, + mocker: pytest_mock.MockerFixture, +): + """Test export job admin progress page with deleted export job. + + Check that page available, but return an error message. + + """ + client.force_login(superuser) + + mocker.patch("import_export_extensions.tasks.export_data_task.apply_async") + artist_export_job = ArtistExportJobFactory() + job_id = artist_export_job.id + artist_export_job.delete() + + expected_error_message = "ExportJob matching query does not exist." + + response = client.post( + path=reverse( + "admin:export_job_progress", + kwargs={ + "job_id": job_id, + }, + ), + ) + + assert response.status_code == status.HTTP_404_NOT_FOUND + assert response.json()["validation_error"] == expected_error_message + + +@pytest.mark.django_db(transaction=True) +def test_export_progress_with_failed_celery_task( + client: Client, + superuser: User, + mocker: pytest_mock.MockerFixture, +): + """Test than after celery fail ExportJob will be in export error status.""" + client.force_login(superuser) + + expected_error_message = "Mocked Error Message" + mocker.patch( + "celery.result.AsyncResult.state", + new=states.FAILURE, + ) + mocker.patch( + "celery.result.AsyncResult.info", + new=ValueError(expected_error_message), + ) + artist_export_job = ArtistExportJobFactory() + artist_export_job.export_status = ExportJob.ExportStatus.EXPORTING + artist_export_job.save() + + response = client.post( + path=reverse( + "admin:export_job_progress", + kwargs={ + "job_id": artist_export_job.id, + }, + ), + ) + + assert response.status_code == status.HTTP_200_OK + assert response.json()["state"] == states.FAILURE + artist_export_job.refresh_from_db() + assert ( + artist_export_job.export_status == ExportJob.ExportStatus.EXPORT_ERROR + ) + + +@pytest.mark.django_db(transaction=True) +def test_cancel_export_admin_action( + client: Client, + superuser: User, + mocker: pytest_mock.MockerFixture, +): + """Test `cancel_export` via admin action.""" + client.force_login(superuser) + + revoke_mock = mocker.patch("celery.current_app.control.revoke") + export_data_mock = mocker.patch( + "import_export_extensions.models.ExportJob.export_data", + ) + job: ExportJob = ArtistExportJobFactory() + + response = client.post( + reverse("admin:import_export_extensions_exportjob_changelist"), + data={ + "action": "cancel_jobs", + "_selected_action": [job.pk], + }, + ) + job.refresh_from_db() + + assert response.status_code == status.HTTP_302_FOUND + assert job.export_status == ExportJob.ExportStatus.CANCELLED + assert ( + response.wsgi_request._messages._queued_messages[0].message + == f"Export of {job} canceled" + ) + export_data_mock.assert_called_once() + revoke_mock.assert_called_once_with(job.export_task_id, terminate=True) + + +@pytest.mark.django_db(transaction=True) +def test_cancel_export_admin_action_with_incorrect_export_job_status( + client: Client, + superuser: User, + mocker: pytest_mock.MockerFixture, +): + """Test `cancel_export` via admin action with wrong export job status.""" + client.force_login(superuser) + + revoke_mock = mocker.patch("celery.current_app.control.revoke") + job: ExportJob = ArtistExportJobFactory() + + expected_error_message = f"ExportJob with id {job.pk} has incorrect status" + + response = client.post( + reverse("admin:import_export_extensions_exportjob_changelist"), + data={ + "action": "cancel_jobs", + "_selected_action": [job.pk], + }, + ) + job.refresh_from_db() + + assert response.status_code == status.HTTP_302_FOUND + assert job.export_status == ExportJob.ExportStatus.EXPORTED + assert ( + expected_error_message + in response.wsgi_request._messages._queued_messages[0].message + ) + revoke_mock.assert_not_called() + + +@pytest.mark.parametrize( + argnames=["job_status", "expected_fieldsets"], + argvalues=[ + pytest.param( + ExportJob.ExportStatus.CREATED, + ( + ( + "export_status", + "_model", + "created", + "export_started", + "export_finished", + ), + ), + id="Get fieldsets for job in status CREATED", + ), + pytest.param( + ExportJob.ExportStatus.EXPORTED, + ( + ( + "export_status", + "_model", + "created", + "export_started", + "export_finished", + ), + ("data_file",), + ), + id="Get fieldsets for job in status EXPORTED", + ), + pytest.param( + ExportJob.ExportStatus.EXPORTING, + ( + ( + "export_status", + "export_progressbar", + ), + ), + id="Get fieldsets for job in status EXPORTING", + ), + pytest.param( + ExportJob.ExportStatus.EXPORT_ERROR, + ( + ( + "export_status", + "_model", + "created", + "export_started", + "export_finished", + ), + ( + "error_message", + "traceback", + ), + ), + id="Get fieldsets for job in status EXPORT_ERROR", + ), + ], +) +def test_get_fieldsets_by_export_job_status( + client: Client, + superuser: User, + job_status: ExportJob.ExportStatus, + expected_fieldsets: tuple[tuple[str]], + mocker: pytest_mock.MockerFixture, +): + """Test that appropriate fieldsets returned for different job statuses.""" + client.force_login(superuser) + + mocker.patch( + "import_export_extensions.models.ExportJob.export_data", + ) + job: ExportJob = ArtistExportJobFactory() + job.export_status = job_status + job.save() + + response = client.get( + reverse( + "admin:import_export_extensions_exportjob_change", + kwargs={"object_id": job.pk}, + ), + ) + + fieldsets = response.context["adminform"].fieldsets + fields = [fields["fields"] for _, fields in fieldsets] + + assert tuple(fields) == ( + *expected_fieldsets, + ( + "resource_path", + "resource_kwargs", + "file_format_path", + ), + ) diff --git a/test_project/tests/test_import_job/test_import_data.py b/test_project/tests/test_import_job/test_import_data.py index fa05619..5b56704 100644 --- a/test_project/tests/test_import_job/test_import_data.py +++ b/test_project/tests/test_import_job/test_import_data.py @@ -1,3 +1,5 @@ +import django.test + import pytest from import_export_extensions.models import ImportJob @@ -156,3 +158,22 @@ def test_import_data_with_validation_error(existing_artist: Artist): job.parse_data() job.refresh_from_db() assert job.import_status == ImportJob.ImportStatus.INPUT_ERROR + + +@django.test.override_settings( + IMPORT_EXPORT_MAX_DATASET_ROWS=1, +) +def test_import_create_with_max_rows( + new_artist: Artist, +): + """Test import job max dataset rows validation.""" + import_job: ImportJob = ArtistImportJobFactory( + artists=[new_artist], + skip_parse_step=True, + is_valid_file=False, + ) + + import_job.import_data() + import_job.refresh_from_db() + assert import_job.import_status == import_job.ImportStatus.IMPORT_ERROR + assert "Too many rows `2`" in import_job.error_message