Add force-import feature

HISTORY.rst

@@ -2,10 +2,12 @@
 History
 =======
 
-0.4.2 (2023-10-20)
+UNRELEASED
 ------------------
-* Add base model for `ImportJob` and `ExportJob`
+* Add base model for ``ImportJob`` and ``ExportJob``
 * Extend import results template: show validation errors in table
+* Add force-import feature: skip rows with errors while importing
+* Add ``skip_parse_step`` parameter for importing API
 
 0.4.1 (2023-09-25)
 ------------------

docs/extensions.rst

@@ -178,6 +178,38 @@ for filtering:
 .. figure:: _static/images/filters-openapi.png
 
 
+------------
+Force import
+------------
+
+This package provides *force import* feature. When force import is enabled,
+then rows with errors will be skipped and rest of the rows will be handled.
+
+__________
+Admin page
+__________
+
+This functionality available in admin:
+
+.. figure:: _static/images/force_import_admin.png
+
+In case if some rows contain any errors it will be reported on parse/import stage:
+
+.. figure:: _static/images/force_import_results.png
+
+___
+API
+___
+
+In api there're 2 fields: ``force_import`` and ``skip_parse_step``.
+
+- ``force_import`` allows you to skip rows with errors.
+
+- ``skip_parse_step`` allows you to run the import task immediately, without having to call the ``confirm`` endpoint.
+
+.. image:: _static/images/start_api.png
+
+
 -------
 Widgets
 -------

import_export_extensions/admin/forms/__init__.py

@@ -1,2 +1,3 @@
-from .export_admin_form import ExportJobAdminForm
-from .import_admin_form import ImportJobAdminForm
+from .export_job_admin_form import ExportJobAdminForm
+from .import_admin_form import ForceImportForm
+from .import_job_admin_form import ImportJobAdminForm

import_export_extensions/admin/forms/import_admin_form.py

@@ -1,57 +1,11 @@
 from django import forms
-from django.urls import reverse
 
-from ... import models
-from ..widgets import ProgressBarWidget
+from import_export import forms as base_forms
 
 
-class ImportJobAdminForm(forms.ModelForm):
-    """Admin form for ``ImportJob`` model.
-
-    Adds custom `import_progressbar` field that displays current import
-    progress using AJAX requests to specified endpoint. Fields widget is
-    defined in `__init__` method.
-
-    """
-
-    import_progressbar = forms.Field(
+class ForceImportForm(base_forms.ImportForm):
+    """Import form with `force_import` option."""
+    force_import = forms.BooleanField(
         required=False,
+        initial=False,
     )
-
-    def __init__(
-        self,
-        instance: models.ImportJob,
-        *args,
-        **kwargs,
-    ):
-        """Provide `import_progressbar` widget the ``ImportJob`` instance."""
-        super().__init__(*args, instance=instance, **kwargs)
-        url_name = "admin:import_job_progress"
-        self.fields["import_progressbar"].label = (
-            "Import progress" if
-            instance.import_status == models.ImportJob.ImportStatus.IMPORTING
-            else "Parsing progress"
-        )
-        self.fields["import_progressbar"].widget = ProgressBarWidget(
-            job=instance,
-            url=reverse(url_name, args=(instance.id,)),
-        )
-
-    class Meta:
-        fields = (
-            "import_status",
-            "resource_path",
-            "data_file",
-            "resource_kwargs",
-            "traceback",
-            "error_message",
-            "result",
-            "parse_task_id",
-            "import_task_id",
-            "parse_finished",
-            "import_started",
-            "import_finished",
-            "created_by",
-            "created",
-            "modified",
-        )

import_export_extensions/admin/forms/import_job_admin_form.py

@@ -0,0 +1,57 @@
+from django import forms
+from django.urls import reverse
+
+from ... import models
+from ..widgets import ProgressBarWidget
+
+
+class ImportJobAdminForm(forms.ModelForm):
+    """Admin form for ``ImportJob`` model.
+
+    Adds custom `import_progressbar` field that displays current import
+    progress using AJAX requests to specified endpoint. Fields widget is
+    defined in `__init__` method.
+
+    """
+
+    import_progressbar = forms.Field(
+        required=False,
+    )
+
+    def __init__(
+        self,
+        instance: models.ImportJob,
+        *args,
+        **kwargs,
+    ):
+        """Provide `import_progressbar` widget the ``ImportJob`` instance."""
+        super().__init__(*args, instance=instance, **kwargs)
+        url_name = "admin:import_job_progress"
+        self.fields["import_progressbar"].label = (
+            "Import progress" if
+            instance.import_status == models.ImportJob.ImportStatus.IMPORTING
+            else "Parsing progress"
+        )
+        self.fields["import_progressbar"].widget = ProgressBarWidget(
+            job=instance,
+            url=reverse(url_name, args=(instance.id,)),
+        )
+
+    class Meta:
+        fields = (
+            "import_status",
+            "resource_path",
+            "data_file",
+            "resource_kwargs",
+            "traceback",
+            "error_message",
+            "result",
+            "parse_task_id",
+            "import_task_id",
+            "parse_finished",
+            "import_started",
+            "import_finished",
+            "created_by",
+            "created",
+            "modified",
+        )

import_export_extensions/admin/mixins/export_mixin.py

@@ -138,9 +138,9 @@ def celery_export_action(self, request, *args, **kwargs):
             resources=self.get_export_resource_classes(),
         )
         resource_kwargs = self.get_export_resource_kwargs(
-            request=request,
             *args,
             **kwargs,
+            request=request,
         )
         if request.method == "POST" and form.is_valid():
             file_format = formats[int(form.cleaned_data["file_format"])]

import_export_extensions/admin/mixins/import_mixin.py

@@ -16,10 +16,10 @@
 from django.utils.translation import gettext_lazy as _
 
 from import_export import admin as base_admin
-from import_export import forms as base_forms
 from import_export import mixins as base_mixins
 
 from ... import models
+from ..forms import ForceImportForm
 from . import types
 
 
@@ -152,7 +152,7 @@ def celery_import_action(
         context = self.get_context_data(request)
         resource_classes = self.get_import_resource_classes()
 
-        form = base_forms.ImportForm(
+        form = ForceImportForm(
             self.get_import_formats(),
             request.POST or None,
             request.FILES or None,
@@ -284,7 +284,7 @@ def celery_import_job_results_view(
 
             if job.import_status != models.ImportJob.ImportStatus.PARSED:
                 # display import form
-                context["import_form"] = base_forms.ImportForm(
+                context["import_form"] = ForceImportForm(
                     import_formats=self.get_import_formats(),
                 )
             else:
@@ -324,6 +324,7 @@ def create_import_job(
             resource_kwargs=resource.resource_init_kwargs,
             created_by=request.user,
             skip_parse_step=getattr(settings, "IMPORT_EXPORT_SKIP_ADMIN_CONFIRM", False),
+            force_import=form.cleaned_data["force_import"],
         )
 
     def get_import_job(

import_export_extensions/api/serializers/import_job.py

@@ -35,6 +35,7 @@ class Meta:
             "progress",
             "import_started",
             "import_finished",
+            "force_import",
             "created",
             "modified",
         )
@@ -50,6 +51,8 @@ class CreateImportJob(serializers.Serializer):
     resource_class: typing.Type[resources.CeleryModelResource]
 
     file = serializers.FileField(required=True)
+    force_import = serializers.BooleanField(default=False, required=False)
+    skip_parse_step = serializers.BooleanField(default=False, required=False)
 
     def __init__(
         self,
@@ -70,6 +73,8 @@ def create(
         """Create import job."""
         return models.ImportJob.objects.create(
             data_file=validated_data["file"],
+            force_import=validated_data["force_import"],
+            skip_parse_step=validated_data["skip_parse_step"],
             resource_path=self.resource_class.class_path,
             resource_kwargs=self._resource_kwargs,
             created_by=self._user,

import_export_extensions/migrations/0005_importjob_force_import.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.7 on 2023-11-16 10:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("import_export_extensions", "0004_alter_exportjob_created_by_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="importjob",
+            name="force_import",
+            field=models.BooleanField(
+                default=False,
+                help_text="Import data with skip invalid rows.",
+                verbose_name="Force import",
+            ),
+        ),
+    ]

import_export_extensions/models/import_job.py

@@ -175,6 +175,12 @@ class ImportStatus(models.TextChoices):
         verbose_name=_("Skip parse step"),
     )
 
+    force_import = models.BooleanField(
+        default=False,
+        help_text=_("Import data with skip invalid rows."),
+        verbose_name=_("Force import"),
+    )
+
     class Meta:
         verbose_name = _("Import job")
         verbose_name_plural = _("Import jobs")
@@ -355,6 +361,7 @@ def _parse_data_inner(self) -> Result:
             dry_run=True,
             raise_errors=False,
             collect_failures=True,
+            force_import=self.force_import,
         )
 
     def confirm_import(self):
@@ -445,6 +452,7 @@ def _import_data_inner(self) -> Result:
             raise_errors=True,
             use_transactions=True,
             collect_failures=True,
+            force_import=self.force_import,
         )
 
     def _get_import_format_by_ext(