Redo records with the remove_and_rerun property in the data, move individual remove and rerun to a background job

.github/workflows/test.yml

@@ -40,7 +40,7 @@ jobs:
         run: bundle exec rake db:migrate db:test:prepare
 
       - name: Run rspec
-        run: bundle exec rake
+        run: bundle exec rake spec
 
       - name: Upload coverage results
         uses: actions/upload-artifact@v2

Rakefile

@@ -25,18 +25,18 @@ require 'bundler/gem_tasks'
 
 require 'solr_wrapper/rake_task' unless Rails.env.production?
 
+require 'rubocop/rake_task'
+
+RuboCop::RakeTask.new(:rubocop) do |t|
+  t.options = ['--display-cop-names', '--ignore-parent-exclusion', '-a']
+end
+
 begin
   require 'rspec/core/rake_task'
 
   RSpec::Core::RakeTask.new(:spec)
 
-  task default: :spec
+  task default: [:rubocop, :spec]
 rescue LoadError # rubocop:disable Lint/HandleExceptions
   # no rspec available
 end
-
-require 'rubocop/rake_task'
-
-RuboCop::RakeTask.new(:rubocop) do |t|
-  t.options = ['--display-cop-names']
-end

app/assets/javascripts/bulkrax/importers.js.erb

@@ -74,12 +74,14 @@ function handleFileToggle(file_path) {
     $('#file_path').hide()
     $('#file_upload').hide()
     $('#cloud').hide()
+    $('#existing_options').hide()
     $('#file_path input').attr('required', null)
     $('#file_upload input').attr('required', null)
   } else {
     $('#file_path').show()
     $('#file_upload').hide()
     $('#cloud').hide()
+    $('#existing_options').hide()
     $('#file_path input').attr('required', 'required')
     $('#file_upload input').attr('required', null)
     $('#importer_parser_fields_file_style_specify_a_path_on_the_server').attr('checked', true)
@@ -89,23 +91,35 @@ function handleFileToggle(file_path) {
     $('#file_path').hide()
     $('#file_upload').show()
     $('#cloud').hide()
+    $('#existing_options').hide()
     $('#file_path input').attr('required', null)
     $('#file_upload input').attr('required', 'required')
   })
   $('#importer_parser_fields_file_style_specify_a_path_on_the_server').click(function(e){
     $('#file_path').show()
     $('#file_upload').hide()
     $('#cloud').hide()
+    $('#existing_options').hide()
     $('#file_path input').attr('required', 'required')
     $('#file_upload input').attr('required', null)
   })
   $('#importer_parser_fields_file_style_add_cloud_file').click(function(e){
     $('#file_path').hide()
     $('#file_upload').hide()
     $('#cloud').show()
+    $('#existing_options').hide()
     $('#file_path input').attr('required', null)
     $('#file_upload input').attr('required', null)
   })
+  $('#importer_parser_fields_file_style_existing_entries').click(function(e){
+    $('#file_path').hide()
+    $('#file_upload').hide()
+    $('#cloud').hide()
+    $('#existing_options').show()
+    $('#file_path input').attr('required', null)
+    $('#file_upload input').attr('required', null)
+  })
+
 }
 
 function handleParserKlass() {
@@ -189,4 +203,4 @@ function setError(selector, error) {
   selector.attr('disabled', true)
 }
 
-$(document).on({'ready': prepBulkrax, 'turbolinks:load': prepBulkrax})
+$(document).on({'ready': prepBulkrax, 'turbolinks:load': prepBulkrax})

app/assets/stylesheets/bulkrax/import_export.scss

@@ -34,4 +34,9 @@ div#s2id_exporter_export_source_collection {
 
 .bulkrax-clear-toggles {
   clear: both;
-}
+}
+
+#existing_options .collection_check_boxes {
+  margin-left: 10px;
+  margin-right: 10px;
+}

app/controllers/bulkrax/entries_controller.rb

@@ -17,13 +17,29 @@ def show
 
     def update
       @entry = Entry.find(params[:id])
-      @entry.factory&.find&.destroy if params[:destroy_first]
-      @entry.build
-      @entry.save
+      type = case @entry.type.downcase
+             when /fileset/
+               'file_set'
+             when /collection/
+               'collection'
+             else
+               'work'
+             end
       item = @entry.importerexporter
+      # do not run counters as it loads the whole parser
+      current_run = item.current_run(skip_counts: true)
+      @entry.set_status_info('Pending', current_run)
+      ScheduleRelationshipsJob.set(wait: 5.minutes).perform_later(importer_id: @entry.importer.id)
+
+      if params[:destroy_first]
+        "Bulkrax::DeleteAndImport#{type.camelize}Job".constantize.perform_later(@entry, current_run)
+      else
+        "Bulkrax::Import#{type.camelize}Job".constantize.perform_later(@entry.id, current_run.id)
+      end
+
       entry_path = item.class.to_s.include?('Importer') ? bulkrax.importer_entry_path(item.id, @entry.id) : bulkrax.exporter_entry_path(item.id, @entry.id)
 
-      redirect_back fallback_location: entry_path, notice: "Entry update ran, new status is #{@entry.status}"
+      redirect_back fallback_location: entry_path, notice: "Entry #{@entry.id} update has been queued"
     end
 
     def destroy

app/controllers/bulkrax/importers_controller.rb

@@ -233,7 +233,7 @@ def importable_params
     end
 
     def importable_parser_fields
-      params&.[](:importer)&.[](:parser_fields)&.except(:file)&.keys
+      params&.[](:importer)&.[](:parser_fields)&.except(:file, :entry_statuses)&.keys&. + [{ "entry_statuses" => [] }]
     end
 
     # Only allow a trusted parameters through.

app/jobs/bulkrax/delete_and_import_collection_job.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class DeleteAndImportCollectionJob < DeleteAndImportJob
+    DELETE_CLASS = Bulkrax::DeleteCollectionJob
+    IMPORT_CLASS = Bulkrax::ImportCollectionJob
+  end
+end

app/jobs/bulkrax/delete_and_import_file_set_job.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class DeleteAndImportFileSetJob < DeleteAndImportJob
+    DELETE_CLASS = Bulkrax::DeleteFileSetJob
+    IMPORT_CLASS = Bulkrax::ImportFileSetJob
+  end
+end

app/jobs/bulkrax/delete_and_import_job.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class DeleteAndImportJob < ApplicationJob
+    queue_as :import
+
+    def perform(entry, importer_run)
+      status = self.class::DELETE_CLASS.perform_now(entry, importer_run)
+      if status.status_message == "Deleted"
+        entry = Bulkrax::Entry.find(entry.id) # maximum reload
+        self.class::IMPORT_CLASS.perform_now(entry.id, importer_run.id)
+      end
+
+    rescue => e
+      entry.set_status_info(e)
+      # this causes caught exception to be reraised
+      raise
+    end
+  end
+end

app/jobs/bulkrax/delete_and_import_work_job.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class DeleteAndImportWorkJob < DeleteAndImportJob
+    DELETE_CLASS = Bulkrax::DeleteWorkJob
+    IMPORT_CLASS = Bulkrax::ImportWorkJob
+  end
+end

app/jobs/bulkrax/delete_job.rb

@@ -15,6 +15,10 @@ def perform(entry, importer_run)
       entry.importer.current_run = ImporterRun.find(importer_run.id)
       entry.importer.record_status
       entry.set_status_info("Deleted", ImporterRun.find(importer_run.id))
+    rescue => e
+      entry.set_status_info(e)
+      # this causes caught exception to be reraised
+      raise
     end
   end
 end

app/jobs/bulkrax/import_job.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class ImportJob < ApplicationJob
+    queue_as :import
+  end
+end

app/models/bulkrax/exporter.rb

@@ -23,6 +23,10 @@ def export
       set_status_info(e)
     end
 
+    def remove_and_rerun
+      self.parser_fields['remove_and_rerun']
+    end
+
     # #export_source accessors
     # Used in form to prevent it from getting confused as to which value to populate #export_source with.
     # Also, used to display the correct selected value when rendering edit form.
@@ -102,9 +106,12 @@ def importers_list
       Importer.all.map { |i| [i.name, i.id] }
     end
 
-    def current_run
+    def current_run(skip_counts: false)
+      @current_run ||= self.exporter_runs.create! if skip_counts
+      return @current_run if @current_run
+
       total = self.limit || parser.total
-      @current_run ||= self.exporter_runs.create!(total_work_entries: total, enqueued_records: total)
+      @current_run = self.exporter_runs.create!(total_work_entries: total, enqueued_records: total)
     end
 
     def last_run

app/models/bulkrax/importer.rb

@@ -103,11 +103,12 @@ def schedulable?
       frequency.to_seconds != 0
     end
 
-    def current_run
+    def current_run(skip_counts: false)
       return @current_run if @current_run.present?
 
       @current_run = self.importer_runs.create!
       return @current_run if file? && zip?
+      return @current_run if skip_counts
 
       entry_counts = {
         total_work_entries: self.limit || parser.works_total,
@@ -166,6 +167,10 @@ def metadata_only?
       parser.parser_fields['metadata_only'] == true
     end
 
+    def existing_entries?
+      parser.parser_fields['file_style']&.match(/Existing Entries/)
+    end
+
     def import_works
       import_objects(['work'])
     end
@@ -188,13 +193,7 @@ def import_objects(types_array = nil)
       self.only_updates ||= false
       self.save if self.new_record? # Object needs to be saved for statuses
       types = types_array || DEFAULT_OBJECT_TYPES
-      if remove_and_rerun
-        self.entries.find_each do |e|
-          e.factory.find&.destroy!
-          e.destroy!
-        end
-      end
-      parser.create_objects(types)
+      existing_entries? ? parser.rebuild_entries(types) : parser.create_objects(types)
       mark_unseen_as_skipped
     rescue StandardError => e
       set_status_info(e)
@@ -203,7 +202,7 @@ def import_objects(types_array = nil)
     # After an import any entries we did not touch are skipped.
     # They are not really pending, complete for the last run, or failed
     def mark_unseen_as_skipped
-      entries.where.not(id. seen).find_each do |entry|
+      entries.where.not(identifier: seen.keys).find_each do |entry|
         entry.set_status_info('Skipped')
       end
     end