Merge pull request #254 from samvera-labs/export-from-date-range

Export from date range and status

app/controllers/bulkrax/exporters_controller.rb

@@ -92,14 +92,20 @@ def set_exporter
       # Only allow a trusted parameters through.
       def exporter_params
         params[:exporter][:export_source] = params[:exporter]["export_source_#{params[:exporter][:export_from]}".to_sym]
-        params.fetch(:exporter).permit(:name, :user_id, :export_source, :export_from, :export_type, :parser_klass, :limit, field_mapping: {})
+        if params[:exporter][:date_filter] == "1"
+          params.fetch(:exporter).permit(:name, :user_id, :export_source, :export_from, :export_type,
+                                         :parser_klass, :limit, :start_date, :finish_date, :work_visibility, field_mapping: {})
+        else
+          params.fetch(:exporter).permit(:name, :user_id, :export_source, :export_from, :export_type,
+                                         :parser_klass, :limit, :work_visibility, field_mapping: {}).merge(start_date: nil, finish_date: nil)
+        end
       end
 
       # Add the field_mapping from the Bulkrax configuration
       def field_mapping_params
         # @todo replace/append once mapping GUI is in place
-        fields = Bulkrax.parsers.map { |m| m[:partial] if m[:class_name] == params[:exporter][:parser_klass] }.compact.first
-        @exporter.field_mapping = Bulkrax.field_mappings[fields.to_sym] if fields
+        field_mapping_key = Bulkrax.parsers.map { |m| m[:class_name] if m[:class_name] == params[:exporter][:parser_klass] }.compact.first
+        @exporter.field_mapping = Bulkrax.field_mappings[field_mapping_key] if field_mapping_key
       end
 
       def add_exporter_breadcrumbs

app/models/bulkrax/csv_entry.rb

@@ -81,20 +81,24 @@ def build_export_metadata
       self.parsed_metadata['id'] = hyrax_record.id
       self.parsed_metadata[self.class.source_identifier_field] = hyrax_record.id
       self.parsed_metadata['model'] = hyrax_record.has_model.first
+      build_mapping_metadata
+      unless hyrax_record.is_a?(Collection)
+        self.parsed_metadata['file'] = hyrax_record.file_sets.map { |fs| filename(fs).to_s unless filename(fs).blank? }.compact.join('; ')
+      end
+      self.parsed_metadata
+    end
+
+    def build_mapping_metadata
       mapping.each do |key, value|
         next if Bulkrax.reserved_properties.include?(key) && !field_supported?(key)
-        next unless hyrax_record.respond_to?(key)
-        data = hyrax_record.send(key)
+        next unless hyrax_record.respond_to?(value['from']&.first.to_s)
+        data = hyrax_record.send(value['from'].first)
         if data.is_a?(ActiveTriples::Relation)
           self.parsed_metadata[key] = data.map { |d| prepare_export_data(d) }.join('; ').to_s unless value[:excluded]
         else
           self.parsed_metadata[key] = prepare_export_data(data)
         end
       end
-      unless hyrax_record.is_a?(Collection)
-        self.parsed_metadata['file'] = hyrax_record.file_sets.map { |fs| filename(fs).to_s unless filename(fs).blank? }.compact.join('; ')
-      end
-      self.parsed_metadata
     end
 
     def prepare_export_data(datum)
@@ -137,7 +141,7 @@ def find_or_create_collection_ids
           self.collection_ids << c.id unless c.blank? || self.collection_ids.include?(c.id)
         end
       end
-      return self.collection_ids
+      self.collection_ids
     end
 
     def required_elements?(keys)

app/models/bulkrax/exporter.rb

@@ -46,6 +46,19 @@ def export_source_worktype
       self.export_source if self.export_from == 'worktype'
     end
 
+    def date_filter
+      self.start_date.present? || self.finish_date.present?
+    end
+
+    def work_status_list
+      [
+        ['Any', ''],
+        [I18n.t('hyrax.visibility.open.text'), Hydra::AccessControls::AccessRight::VISIBILITY_TEXT_VALUE_PUBLIC],
+        [I18n.t('hyrax.visibility.restricted.text'), Hydra::AccessControls::AccessRight::VISIBILITY_TEXT_VALUE_PRIVATE],
+        [I18n.t('hyrax.visibility.authenticated.text', institution: I18n.t('hyrax.institution_name')), Hydra::AccessControls::AccessRight::VISIBILITY_TEXT_VALUE_AUTHENTICATED]
+      ]
+    end
+
     # If field_mapping is empty, setup a default based on the export_properties
     def mapping
       @mapping ||= self.field_mapping ||

app/parsers/bulkrax/csv_parser.rb

@@ -103,35 +103,40 @@ def create_parent_child_relationships
       super
     end
 
-    def create_from_importer
-      importer = Bulkrax::Importer.find(importerexporter.export_source)
-      non_errored_entries = importer.entries.where(type: importer.parser.entry_class.to_s, last_error: [nil, {}, ''])
-      non_errored_entries.each_with_index do |entry, index|
-        break if limit_reached?(limit, index)
-        query = "#{ActiveFedora.index_field_mapper.solr_name(Bulkrax.system_identifier_field)}:\"#{entry.identifier}\""
-        work_id = ActiveFedora::SolrService.query(query, fl: 'id', rows: 1).first['id']
-        new_entry = find_or_create_entry(entry_class, work_id, 'Bulkrax::Exporter')
-        Bulkrax::ExportWorkJob.perform_now(new_entry.id, current_run.id)
+    def extra_filters
+      output = ""
+      if importerexporter.start_date.present?
+        start_dt = importerexporter.start_date.to_datetime.strftime('%FT%TZ')
+        finish_dt = importerexporter.finish_date.present? ? importerexporter.finish_date.to_datetime.end_of_day.strftime('%FT%TZ') : "NOW"
+        output += " AND system_modified_dtsi:[#{start_dt} TO #{finish_dt}]"
       end
+      output += importerexporter.work_visibility.present? ? " AND visibility_ssi:#{importerexporter.work_visibility}" : ""
+      output
     end
 
-    def create_from_collection
-      work_ids = ActiveFedora::SolrService.query("member_of_collection_ids_ssim:#{importerexporter.export_source}", rows: 2_000_000_000).map(&:id)
-      work_ids.each_with_index do |wid, index|
-        break if limit_reached?(limit, index)
-        new_entry = find_or_create_entry(entry_class, wid, 'Bulkrax::Exporter')
-        Bulkrax::ExportWorkJob.perform_now(new_entry.id, current_run.id)
+    def current_work_ids
+      case importerexporter.export_from
+      when 'collection'
+        ActiveFedora::SolrService.query("member_of_collection_ids_ssim:#{importerexporter.export_source + extra_filters}", rows: 2_000_000_000).map(&:id)
+      when 'worktype'
+        ActiveFedora::SolrService.query("has_model_ssim:#{importerexporter.export_source + extra_filters}", rows: 2_000_000_000).map(&:id)
+      when 'importer'
+        importer = Bulkrax::Importer.find(importerexporter.export_source)
+        entry_ids = importer.entries.where(type: importer.parser.entry_class.to_s, last_error: [nil, {}, '']).map(&:identifier)
+        ActiveFedora::SolrService.query("#{Bulkrax.system_identifier_field}_tesim:(#{entry_ids.join(' OR ')})#{extra_filters}", rows: 2_000_000_000).map(&:id)
       end
     end
 
-    def create_from_worktype
-      work_ids = ActiveFedora::SolrService.query("has_model_ssim:#{importerexporter.export_source}", rows: 2_000_000_000).map(&:id)
-      work_ids.each_with_index do |wid, index|
+    def create_new_entries
+      current_work_ids.each_with_index do |wid, index|
         break if limit_reached?(limit, index)
         new_entry = find_or_create_entry(entry_class, wid, 'Bulkrax::Exporter')
         Bulkrax::ExportWorkJob.perform_now(new_entry.id, current_run.id)
       end
     end
+    alias create_from_collection create_new_entries
+    alias create_from_importer create_new_entries
+    alias create_from_worktype create_new_entries
 
     def entry_class
       CsvEntry
@@ -181,7 +186,7 @@ def retrieve_cloud_files(files)
 
     def write_files
       CSV.open(setup_export_file, "w", headers: export_headers, write_headers: true) do |csv|
-        importerexporter.entries.each do |e|
+        importerexporter.entries.where(identifier: current_work_ids)[0..limit || total].each do |e|
           csv << e.parsed_metadata
         end
       end

app/views/bulkrax/exporters/_form.html.erb

@@ -60,6 +60,23 @@
     hint: 'leave blank or 0 for all records',
     label: t('bulkrax.exporter.labels.limit') %>
 
+  <%= form.input :date_filter,
+                 as: :boolean,
+                 label: t('bulkrax.exporter.labels.filter_by_date') %>
+  <div id="date_filter_picker" class="hidden">
+    <%= form.input :start_date,
+                   as: :date,
+                   label: t('bulkrax.exporter.labels.start_date') %>
+
+    <%= form.input :finish_date,
+                   as: :date,
+                   label: t('bulkrax.exporter.labels.finish_date') %>
+  </div>
+  <%= form.input :work_visibility,
+                 collection: form.object.work_status_list,
+                 label: t('bulkrax.exporter.labels.status') %>
+
+
   <%= form.input :parser_klass, 
     collection: Bulkrax.parsers.map {|p| [p[:name], p[:class_name], {'data-partial' => p[:partial]}] if p[:class_name].constantize.export_supported? }.compact, 
     label: t('bulkrax.exporter.labels.export_format') %>
@@ -85,5 +102,16 @@
       var selectedVal = $('.exporter_export_from option:selected').val();
       hideUnhide(selectedVal);
     });
+
+    // get the date filter option and show the corresponding date selectors
+    $('.exporter_date_filter').change(function() {
+        if($('.exporter_date_filter').find(".boolean").is(":checked"))
+            $('#date_filter_picker').removeClass('hidden');
+        else
+            $('#date_filter_picker').addClass('hidden');
+    });
+
+    if($('.exporter_date_filter').find(".boolean").is(":checked"))
+        $('#date_filter_picker').removeClass('hidden');
   });
 </script>

config/locales/bulkrax.en.yml

@@ -12,13 +12,17 @@ en:
         export_source: Export Source
         export_type: Type of Export
         field_mapping: Field Mapping
+        filter_by_date: Filter By Date
+        finish_date: End Date
         full: Metadata and Files
         importer: Importer
         limit: Limit
         metadata: Metadata Only
         name: Name
         parser_fields: Parser Fields
         parser_klass: Parser
+        start_date: Start Date
+        status: Status
         total_work_entries: Total Works
         user: User
         worktype: Work Type

db/migrate/20201106014204_add_date_filter_and_status_to_bulkrax_exporters.rb

@@ -0,0 +1,7 @@
+class AddDateFilterAndStatusToBulkraxExporters < ActiveRecord::Migration[5.1]
+  def change
+    add_column :bulkrax_exporters, :start_date, :date
+    add_column :bulkrax_exporters, :finish_date, :date
+    add_column :bulkrax_exporters, :work_visibility, :string
+  end
+end

spec/jobs/bulkrax/exporter_job_spec.rb

@@ -13,6 +13,7 @@ module Bulkrax
       allow(exporter).to receive(:exporter_runs).and_return([bulkrax_exporter_run])
       allow(exporter).to receive(:mapping).and_return("title" => {})
       exporter.setup_export_path
+      allow(exporter.parser).to receive(:write_files).and_return(exporter.exporter_export_path)
     end
 
     describe 'successful job', clean_downloads: true do

spec/parsers/bulkrax/csv_parser_spec.rb

@@ -121,88 +121,7 @@ module Bulkrax
       end
     end
 
-    describe '#create_from_importer' do
-      subject(:parser) { described_class.new(exporter) }
-      let(:exporter)   { FactoryBot.create(:bulkrax_exporter, export_source: importer.id) }
-      let(:importer)   { FactoryBot.create(:bulkrax_importer_csv, entries: [entry_1, entry_2, entry_3]) }
-      let(:entry_1)    { FactoryBot.create(:bulkrax_csv_entry) }
-      let(:entry_2)    { FactoryBot.create(:bulkrax_csv_entry) }
-      let(:entry_3)    { FactoryBot.create(:bulkrax_csv_entry_collection) }
-
-      it 'invokes Bulkrax::ExportWorkJob once per non-Collection Entry' do
-        expect(ActiveFedora::SolrService)
-          .to receive(:query)
-          .and_return([{ id: SecureRandom.alphanumeric(9) }])
-          .exactly(2).times
-        expect(Bulkrax::ExportWorkJob).to receive(:perform_now).exactly(2).times
-        parser.create_from_importer
-      end
-
-      context 'with an export limit of 1' do
-        let(:exporter) { FactoryBot.create(:bulkrax_exporter, export_source: importer.id, limit: 1) }
-
-        it 'invokes Bulkrax::ExportWorkJob once' do
-          expect(ActiveFedora::SolrService)
-            .to receive(:query)
-            .and_return([{ id: SecureRandom.alphanumeric(9) }])
-            .exactly(1).times
-          expect(Bulkrax::ExportWorkJob).to receive(:perform_now).exactly(1).times
-          parser.create_from_importer
-        end
-      end
-
-      context 'with an export limit of 0' do
-        let(:exporter) { FactoryBot.create(:bulkrax_exporter, export_source: importer.id, limit: 0) }
-
-        it 'invokes Bulkrax::ExportWorkJob once per non-Collection Entry' do
-          expect(ActiveFedora::SolrService)
-            .to receive(:query)
-            .and_return([{ id: SecureRandom.alphanumeric(9) }])
-            .exactly(2).times
-          expect(Bulkrax::ExportWorkJob).to receive(:perform_now).exactly(2).times
-          parser.create_from_importer
-        end
-      end
-    end
-
-    describe '#create_from_collection' do
-      subject(:parser) { described_class.new(exporter) }
-      let(:exporter)   { FactoryBot.create(:bulkrax_exporter_collection) }
-
-      it 'invokes Bulkrax::ExportWorkJob once per Entry' do
-        # Use OpenStructs to simulate the behavior of ActiveFedora::SolrHit instances.
-        work_ids = [OpenStruct.new(id: SecureRandom.alphanumeric(9)), OpenStruct.new(id: SecureRandom.alphanumeric(9))]
-        expect(ActiveFedora::SolrService).to receive(:query).and_return(work_ids)
-        expect(Bulkrax::ExportWorkJob).to receive(:perform_now).exactly(2).times
-        parser.create_from_collection
-      end
-
-      context 'with an export limit of 1' do
-        let(:exporter) { FactoryBot.create(:bulkrax_exporter_collection, limit: 1) }
-
-        it 'invokes Bulkrax::ExportWorkJob once' do
-          # Use OpenStructs to simulate the behavior of ActiveFedora::SolrHit instances.
-          work_ids = [OpenStruct.new(id: SecureRandom.alphanumeric(9)), OpenStruct.new(id: SecureRandom.alphanumeric(9))]
-          expect(ActiveFedora::SolrService).to receive(:query).and_return(work_ids)
-          expect(Bulkrax::ExportWorkJob).to receive(:perform_now).exactly(1).times
-          parser.create_from_collection
-        end
-      end
-
-      context 'with an export limit of 0' do
-        let(:exporter) { FactoryBot.create(:bulkrax_exporter_collection, limit: 0) }
-
-        it 'invokes Bulkrax::ExportWorkJob once per Entry' do
-          # Use OpenStructs to simulate the behavior of ActiveFedora::SolrHit instances.
-          work_ids = [OpenStruct.new(id: SecureRandom.alphanumeric(9)), OpenStruct.new(id: SecureRandom.alphanumeric(9))]
-          expect(ActiveFedora::SolrService).to receive(:query).and_return(work_ids)
-          expect(Bulkrax::ExportWorkJob).to receive(:perform_now).exactly(2).times
-          parser.create_from_collection
-        end
-      end
-    end
-
-    describe '#create_from_worktype' do
+    describe '#create_new_entries' do
       subject(:parser) { described_class.new(exporter) }
       let(:exporter)   { FactoryBot.create(:bulkrax_exporter_worktype) }
 
@@ -211,7 +130,7 @@ module Bulkrax
         work_ids = [OpenStruct.new(id: SecureRandom.alphanumeric(9)), OpenStruct.new(id: SecureRandom.alphanumeric(9))]
         expect(ActiveFedora::SolrService).to receive(:query).and_return(work_ids)
         expect(Bulkrax::ExportWorkJob).to receive(:perform_now).exactly(2).times
-        parser.create_from_worktype
+        parser.create_new_entries
       end
 
       context 'with an export limit of 1' do
@@ -222,7 +141,7 @@ module Bulkrax
           work_ids = [OpenStruct.new(id: SecureRandom.alphanumeric(9)), OpenStruct.new(id: SecureRandom.alphanumeric(9))]
           expect(ActiveFedora::SolrService).to receive(:query).and_return(work_ids)
           expect(Bulkrax::ExportWorkJob).to receive(:perform_now).exactly(1).times
-          parser.create_from_worktype
+          parser.create_new_entries
         end
       end
 
@@ -234,7 +153,7 @@ module Bulkrax
           work_ids = [OpenStruct.new(id: SecureRandom.alphanumeric(9)), OpenStruct.new(id: SecureRandom.alphanumeric(9))]
           expect(ActiveFedora::SolrService).to receive(:query).and_return(work_ids)
           expect(Bulkrax::ExportWorkJob).to receive(:perform_now).exactly(2).times
-          parser.create_from_worktype
+          parser.create_new_entries
         end
       end
     end