working version, still lots of todos and missing tests

.rspec

@@ -0,0 +1 @@
+--require spec_helper

Gemfile

@@ -11,4 +11,7 @@ gemspec
 # your gem to rubygems.org.
 
 # To use a debugger
-# gem 'byebug', group: [:development, :test]
+gem 'byebug', group: [:development, :test]
+gem "rspec-rails"
+gem "factory_bot_rails"
+gem "oai"

Gemfile.lock

@@ -0,0 +1,167 @@
+PATH
+  remote: .
+  specs:
+    bulkrax (0.1.0)
+      iso8601 (~> 0.9.0)
+      language_list (~> 1.2, >= 1.2.1)
+      libxml-ruby (~> 3.1.0)
+      oai (~> 0.4)
+      rails (~> 5.1.6)
+      simple_form (~> 3.2, <= 3.5.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actioncable (5.1.6)
+      actionpack (= 5.1.6)
+      nio4r (~> 2.0)
+      websocket-driver (~> 0.6.1)
+    actionmailer (5.1.6)
+      actionpack (= 5.1.6)
+      actionview (= 5.1.6)
+      activejob (= 5.1.6)
+      mail (~> 2.5, >= 2.5.4)
+      rails-dom-testing (~> 2.0)
+    actionpack (5.1.6)
+      actionview (= 5.1.6)
+      activesupport (= 5.1.6)
+      rack (~> 2.0)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    actionview (5.1.6)
+      activesupport (= 5.1.6)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.3)
+    activejob (5.1.6)
+      activesupport (= 5.1.6)
+      globalid (>= 0.3.6)
+    activemodel (5.1.6)
+      activesupport (= 5.1.6)
+    activerecord (5.1.6)
+      activemodel (= 5.1.6)
+      activesupport (= 5.1.6)
+      arel (~> 8.0)
+    activesupport (5.1.6)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    arel (8.0.0)
+    builder (3.2.3)
+    byebug (10.0.2)
+    concurrent-ruby (1.0.5)
+    crass (1.0.4)
+    diff-lcs (1.3)
+    erubi (1.7.1)
+    factory_bot (4.11.1)
+      activesupport (>= 3.0.0)
+    factory_bot_rails (4.11.1)
+      factory_bot (~> 4.11.1)
+      railties (>= 3.0.0)
+    faraday (0.15.3)
+      multipart-post (>= 1.2, < 3)
+    faraday_middleware (0.12.2)
+      faraday (>= 0.7.4, < 1.0)
+    globalid (0.4.1)
+      activesupport (>= 4.2.0)
+    i18n (1.1.0)
+      concurrent-ruby (~> 1.0)
+    iso8601 (0.9.1)
+    language_list (1.2.1)
+    libxml-ruby (3.1.0)
+    loofah (2.2.2)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
+    mail (2.7.1)
+      mini_mime (>= 0.1.1)
+    method_source (0.9.0)
+    mini_mime (1.0.1)
+    mini_portile2 (2.3.0)
+    minitest (5.11.3)
+    multipart-post (2.0.0)
+    nio4r (2.3.1)
+    nokogiri (1.8.5)
+      mini_portile2 (~> 2.3.0)
+    oai (0.4.0)
+      builder (>= 3.1.0)
+      faraday
+      faraday_middleware
+    rack (2.0.5)
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rails (5.1.6)
+      actioncable (= 5.1.6)
+      actionmailer (= 5.1.6)
+      actionpack (= 5.1.6)
+      actionview (= 5.1.6)
+      activejob (= 5.1.6)
+      activemodel (= 5.1.6)
+      activerecord (= 5.1.6)
+      activesupport (= 5.1.6)
+      bundler (>= 1.3.0)
+      railties (= 5.1.6)
+      sprockets-rails (>= 2.0.0)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.0.4)
+      loofah (~> 2.2, >= 2.2.2)
+    railties (5.1.6)
+      actionpack (= 5.1.6)
+      activesupport (= 5.1.6)
+      method_source
+      rake (>= 0.8.7)
+      thor (>= 0.18.1, < 2.0)
+    rake (12.3.1)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-rails (3.8.0)
+      actionpack (>= 3.0)
+      activesupport (>= 3.0)
+      railties (>= 3.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    simple_form (3.5.0)
+      actionpack (> 4, < 5.2)
+      activemodel (> 4, < 5.2)
+    sprockets (3.7.2)
+      concurrent-ruby (~> 1.0)
+      rack (> 1, < 3)
+    sprockets-rails (3.2.1)
+      actionpack (>= 4.0)
+      activesupport (>= 4.0)
+      sprockets (>= 3.0.0)
+    sqlite3 (1.3.13)
+    thor (0.20.0)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+    websocket-driver (0.6.5)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bulkrax!
+  byebug
+  factory_bot_rails
+  oai
+  rspec-rails
+  sqlite3
+
+BUNDLED WITH
+   1.17.2

app/assets/javascripts/bulkrax/application.js

@@ -10,4 +10,5 @@
 // Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
 // about supported directives.
 //
+
 //= require_tree .

app/assets/javascripts/bulkrax/importers.js

@@ -0,0 +1,66 @@
+// Place all the behaviors and hooks related to the matching controller here.
+// All this logic will automatically be available in application.js.
+$(document).ready(function() {
+  var refresh_button = $('.refresh-set-source')
+  var base_url = $('#importer_parser_fields_base_url')
+  var external_set_select = $("#importer_parser_fields_set")
+  var initial_base_url = base_url.val()
+
+  // handle refreshing/loading of external setes via button click
+  $('body').on('click', '.refresh-set-source', function(e) {
+    e.preventDefault()
+
+    handleSourceLoad(refresh_button, base_url, external_set_select)
+  })
+
+  // handle refreshing/loading of external sets via blur event for the base_url field
+  $('body').on('blur', '#importer_parser_fields_base_url', function(e) {
+    e.preventDefault()
+
+    // ensure we don't make another query if the value is the same -- this can be forced by clicking the refresh button
+    if (initial_base_url != base_url.val()) {
+      handleSourceLoad(refresh_button, base_url, external_set_select)
+      initial_base_url = base_url.val()
+    }
+  })
+});
+
+function handleSourceLoad(refresh_button, base_url, external_set_select) {
+  if (base_url.val() == "") { // ignore empty base_url value
+    return
+  }
+
+  var initial_button_text = refresh_button.html()
+
+  refresh_button.html('Refreshing...')
+  refresh_button.attr('disabled', true)
+
+  $.post('/importers/external_sets', {
+    base_url: base_url.val(),
+  }, function(res) {
+    if (!res.error) {
+      genExternalSetOptions(external_set_select, res.sets) // sets is [[name, spec]...]
+    } else {
+      setError(external_set_select, res.error)
+    }
+
+    refresh_button.html(initial_button_text)
+    refresh_button.attr('disabled', false)
+  })
+}
+
+function genExternalSetOptions(selector, sets) {
+  out = '<option value="">- Select One -</option>'
+
+  out += sets.map(function(set) {
+    return '<option value="'+set[1]+'">'+set[0]+'</option>'
+  })
+
+  selector.html(out)
+  selector.attr('disabled', false)
+}
+
+function setError(selector, error) {
+  selector.html('<option value="none">Error - Please enter Base URL and try again</option>')
+  selector.attr('disabled', true)
+}

app/controllers/bulkrax/importers_controller.rb

@@ -0,0 +1,108 @@
+require_dependency "bulkrax/application_controller"
+require_dependency "oai"
+
+module Bulkrax
+  class ImportersController < ApplicationController
+    include Hyrax::ThemedLayoutController
+
+    before_action :set_importer, only: [:show, :edit, :update, :destroy]
+    with_themed_layout 'dashboard'
+
+    # GET /importers
+    def index
+      add_breadcrumb t(:'hyrax.controls.home'), main_app.root_path
+      add_breadcrumb t(:'hyrax.dashboard.breadcrumbs.admin'), hyrax.dashboard_path
+      add_breadcrumb 'Importers', bulkrax.importers_path
+      @importers = Importer.all
+    end
+
+    # GET /importers/1
+    def show
+    end
+
+    # GET /importers/new
+    def new
+      add_breadcrumb t(:'hyrax.controls.home'), main_app.root_path
+      add_breadcrumb t(:'hyrax.dashboard.breadcrumbs.admin'), hyrax.dashboard_path
+      add_breadcrumb 'Importers', bulkrax.importers_path
+      @importer = Importer.new
+    end
+
+    # GET /importers/1/edit
+    def edit
+      add_breadcrumb t(:'hyrax.controls.home'), main_app.root_path
+      add_breadcrumb t(:'hyrax.dashboard.breadcrumbs.admin'), hyrax.dashboard_path
+      add_breadcrumb 'Importers', bulkrax.importers_path
+    end
+
+    # POST /importers
+    def create
+      @importer = Importer.new(importer_params)
+
+      if @importer.save
+        redirect_to @importer, notice: 'Importer was successfully created.'
+      else
+        render :new
+      end
+    end
+
+    # PATCH/PUT /importers/1
+    def update
+      if @importer.update(importer_params)
+        redirect_to @importer, notice: 'Importer was successfully updated.'
+      else
+        render :edit
+      end
+    end
+
+    # DELETE /importers/1
+    def destroy
+      @importer.destroy
+      redirect_to importers_url, notice: 'Importer was successfully destroyed.'
+    end
+
+    def external_sets
+      if list_external_sets
+        render json: { base_url: params[:base_url], sets: @sets }
+      else
+        render json: { base_url: params[:base_url], error: "unable to pull data from #{params[:base_url]}" }
+      end
+    end
+
+    private
+      # Use callbacks to share common setup or constraints between actions.
+      def set_importer
+        @importer = Importer.find(params[:id])
+      end
+
+      # Only allow a trusted parameter "white list" through.
+      def importer_params
+        params.require(:importer).permit(:name, :admin_set_id, :user_id, :frequency, :parser_klass, :limit, parser_fields: {}, field_mapping: {})
+      end
+
+      def list_external_sets
+        url = params[:base_url] || (@harvester ? @harvester.base_url : nil)
+        setup_client(url) if url.present?
+
+        @sets = [['All', 'all']]
+
+        begin
+          @client.list_sets.each do |s|
+            @sets << [s.name, s.spec]
+          end
+        rescue
+          return false
+        end
+
+        @sets
+      end
+
+      def setup_client(url)
+        return false if url.nil?
+
+        headers = { from: '[email protected]' }
+
+        @client ||= OAI::Client.new(url, headers: headers, parser: 'libxml', metadata_prefix: 'oai_dc')
+      end
+  end
+end

app/helpers/bulkrax/application_helper.rb

@@ -1,4 +1,6 @@
 module Bulkrax
   module ApplicationHelper
+    include Hyrax::HyraxHelperBehavior
+
   end
 end

app/helpers/bulkrax/importers_helper.rb

@@ -0,0 +1,11 @@
+module Bulkrax
+  module ImportersHelper
+    # borrowd from batch-importer https://github.com/samvera-labs/hyrax-batch_ingest/blob/master/app/controllers/hyrax/batch_ingest/batches_controller.rb
+    def available_admin_sets
+      # Restrict available_admin_sets to only those current user can desposit to.
+      @available_admin_sets ||= Hyrax::Collections::PermissionsService.source_ids_for_deposit(ability: current_ability, source_type: 'admin_set').map do |admin_set_id|
+        [AdminSet.find(admin_set_id).title.first, admin_set_id]
+      end
+    end
+  end
+end

app/jobs/bulkrax/import_work_job.rb

@@ -0,0 +1,10 @@
+module Bulkrax
+  class ImportWorkJob < ApplicationJob
+    queue_as :import
+
+    def perform(*args)
+      @importer = Importer.find(args[0])
+      @importer.import_work(args[1])
+    end
+  end
+end