Merge pull request #727 from scientist-softserv/i721-send-jobs-to-aux…

…iliary

Split jobs to a new queue

.env

@@ -1,4 +1,4 @@
-AUXILIARY_QUEUE_PRIORITY=100
+AUXILIARY_QUEUE_TENANTS="sdapi"
 CHROME_HOSTNAME=chrome
 DATABASE_ADAPTER=postgresql
 DATABASE_HOST=db

Gemfile

@@ -89,7 +89,7 @@ group :development do
   gem 'spring-watcher-listen', '~> 2.0.0'
 end
 
-gem 'bulkrax', '~> 5.0'
+gem 'bulkrax', '~> 5.3.0'
 
 gem 'blacklight', '~> 6.7'
 gem 'blacklight_oai_provider', '~> 6.1', '>= 6.1.1'
@@ -142,7 +142,7 @@ gem 'blacklight_range_limit', '6.5.0'
 gem 'derivative-rodeo', git: 'https://github.com/scientist-softserv/derivative_rodeo.git', branch: 'main'
 gem 'dog_biscuits', git: 'https://github.com/samvera-labs/dog_biscuits.git'
 gem 'hyrax-v2_graph_indexer', git: 'https://github.com/scientist-softserv/hyrax-v2_graph_indexer', branch: 'main'
-gem 'iiif_print', git: 'https://github.com/scientist-softserv/iiif_print.git', branch: 'main'
+gem 'iiif_print', git: 'https://github.com/scientist-softserv/iiif_print.git', branch: 'adventist-custom-queue'
 gem 'order_already'
 gem 'redis', '~> 4.0'
 gem 'redlock', '~> 1.2.1'

Gemfile.lock

@@ -31,8 +31,8 @@ GIT
 
 GIT
   remote: https://github.com/scientist-softserv/iiif_print.git
-  revision: e476998ab453afabf1bcb8afa059b4416af9b705
-  branch: main
+  revision: a9b270c2f07af4cc11111e971b50e4aeb0dc44b6
+  branch: adventist-custom-queue
   specs:
     iiif_print (1.0.0)
       blacklight_iiif_search (>= 1.0, < 3.0)
@@ -158,7 +158,7 @@ GEM
     babel-transpiler (0.7.0)
       babel-source (>= 4.0, < 6)
       execjs (~> 2.0)
-    bagit (0.4.5)
+    bagit (0.4.6)
       docopt (~> 0.5.0)
       validatable (~> 1.6)
     bcp47 (0.3.3)
@@ -216,7 +216,7 @@ GEM
       signet (~> 0.8)
       typhoeus
     builder (3.2.4)
-    bulkrax (5.3.0)
+    bulkrax (5.3.1)
       bagit (~> 0.4)
       coderay
       dry-monads (~> 1.4.0)
@@ -1163,7 +1163,7 @@ DEPENDENCIES
   blacklight_oai_provider (~> 6.1, >= 6.1.1)
   blacklight_range_limit (= 6.5.0)
   bootstrap-datepicker-rails
-  bulkrax (~> 5.0)
+  bulkrax (~> 5.3.0)
   byebug
   capybara
   carrierwave-aws (~> 1.3)

app/jobs/application_job.rb

@@ -1,18 +1,65 @@
 # frozen_string_literal: true
 
 class ApplicationJob < ActiveJob::Base
-  before_enqueue :set_auxiliary_queue_priority
+  PRIORITY_QUEUE_NAME = :auxiliary
+  PRIORITY_QUEUE_ADDITIONAL_PRIORITY = 110
 
   # limit to 5 attempts
   retry_on StandardError, wait: :exponentially_longer, attempts: 5 do |_job, _exception|
     # Log error, do nothing, etc.
   end
 
+  queue_as Hyrax.config.ingest_queue_name
+
+  queue_with_priority do
+    # we used to do these via GoodJobs config, but that was swallowing this queue_with_priority method.
+    # Note: Higher priority numbers run first in all versions of GoodJob v3.x and below.
+    # GoodJob v4.x will change job priority to give smaller numbers higher priority (default: 0),
+    # in accordance with Active Job's definition of priority.
+    case self
+    when  Bulkrax::ScheduleRelationshipsJob
+      calculate_priority base: 50
+    when  CreateDerivativesJob
+      calculate_priority base: 40
+    when  CharacterizeJob
+      calculate_priority base: 30
+    when  Hyrax::GrantEditToMembersJob, ImportUrlJob, IngestJob
+      calculate_priority base: 10
+    when  AttachFilesToWorkJob
+      calculate_priority base: -1
+    when  Bulkrax::ImportWorkJob
+      calculate_priority base: -5
+    when  Bulkrax::ImportFileSetJob
+      calculate_priority base: -15
+    when  IiifPrint::Jobs::ChildWorksFromPdfJob
+      calculate_priority base: -17
+    when  Bulkrax::CreateRelationshipsJob, Bulkrax::ImporterJob, IiifPrint::Jobs::CreateRelationshipsJob
+      calculate_priority base: -20
+    when  ContentDepositEventJob, ContentUpdateEventJob
+      calculate_priority base: -50
+    else
+      calculate_priority base: 0
+    end
+  end
+
   private
 
-    def set_auxiliary_queue_priority
-      return unless queue_name.to_sym == :auxiliary
+    def redirect_priority_jobs
+      return :ingest unless priority_tenants_array.include? tenant_name
+      PRIORITY_QUEUE_NAME
+    end
+
+    def calculate_priority(base:)
+      return base unless priority_tenants_array.include? tenant_name
+      PRIORITY_QUEUE_ADDITIONAL_PRIORITY + base
+    end
+
+    def priority_tenants_array
+      @priority_array ||= ENV.fetch('AUXILIARY_QUEUE_TENANTS', '').split(',')
+    end
 
-      self.priority = ENV.fetch('AUXILIARY_QUEUE_PRIORITY', 100).to_i
+    def tenant_name
+      return nil unless current_tenant
+      Account.find_by(tenant: current_tenant)&.name
     end
 end

config/initializers/good_job.rb

@@ -31,29 +31,10 @@
 end
 
 # Wrapping this in an after_initialize block to ensure that all constants are loaded
-Rails.application.config.after_initialize do
-  # baseline of 0, higher is sooner
-
-  # Commented out the following two jobs because they were
-  # specfically used for the sdapi ingests.
-  # see sdapi_ingest_script directory and
-  # ref: https://github.com/scientist-softserv/adventist-dl/issues/468
-  # CollectionMembershipJob.priority = 70
-  # UpdateCollectionMembershipJob.priority = 60
-  Bulkrax::ScheduleRelationshipsJob.priority = 50
-  CreateDerivativesJob.priority = 40
-  CharacterizeJob.priority = 30
-  Hyrax::GrantEditToMembersJob.priority = 10
-  ImportUrlJob.priority = 10
-  IngestJob.priority = 10
-  ApplicationJob.priority = 0
-  AttachFilesToWorkJob.priority = -1
-  Bulkrax::ImportWorkJob.priority = -5
-  Bulkrax::ImportFileSetJob.priority = -15
-  IiifPrint::Jobs::ChildWorksFromPdfJob.priority = -17
-  Bulkrax::CreateRelationshipsJob.priority = -20
-  Bulkrax::ImporterJob.priority = -20
-  IiifPrint::Jobs::CreateRelationshipsJob.priority = -20
-  ContentDepositEventJob.priority = -50
-  ContentUpdateEventJob.priority = -50
-end
+# Rails.application.config.after_initialize do
+#   # baseline of 0, higher is sooner
+
+#   Job prioritization has been moved to application_job. Using
+#   the GoodJob config swallowed queue_with_priority behavior.
+
+# end

config/initializers/hyrax.rb

@@ -1,4 +1,6 @@
 Hyrax.config do |config|
+  config.ingest_queue_name = -> { redirect_priority_jobs }
+
   config.register_curation_concern :generic_work
   # Injected via `rails g hyrax:work Image`
   config.register_curation_concern :image

lib/active_job_tenant.rb

@@ -10,6 +10,14 @@ module ActiveJobTenant
   end
 
   module ClassMethods
+    def queue_as(part_name = nil, &block)
+      if part_name.is_a?(Proc)
+        self.queue_name = part_name
+      else
+        super
+      end
+    end
+
     def deserialize(job_data)
       super.tap do |job|
         job.tenant = job_data['tenant']

ops/dev-deploy.tmpl.yaml

@@ -93,8 +93,8 @@ extraEnvVars: &envVars
     value: /rest
   - name: FEDORA_URL
     value: http://fcrepo.default.svc.cluster.local:8080/rest
-  - name: AUXILIARY_QUEUE_PRIORITY
-    value: "100"
+  - name: AUXILIARY_QUEUE_TENANTS
+    value: "sdapi"
   - name: IN_DOCKER
     value: "true"
   - name: LD_LIBRARY_PATH