Actually switch to Solid Queue + Mission Control (#589)

Gemfile

@@ -27,8 +27,8 @@ gem "image_processing", ">= 1.2"
 gem "active_storage_validations"
 
 # Background jobs
-gem "sidekiq"
-gem "sidekiq-cron"
+gem "solid_queue"
+gem "mission_control-jobs"
 
 # API
 gem "jbuilder" # JSON templating

Gemfile.lock

@@ -298,6 +298,16 @@ GEM
     mini_magick (4.13.2)
     mini_mime (1.1.5)
     minitest (5.25.2)
+    mission_control-jobs (0.6.0)
+      actioncable (>= 7.1)
+      actionpack (>= 7.1)
+      activejob (>= 7.1)
+      activerecord (>= 7.1)
+      importmap-rails (>= 1.2.1)
+      irb (~> 1.13)
+      railties (>= 7.1)
+      stimulus-rails
+      turbo-rails
     msgpack (1.7.2)
     net-http (0.6.0)
       uri
@@ -404,19 +414,16 @@ GEM
     sass-embedded (1.81.0-x86_64-linux-gnu)
       google-protobuf (~> 4.28)
     securerandom (0.4.0)
-    sidekiq (7.3.2)
-      concurrent-ruby (< 2)
-      connection_pool (>= 2.3.0)
-      logger
-      rack (>= 2.2.4)
-      redis-client (>= 0.22.2)
-    sidekiq-cron (1.12.0)
-      fugit (~> 1.8)
-      globalid (>= 1.0.1)
-      sidekiq (>= 6)
     simple_form (5.3.1)
       actionpack (>= 5.2)
       activemodel (>= 5.2)
+    solid_queue (1.0.2)
+      activejob (>= 7.1)
+      activerecord (>= 7.1)
+      concurrent-ruby (>= 1.3.1)
+      fugit (~> 1.11.0)
+      railties (>= 7.1)
+      thor (~> 1.3.1)
     sprockets (4.2.1)
       concurrent-ruby (~> 1.0)
       rack (>= 2.2.4, < 4)
@@ -514,16 +521,16 @@ DEPENDENCIES
   letter_opener_web
   local_time
   lograge
+  mission_control-jobs
   pagy
   premailer-rails
   puma
   rack-cors
   rack-mini-profiler
   rails!
   redis
-  sidekiq
-  sidekiq-cron
   simple_form
+  solid_queue
   sprockets-rails
   sqlite3
   standard

README.md

@@ -19,8 +19,8 @@ _The thing that powers [hackathons.hackclub.com](https://hackathons.hackclub.com
 
 ## Contributing
 
-This app is built with 🛤️ [Ruby on Rails](https://rubyonrails.org/) (running [on the edge](https://shopify.engineering/living-on-the-edge-of-rails))
-and uses 🥋 [Sidekiq](https://sidekiq.org/)/[Redis](https://redis.io/) for running background jobs.
+This app is built with 🛤️ [Ruby on Rails](https://rubyonrails.org) (running [on the edge](https://shopify.engineering/living-on-the-edge-of-rails))
+and uses 🥋 [Solid Queue](https://github.com/rails/solid_queue) for running background jobs.
 
 ### Getting Started
 
@@ -65,7 +65,7 @@ brew install vips
 - Heroku
   - Redis (Heroku Data for Redis `premium0`)
 - Hetzner
-  - Runs the Rails app and Sidekiq (3 vCPU, 4 GB)
+  - Runs the Rails app and Solid Queue (3 vCPU, 4 GB)
   - Deployed via [Kamal](https://kamal-deploy.org)
 
 ### Kamal
@@ -92,12 +92,12 @@ Then, run the following locally on your computer:
 bin/console prod
 ```
 
-### Sidekiq
+### Solid Queue
 
-Sidekiq is used to process background jobs in production. In development, we use
+Solid Queue is used to process background jobs in production. In development, we use
 the good old default Active Job Async queue adapter.
 
-To check up on Sidekiq, visit `/admin/sidekiq` on the production site. You must
+To check up on jobs, visit `/admin/jobs` on the production site. You must
 be logged in as an admin to access this page.
 
 ---

app/jobs/hackathons/digests_delivery_job.rb

@@ -1,6 +1,4 @@
 class Hackathons::DigestsDeliveryJob < ApplicationJob
-  sidekiq_options retry: 20 # up to 6.5 days
-
   def perform
     sent_digest_ids = []
     current_subscribers.find_each do |subscriber|

config/deploy.yml

@@ -16,7 +16,7 @@ servers:
   jobs:
     hosts:
       - app-1.hackathons.hackclub.com
-    cmd: bundle exec sidekiq
+    cmd: bundle exec rails solid_queue:start
     options:
       memory: 1G
       cpus: 1

config/environments/production.rb

@@ -70,7 +70,7 @@
   # Use a different cache store in production.
   # config.cache_store = :mem_cache_store
 
-  config.active_job.queue_adapter = :sidekiq
+  config.active_job.queue_adapter = :solid_queue
 
   # Send emails via SMTP using Amazon SES
   config.action_mailer.delivery_method = :smtp

config/initializers/action_mailer_concurrency.rb

@@ -0,0 +1,4 @@
+Rails.application.config.after_initialize do
+  # AWS SES rate-limits to 14/second, so we'll set it to 10 to be safe
+  ActionMailer::MailDeliveryJob.limits_concurrency key: "mail", to: 10 * 1.minute.to_i, duration: 1.minute
+end

config/queue.yml

@@ -0,0 +1,15 @@
+default: &default
+  workers:
+  - queues:
+    - critical
+    - default
+    - low
+
+development:
+  <<: *default
+
+test:
+  <<: *default
+
+production:
+  <<: *default

config/recurring.yml

@@ -0,0 +1,12 @@
+production:
+  digest_deliveries:
+    class: Hackathons::DigestsDeliveryJob
+    schedule: every Tuesday @ 10am in America/Los_Angeles
+
+  website_status_refreshes:
+    class: Hackathons::WebsiteStatusesRefreshJob
+    schedule: every day @ 9am in America/Los_Angeles
+
+  website_archivals:
+    class: Hackathons::WebsiteArchivalsJob
+    schedule: every Monday @ 10am in America/Los_Angeles

config/routes.rb

@@ -1,6 +1,3 @@
-require "sidekiq/web"
-require "sidekiq/cron/web"
-
 Rails.application.routes.draw do
   root "hackathons#index"
 
@@ -28,10 +25,10 @@
   resources :database_dumps, except: [:new, :show]
 
   constraints Constraints::Admin do
-    mount Sidekiq::Web => "/admin/sidekiq" if Rails.env.production?
-    mount Audits1984::Engine => "/admin/audits"
-
     namespace :admin do
+      mount MissionControl::Jobs::Engine => "jobs" if Rails.env.production?
+      mount Audits1984::Engine => "audits"
+
       resources :hackathons, except: [:new, :create] do
         scope module: :hackathons do
           resource :hold, only: :create

db/migrate/20241125161900_create_solid_queue_tables.rb

@@ -0,0 +1,131 @@
+class CreateSolidQueueTables < ActiveRecord::Migration[8.0]
+  def change
+    create_table "solid_queue_blocked_executions", force: :cascade do |t|
+      t.bigint "job_id", null: false
+      t.string "queue_name", null: false
+      t.integer "priority", default: 0, null: false
+      t.string "concurrency_key", null: false
+      t.datetime "expires_at", null: false
+      t.datetime "created_at", null: false
+      t.index ["concurrency_key", "priority", "job_id"], name: "index_solid_queue_blocked_executions_for_release"
+      t.index ["expires_at", "concurrency_key"], name: "index_solid_queue_blocked_executions_for_maintenance"
+      t.index ["job_id"], name: "index_solid_queue_blocked_executions_on_job_id", unique: true
+    end
+
+    create_table "solid_queue_claimed_executions", force: :cascade do |t|
+      t.bigint "job_id", null: false
+      t.bigint "process_id"
+      t.datetime "created_at", null: false
+      t.index ["job_id"], name: "index_solid_queue_claimed_executions_on_job_id", unique: true
+      t.index ["process_id", "job_id"], name: "index_solid_queue_claimed_executions_on_process_id_and_job_id"
+    end
+
+    create_table "solid_queue_failed_executions", force: :cascade do |t|
+      t.bigint "job_id", null: false
+      t.text "error"
+      t.datetime "created_at", null: false
+      t.index ["job_id"], name: "index_solid_queue_failed_executions_on_job_id", unique: true
+    end
+
+    create_table "solid_queue_jobs", force: :cascade do |t|
+      t.string "queue_name", null: false
+      t.string "class_name", null: false
+      t.text "arguments"
+      t.integer "priority", default: 0, null: false
+      t.string "active_job_id"
+      t.datetime "scheduled_at"
+      t.datetime "finished_at"
+      t.string "concurrency_key"
+      t.datetime "created_at", null: false
+      t.datetime "updated_at", null: false
+      t.index ["active_job_id"], name: "index_solid_queue_jobs_on_active_job_id"
+      t.index ["class_name"], name: "index_solid_queue_jobs_on_class_name"
+      t.index ["finished_at"], name: "index_solid_queue_jobs_on_finished_at"
+      t.index ["queue_name", "finished_at"], name: "index_solid_queue_jobs_for_filtering"
+      t.index ["scheduled_at", "finished_at"], name: "index_solid_queue_jobs_for_alerting"
+    end
+
+    create_table "solid_queue_pauses", force: :cascade do |t|
+      t.string "queue_name", null: false
+      t.datetime "created_at", null: false
+      t.index ["queue_name"], name: "index_solid_queue_pauses_on_queue_name", unique: true
+    end
+
+    create_table "solid_queue_processes", force: :cascade do |t|
+      t.string "kind", null: false
+      t.datetime "last_heartbeat_at", null: false
+      t.bigint "supervisor_id"
+      t.integer "pid", null: false
+      t.string "hostname"
+      t.text "metadata"
+      t.datetime "created_at", null: false
+      t.string "name", null: false
+      t.index ["last_heartbeat_at"], name: "index_solid_queue_processes_on_last_heartbeat_at"
+      t.index ["name", "supervisor_id"], name: "index_solid_queue_processes_on_name_and_supervisor_id", unique: true
+      t.index ["supervisor_id"], name: "index_solid_queue_processes_on_supervisor_id"
+    end
+
+    create_table "solid_queue_ready_executions", force: :cascade do |t|
+      t.bigint "job_id", null: false
+      t.string "queue_name", null: false
+      t.integer "priority", default: 0, null: false
+      t.datetime "created_at", null: false
+      t.index ["job_id"], name: "index_solid_queue_ready_executions_on_job_id", unique: true
+      t.index ["priority", "job_id"], name: "index_solid_queue_poll_all"
+      t.index ["queue_name", "priority", "job_id"], name: "index_solid_queue_poll_by_queue"
+    end
+
+    create_table "solid_queue_recurring_executions", force: :cascade do |t|
+      t.bigint "job_id", null: false
+      t.string "task_key", null: false
+      t.datetime "run_at", null: false
+      t.datetime "created_at", null: false
+      t.index ["job_id"], name: "index_solid_queue_recurring_executions_on_job_id", unique: true
+      t.index ["task_key", "run_at"], name: "index_solid_queue_recurring_executions_on_task_key_and_run_at", unique: true
+    end
+
+    create_table "solid_queue_recurring_tasks", force: :cascade do |t|
+      t.string "key", null: false
+      t.string "schedule", null: false
+      t.string "command", limit: 2048
+      t.string "class_name"
+      t.text "arguments"
+      t.string "queue_name"
+      t.integer "priority", default: 0
+      t.boolean "static", default: true, null: false
+      t.text "description"
+      t.datetime "created_at", null: false
+      t.datetime "updated_at", null: false
+      t.index ["key"], name: "index_solid_queue_recurring_tasks_on_key", unique: true
+      t.index ["static"], name: "index_solid_queue_recurring_tasks_on_static"
+    end
+
+    create_table "solid_queue_scheduled_executions", force: :cascade do |t|
+      t.bigint "job_id", null: false
+      t.string "queue_name", null: false
+      t.integer "priority", default: 0, null: false
+      t.datetime "scheduled_at", null: false
+      t.datetime "created_at", null: false
+      t.index ["job_id"], name: "index_solid_queue_scheduled_executions_on_job_id", unique: true
+      t.index ["scheduled_at", "priority", "job_id"], name: "index_solid_queue_dispatch_all"
+    end
+
+    create_table "solid_queue_semaphores", force: :cascade do |t|
+      t.string "key", null: false
+      t.integer "value", default: 1, null: false
+      t.datetime "expires_at", null: false
+      t.datetime "created_at", null: false
+      t.datetime "updated_at", null: false
+      t.index ["expires_at"], name: "index_solid_queue_semaphores_on_expires_at"
+      t.index ["key", "value"], name: "index_solid_queue_semaphores_on_key_and_value"
+      t.index ["key"], name: "index_solid_queue_semaphores_on_key", unique: true
+    end
+
+    add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+    add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+    add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+    add_foreign_key "solid_queue_ready_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+    add_foreign_key "solid_queue_recurring_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+    add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+  end
+end