From 9cd703f85dec7df2a72776a24024eee4741f6318 Mon Sep 17 00:00:00 2001 From: Matt Almeida Date: Fri, 10 Jan 2025 14:30:24 -0500 Subject: [PATCH] Coolify! --- .env.erb | 10 ----- .github/workflows/cd.yml | 29 -------------- .gitignore | 1 - .kamal/hooks/post-deploy | 3 -- .kamal/secrets | 9 ----- Dockerfile | 2 +- Gemfile | 10 ++--- Gemfile.lock | 55 +++++++++++++------------- README.md | 32 --------------- bin/console | 47 ---------------------- config/deploy.yml | 62 ------------------------------ config/initializers/console1984.rb | 2 + config/initializers/litestream.rb | 7 ++++ config/litestream.yml | 14 +++---- config/puma.rb | 1 + docker-compose.yml | 26 +++++++++++++ 16 files changed, 76 insertions(+), 234 deletions(-) delete mode 100644 .env.erb delete mode 100644 .github/workflows/cd.yml delete mode 100755 .kamal/hooks/post-deploy delete mode 100644 .kamal/secrets delete mode 100755 bin/console delete mode 100644 config/deploy.yml create mode 100644 config/initializers/litestream.rb create mode 100644 docker-compose.yml diff --git a/.env.erb b/.env.erb deleted file mode 100644 index a8b9babd..00000000 --- a/.env.erb +++ /dev/null @@ -1,10 +0,0 @@ -RAILS_MASTER_KEY=<%= ENV["RAILS_MASTER_KEY"] %> - -<% ENV["RAILS_ENV"] = "production" %> -<% require_relative "config/application" %> -<% aws = Rails.application.credentials.aws! %> - -LITESTREAM_ACCESS_KEY_ID: <%= aws.access_key_id %> -LITESTREAM_SECRET_ACCESS_KEY: <%= aws.secret_access_key %> -LITESTREAM_BUCKET: <%= aws.bucket %> -LITESTREAM_REGION: <%= aws.region %> diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml deleted file mode 100644 index 1efa7e69..00000000 --- a/.github/workflows/cd.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: CD -on: - push: - branches: [main] -jobs: - deploy: - name: Deploy - runs-on: ubuntu-latest - steps: - - name: Configure SSH - uses: shimataro/ssh-key-action@v2 - with: - key: ${{ secrets.SSH_KEY }} - known_hosts: ${{ vars.KNOWN_HOSTS }} - - name: Checkout code - uses: actions/checkout@v4 - - name: Setup Ruby - uses: ruby/setup-ruby@v1 - with: - bundler-cache: true - - name: Setup Docker - uses: docker/setup-buildx-action@v3 - - uses: crazy-max/ghaction-github-runtime@v3 - - name: Deploy to production! - run: bundle exec kamal deploy - env: - KAMAL_REGISTRY_USERNAME: ${{ github.actor }} - KAMAL_REGISTRY_PASSWORD: ${{ github.token }} - RAILS_MASTER_KEY: ${{ secrets.RAILS_MASTER_KEY }} diff --git a/.gitignore b/.gitignore index e6d53535..fb6072f2 100644 --- a/.gitignore +++ b/.gitignore @@ -36,7 +36,6 @@ # Environment variables .env* !.env*example -!.env.erb # IDEs .idea diff --git a/.kamal/hooks/post-deploy b/.kamal/hooks/post-deploy deleted file mode 100755 index 3056c84f..00000000 --- a/.kamal/hooks/post-deploy +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -echo "Deployed $KAMAL_VERSION -> $KAMAL_DESTINATION in $KAMAL_RUNTIME seconds" diff --git a/.kamal/secrets b/.kamal/secrets deleted file mode 100644 index cd533d83..00000000 --- a/.kamal/secrets +++ /dev/null @@ -1,9 +0,0 @@ -KAMAL_REGISTRY_USERNAME=$KAMAL_REGISTRY_USERNAME -KAMAL_REGISTRY_PASSWORD=$KAMAL_REGISTRY_PASSWORD - -RAILS_MASTER_KEY=$RAILS_MASTER_KEY - -LITESTREAM_ACCESS_KEY_ID=$LITESTREAM_ACCESS_KEY_ID -LITESTREAM_SECRET_ACCESS_KEY=$LITESTREAM_SECRET_ACCESS_KEY -LITESTREAM_BUCKET=$LITESTREAM_BUCKET -LITESTREAM_REGION=$LITESTREAM_REGION diff --git a/Dockerfile b/Dockerfile index 432a8c0a..1c7d3ff0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ ENV RAILS_ENV="production" \ FROM base AS build RUN apt-get update -qq && \ - apt-get install --no-install-recommends -y build-essential git libvips pkg-config sqlite3 + apt-get install --no-install-recommends -y build-essential git pkg-config libyaml-dev sqlite3 libvips COPY .ruby-version Gemfile Gemfile.lock ./ RUN bundle install && \ diff --git a/Gemfile b/Gemfile index 4b8b8589..fb70af31 100644 --- a/Gemfile +++ b/Gemfile @@ -54,13 +54,13 @@ gem "hashid-rails" # Non-sequential IDs gem "tzinfo-data", platforms: %i[mingw mswin x64_mingw jruby] # Windows doesn't include zoneinfo files gem "bootsnap", require: false # reduces boot times through caching; required in config/boot.rb -gem "kamal" # Deployments -gem "console1984" -gem "audits1984" +gem "litestream" -# Application monitoring (errors, performance, etc.) gem "appsignal" -gem "lograge" # Log formatting +gem "lograge" + +gem "console1984" +gem "audits1984" group :development, :test do gem "debug", platforms: %i[mri mingw x64_mingw] diff --git a/Gemfile.lock b/Gemfile.lock index b2ca4602..dc0a5675 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -136,8 +136,6 @@ GEM aws-sigv4 (1.10.1) aws-eventstream (~> 1, >= 1.0.2) base64 (0.2.0) - bcrypt_pbkdf (1.1.1) - bcrypt_pbkdf (1.1.1-arm64-darwin) benchmark (0.4.0) better_errors (2.10.1) erubi (>= 1.0.0) @@ -194,7 +192,6 @@ GEM dotenv (= 3.1.7) railties (>= 6.1) drb (2.2.1) - ed25519 (1.3.0) erubi (1.13.1) et-orbi (1.2.11) tzinfo @@ -255,17 +252,6 @@ GEM activesupport (>= 5.0.0) jmespath (1.6.2) json (2.9.1) - kamal (2.4.0) - activesupport (>= 7.0) - base64 (~> 0.2) - bcrypt_pbkdf (~> 1.0) - concurrent-ruby (~> 1.2) - dotenv (~> 3.1) - ed25519 (~> 1.2) - net-ssh (~> 7.3) - sshkit (>= 1.23.0, < 2.0) - thor (~> 1.3) - zeitwerk (>= 2.6.18, < 3.0) language_server-protocol (3.17.0.3) launchy (3.0.1) addressable (~> 2.8) @@ -278,7 +264,32 @@ GEM railties (>= 6.1) rexml lint_roller (1.1.0) + litestream (0.12.0) + actionpack (>= 7.0) + actionview (>= 7.0) + activejob (>= 7.0) + activesupport (>= 7.0) + logfmt (>= 0.0.10) + railties (>= 7.0) + sqlite3 + litestream (0.12.0-arm64-darwin) + actionpack (>= 7.0) + actionview (>= 7.0) + activejob (>= 7.0) + activesupport (>= 7.0) + logfmt (>= 0.0.10) + railties (>= 7.0) + sqlite3 + litestream (0.12.0-x86_64-linux) + actionpack (>= 7.0) + actionview (>= 7.0) + activejob (>= 7.0) + activesupport (>= 7.0) + logfmt (>= 0.0.10) + railties (>= 7.0) + sqlite3 local_time (3.0.2) + logfmt (0.0.10) logger (1.6.4) lograge (0.14.0) actionpack (>= 4) @@ -318,13 +329,8 @@ GEM net-protocol net-protocol (0.2.2) timeout - net-scp (4.0.0) - net-ssh (>= 2.6.5, < 8.0.0) - net-sftp (4.0.0) - net-ssh (>= 5.0.0, < 8.0.0) net-smtp (0.5.0) net-protocol - net-ssh (7.3.0) nio4r (2.7.4) nokogiri (1.18.1-aarch64-linux-gnu) racc (~> 1.4) @@ -332,7 +338,6 @@ GEM racc (~> 1.4) nokogiri (1.18.1-x86_64-linux-gnu) racc (~> 1.4) - ostruct (0.6.1) pagy (6.4.3) parallel (1.26.3) parser (3.3.6.0) @@ -379,7 +384,7 @@ GEM regexp_parser (2.10.0) reline (0.6.0) io-console (~> 0.5) - request_store (1.5.1) + request_store (1.6.0) rack (>= 1.4) rexml (3.3.9) rinku (2.0.6) @@ -435,12 +440,6 @@ GEM sqlite3 (2.5.0-aarch64-linux-gnu) sqlite3 (2.5.0-arm64-darwin) sqlite3 (2.5.0-x86_64-linux-gnu) - sshkit (1.23.2) - base64 - net-scp (>= 1.1.2) - net-sftp (>= 2.1.2) - net-ssh (>= 2.8.0) - ostruct standard (1.43.0) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.0) @@ -520,8 +519,8 @@ DEPENDENCIES image_processing (>= 1.2) importmap-rails jbuilder - kamal letter_opener_web + litestream local_time lograge mission_control-jobs diff --git a/README.md b/README.md index 3df03655..3831f05f 100644 --- a/README.md +++ b/README.md @@ -58,38 +58,6 @@ dependency installed on your machine. For macs, run: brew install vips ``` -## Production Deployment - -**Vendors:** - -- Hetzner - - Runs the Rails app and Solid Queue (3 vCPU, 4 GB) - - Deployed via [Kamal](https://kamal-deploy.org) - -### Kamal - -All pushes to the `main` branch are automatically deployed by Kamal. - -- Environment variables are stored on GitHub and accessed by GitHub Actions - when deploying. -- Deployments take 2-5 minutes to complete. -- After pushing to `main`, please monitor the `CD / Deploy` check for the status - of the deployment. - -### Production Rails Console - -We audit the use of the production console with [`console1984`](https://github.com/basecamp/console1984) -and [`audits1984`](https://github.com/basecamp/audits1984). - -To use the production console, you must first have SSH access to the Hetzner -server(s). Please ask [`@garyhtou`](https://garytou.com) for access. - -Then, run the following locally on your computer: - -```sh -bin/console prod -``` - ### Solid Queue Solid Queue is used to process background jobs in production. In development, we use diff --git a/bin/console b/bin/console deleted file mode 100755 index 0e980164..00000000 --- a/bin/console +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env ruby - -CONSOLE_USER_FILE = "tmp/.console-user" -ENVIRONMENTS = { - %w[prod production] => :production, - %w[dev development] => :development, -} -environment = ENVIRONMENTS.find { |k, _| k.include? ARGV[0] }&.last || :development - -def console_user - ask_for_console_user while (user = read_console_user).to_s.empty? - user -end - -def ask_for_console_user - print "To access the production console, enter your name: " - File.write(CONSOLE_USER_FILE, STDIN.gets.chomp) -end - -def read_console_user - File.read(CONSOLE_USER_FILE).strip if File.exist?(CONSOLE_USER_FILE) -end - -def escape_str(string) - string.gsub("\"", "\\\"") -end - -def kamal_exec(cmd) - exec "bundle exec kamal app exec --reuse -i \"#{cmd.gsub("\"", "\\\"")}\"" -end - -if environment == :production - begin - puts "❗ ARE YOU SURE YOU WANT TO ACCESS THE **REAL** PRODUCTION CONSOLE? (y/N)" - confirmed = STDIN.gets&.chomp == "y" # `gets` may raise Interrupt - - raise Interrupt unless confirmed - user = console_user - puts # blank line to separate from Kamal output - - kamal_exec "env CONSOLE_USER=\"#{user}\" bin/rails console" - rescue Interrupt - puts "\nphew! you're safe 😅" - end -else - exec "bin/rails console" -end diff --git a/config/deploy.yml b/config/deploy.yml deleted file mode 100644 index 27043c61..00000000 --- a/config/deploy.yml +++ /dev/null @@ -1,62 +0,0 @@ -<% ENV["RAILS_ENV"] = "production" %> -<% require_relative "config/application" %> - -minimum_version: 2.2.2 - -service: hackathons -image: hackclub/hackathons - -servers: - web: - hosts: - - app-1.hackathons.hackclub.com - options: - memory: 2G - cpus: 2 - jobs: - hosts: - - app-1.hackathons.hackclub.com - cmd: bundle exec rails solid_queue:start - options: - memory: 1G - cpus: 1 - -registry: - server: ghcr.io - username: - - KAMAL_REGISTRY_USERNAME - password: - - KAMAL_REGISTRY_PASSWORD - -builder: - arch: amd64 - cache: - type: gha - options: mode=max - -env: - clear: - HOST: dash.hackathons.hackclub.com - WEB_CONCURRENCY: 2 - secret: - - RAILS_MASTER_KEY - -asset_path: "/hackathons/public/assets" -volumes: ["hackathons-storage:/hackathons/storage"] - -proxy: - app_port: 3000 - -accessories: - litestream: - roles: ["web"] - image: litestream/litestream - files: ["config/litestream.yml:/etc/litestream.yml"] - volumes: ["hackathons-storage:/storage"] - cmd: replicate - env: - secret: - <% %w[BUCKET REGION ACCESS_KEY_ID SECRET_ACCESS_KEY].each do |secret| %> - <% ENV["LITESTREAM_#{secret}"] = Rails.application.credentials.aws!.fetch(secret.downcase.to_sym) %> - - LITESTREAM_<%= secret %> - <% end %> diff --git a/config/initializers/console1984.rb b/config/initializers/console1984.rb index 7b1cad8f..27319950 100644 --- a/config/initializers/console1984.rb +++ b/config/initializers/console1984.rb @@ -1,5 +1,7 @@ Rails.application.configure do + config.console1984.ask_for_username_if_empty = true config.console1984.incinerate = false + config.console1984.production_data_warning = <<~WARNING You have access to production data here. That's a big deal. To keep sensitive info safe and private, we audit the commands you type here. diff --git a/config/initializers/litestream.rb b/config/initializers/litestream.rb new file mode 100644 index 00000000..dac5a018 --- /dev/null +++ b/config/initializers/litestream.rb @@ -0,0 +1,7 @@ +Rails.application.configure do + aws = Rails.application.credentials.aws + + config.litestream.replica_bucket = aws&.bucket + config.litestream.replica_key_id = aws&.access_key_id + config.litestream.replica_access_key = aws&.secret_access_key +end diff --git a/config/litestream.yml b/config/litestream.yml index e88cc14f..6ad961c8 100644 --- a/config/litestream.yml +++ b/config/litestream.yml @@ -1,8 +1,8 @@ -access-key-id: $LITESTREAM_ACCESS_KEY_ID -secret-access-key: $LITESTREAM_SECRET_ACCESS_KEY - dbs: -- path: /storage/production.sqlite3 - replicas: - - url: s3://${LITESTREAM_BUCKET}/db - region: $LITESTREAM_REGION + - path: storage/production.sqlite3 + replicas: + - type: s3 + bucket: $LITESTREAM_REPLICA_BUCKET + path: production.sqlite3 + access-key-id: $LITESTREAM_ACCESS_KEY_ID + secret-access-key: $LITESTREAM_SECRET_ACCESS_KEY diff --git a/config/puma.rb b/config/puma.rb index bda2fe66..2199ab68 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -41,3 +41,4 @@ $LOAD_PATH << File.expand_path("../lib", __dir__) plugin :dartsass if Rails.env.development? +plugin :litestream if Rails.env.production? diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..ea6c1496 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,26 @@ +services: + web: + build: . + deploy: + resources: + limits: + cpus: 1 + memory: 2G + environment: + WEB_CONCURRENCY: 2 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost/up"] + volumes: ["storage:/hackathons/storage"] + jobs: + volumes: ["storage:/hackathons/storage"] + build: . + depends_on: [web] + command: bin/rails solid_queue:start + deploy: + resources: + limits: + cpus: 1 + memory: 1G + +volumes: + storage: {}