From 6c2bb3035a1c2e0a14bde5bac7fa8f618b9faef2 Mon Sep 17 00:00:00 2001 From: Ruslan Forostianov Date: Tue, 19 Apr 2022 10:52:37 +0200 Subject: [PATCH 01/13] WIP: start dockerizing civic-server --- .dockerignore | 1 + Dockerfile | 13 +++++++++++++ config/database.yml | 2 +- docker-compose.yml | 11 +++++++++++ 4 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..94143827 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..ea730499 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM ruby:2 + +RUN apt-get update +RUN apt-get install -y rbenv libxml2 libxslt-dev openssl nodejs +COPY $PWD /civic-server +WORKDIR /civic-server +RUN gem install bundler +RUN bundle update --bundler +RUN rbenv rehash +RUN bundle install +RUN rbenv rehash +RUN rake db:create +RUN rake db:migrate diff --git a/config/database.yml b/config/database.yml index 33ea5846..be33e606 100644 --- a/config/database.yml +++ b/config/database.yml @@ -2,7 +2,7 @@ default: &default adapter: postgresql pool: 5 encoding: unicode - host: localhost + host: <%= ENV['DATABASE_HOST'] %> development: <<: *default diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..33e8631f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,11 @@ +version: '3' + +services: + civic: + build: . + environment: + DATABASE_HOST: db + db: + image: postgres:14 + ports: + - "5432:5432" From eb8bced0399f04ac1cdd8c105c8637f81fbe91b6 Mon Sep 17 00:00:00 2001 From: Pieter Lukasse Date: Tue, 19 Apr 2022 15:20:20 +0200 Subject: [PATCH 02/13] adding entrypoint to initialize DB when necessary --- Dockerfile | 4 +--- docker-entrypoint.sh | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 docker-entrypoint.sh diff --git a/Dockerfile b/Dockerfile index ea730499..b549f0c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM ruby:2 RUN apt-get update -RUN apt-get install -y rbenv libxml2 libxslt-dev openssl nodejs +RUN apt-get install -y rbenv libxml2 libxslt-dev openssl nodejs postgresql-client COPY $PWD /civic-server WORKDIR /civic-server RUN gem install bundler @@ -9,5 +9,3 @@ RUN bundle update --bundler RUN rbenv rehash RUN bundle install RUN rbenv rehash -RUN rake db:create -RUN rake db:migrate diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100644 index 00000000..8aca8711 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -e +set -u # unset variables throw error +set -o pipefail # pipes fail when partial command fails + +# make sure env $DATABASE_* items are set! +# Attempt connect to db: +psql postgresql://[$DATABASE_USER[:$DATABASE_PASSWORD]@][$DATABASE_HOST][:5432][/civic] + +# If it fails, then assume db does not exist and +# db creation should be executed: +# TODO - add IF +if .... + rake db:create + + # migrate: + rake db:migrate + + # load the sanitized version of a recent database backup found in ./db: + rake civic:load[force] +fi + +rails s + +# TODO - add this .sh as entrypoint in the Dockerfile... From 6dea64252797453adee7cefa8c44c23f186034ca Mon Sep 17 00:00:00 2001 From: Ruslan Forostianov Date: Thu, 21 Apr 2022 08:08:29 +0200 Subject: [PATCH 03/13] Use the docker entry point --- Dockerfile | 18 +++++++++++++----- config/database.yml | 2 ++ docker-compose.yml | 14 ++++++++++++-- docker-entrypoint.sh | 11 +++++++++-- 4 files changed, 36 insertions(+), 9 deletions(-) mode change 100644 => 100755 docker-entrypoint.sh diff --git a/Dockerfile b/Dockerfile index b549f0c3..042db32c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,19 @@ FROM ruby:2 +ENV RAILS_ENV=production RUN apt-get update RUN apt-get install -y rbenv libxml2 libxslt-dev openssl nodejs postgresql-client +RUN gem install bundler + +#RUN mkdir /civic-server +#RUN cd /civic-server && bundle install + +COPY Gemfile /civic-server/Gemfile +COPY Gemfile.lock /civic-server/Gemfile.lock + +RUN cd /civic-server && bundle update --bundler && rbenv rehash && bundle install + COPY $PWD /civic-server WORKDIR /civic-server -RUN gem install bundler -RUN bundle update --bundler -RUN rbenv rehash -RUN bundle install -RUN rbenv rehash + +ENTRYPOINT ["/civic-server/docker-entrypoint.sh"] diff --git a/config/database.yml b/config/database.yml index be33e606..30515af6 100644 --- a/config/database.yml +++ b/config/database.yml @@ -3,6 +3,8 @@ default: &default pool: 5 encoding: unicode host: <%= ENV['DATABASE_HOST'] %> + username: <%= ENV['DATABASE_USER'] %> + password: <%= ENV['DATABASE_PASSWORD'] %> development: <<: *default diff --git a/docker-compose.yml b/docker-compose.yml index 33e8631f..76bf81a2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,17 @@ services: build: . environment: DATABASE_HOST: db + DATABASE_USER: postgres + DATABASE_PASSWORD: psw + depends_on: + db: + condition: service_healthy db: image: postgres:14 - ports: - - "5432:5432" + environment: + POSTGRES_PASSWORD: psw + healthcheck: + test: "PASSWORD=psw pg_isready -U postgres" + interval: 10s + timeout: 5s + retries: 5 diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh old mode 100644 new mode 100755 index 8aca8711..7a499bae --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -4,21 +4,28 @@ set -e set -u # unset variables throw error set -o pipefail # pipes fail when partial command fails +bundle update --bundler # make sure env $DATABASE_* items are set! # Attempt connect to db: -psql postgresql://[$DATABASE_USER[:$DATABASE_PASSWORD]@][$DATABASE_HOST][:5432][/civic] +# psql postgresql://[$DATABASE_USER[:$DATABASE_PASSWORD]@][$DATABASE_HOST][:5432][/civic] # If it fails, then assume db does not exist and # db creation should be executed: # TODO - add IF -if .... +if PGPASSWORD=$DATABASE_PASSWORD psql -lqt -U$DATABASE_USER -h$DATABASE_HOST | cut -d \| -f 1 | grep -qw civic; then + echo civic database already exists. Skipping database initialisation. +else + echo Creating civic database rake db:create + echo Migrating civic database # migrate: rake db:migrate + echo Loading a recent database backup # load the sanitized version of a recent database backup found in ./db: rake civic:load[force] + echo Done fi rails s From 0348b6d6f1903277b554748b167056a1c2e2121e Mon Sep 17 00:00:00 2001 From: Pieter Lukasse Date: Thu, 21 Apr 2022 11:45:32 +0200 Subject: [PATCH 04/13] skipping big folders to avoid sending them to docker context --- .dockerignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.dockerignore b/.dockerignore index 94143827..144c8ce4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,4 @@ Dockerfile +.git +.github +public \ No newline at end of file From af1a3406c9bca84109192a4e923d9746b0739f3b Mon Sep 17 00:00:00 2001 From: Pieter Lukasse Date: Thu, 21 Apr 2022 12:54:12 +0200 Subject: [PATCH 05/13] removed runtime ENV from Dockerfile --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 042db32c..a802e597 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,5 @@ FROM ruby:2 -ENV RAILS_ENV=production RUN apt-get update RUN apt-get install -y rbenv libxml2 libxslt-dev openssl nodejs postgresql-client RUN gem install bundler From a6a09261288e81203b2be085be5085e3d103264b Mon Sep 17 00:00:00 2001 From: Pieter Lukasse Date: Thu, 21 Apr 2022 12:56:31 +0200 Subject: [PATCH 06/13] set RAILS_ENV to dev; simplified db depends_on; downgraded postgres to 13.5 to match app expectations TODO - remove db exposed port --- docker-compose.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 76bf81a2..d5360162 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,14 +4,17 @@ services: civic: build: . environment: + # setting to "development" for now as "production" also requires other infra related items that are not documented (?) + RAILS_ENV: development DATABASE_HOST: db DATABASE_USER: postgres DATABASE_PASSWORD: psw depends_on: - db: - condition: service_healthy + - db db: - image: postgres:14 + image: postgres:13.5 + ports: + - "5432:5432" environment: POSTGRES_PASSWORD: psw healthcheck: From fe82950b2df0e3085783c6c068134e8b86ddbd2c Mon Sep 17 00:00:00 2001 From: Pieter Lukasse Date: Thu, 21 Apr 2022 12:58:24 +0200 Subject: [PATCH 07/13] removed TODO notes for parts that are implemented; made grep check more generic to support civic_dev.. db as well --- docker-entrypoint.sh | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 7a499bae..de48eef2 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -4,30 +4,29 @@ set -e set -u # unset variables throw error set -o pipefail # pipes fail when partial command fails -bundle update --bundler -# make sure env $DATABASE_* items are set! -# Attempt connect to db: -# psql postgresql://[$DATABASE_USER[:$DATABASE_PASSWORD]@][$DATABASE_HOST][:5432][/civic] +bundle update --bundler -# If it fails, then assume db does not exist and +# Note: make sure env $DATABASE_* items are set! + +# Attempt connect to db and if +# it fails, then assume db does not exist and # db creation should be executed: -# TODO - add IF -if PGPASSWORD=$DATABASE_PASSWORD psql -lqt -U$DATABASE_USER -h$DATABASE_HOST | cut -d \| -f 1 | grep -qw civic; then - echo civic database already exists. Skipping database initialisation. +if PGPASSWORD=$DATABASE_PASSWORD psql -lqt -U$DATABASE_USER -h$DATABASE_HOST | cut -d \| -f 1 | grep -q civic; then + echo "civic database already exists. Skipping database initialisation." else - echo Creating civic database + echo "Creating civic database" rake db:create - echo Migrating civic database + echo "Migrating civic database" # migrate: rake db:migrate - echo Loading a recent database backup + echo "Loading a recent database backup" + # unpack ./db/data.sql.gz (required before loading data dump): + gunzip ./db/data.sql.gz # load the sanitized version of a recent database backup found in ./db: rake civic:load[force] - echo Done + echo "Done" fi rails s - -# TODO - add this .sh as entrypoint in the Dockerfile... From c21bd24af44c7ddceaa33d7cc37d05599f5e9cc6 Mon Sep 17 00:00:00 2001 From: Pieter Lukasse Date: Thu, 21 Apr 2022 14:19:56 +0200 Subject: [PATCH 08/13] exposing civic port --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index d5360162..25a8f9fe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,8 @@ version: '3' services: civic: build: . + ports: + - "3000:3000" environment: # setting to "development" for now as "production" also requires other infra related items that are not documented (?) RAILS_ENV: development From be532814dfec68c0765839366aab266f9ed5f70f Mon Sep 17 00:00:00 2001 From: Pieter Lukasse Date: Fri, 22 Apr 2022 13:40:12 +0200 Subject: [PATCH 09/13] fix to ensure the server is really listening --- docker-entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index de48eef2..22a8cfe9 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -29,4 +29,5 @@ else echo "Done" fi -rails s +echo "Starting server now. You can test it with e.g. http://localhost:3000/api/genes/3845?identifier_type=entrez_id" +rails server -b 0.0.0.0 From 19434f16998d657ac63c7280ce4735f991087c2b Mon Sep 17 00:00:00 2001 From: Ruslan Forostianov Date: Sun, 8 May 2022 18:22:19 +0200 Subject: [PATCH 10/13] Start civic in production envirounment --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 25a8f9fe..5c8b1774 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,8 +6,8 @@ services: ports: - "3000:3000" environment: - # setting to "development" for now as "production" also requires other infra related items that are not documented (?) - RAILS_ENV: development + RAILS_ENV: production + SECRET_KEY_BASE: production_key_eretred547568 DATABASE_HOST: db DATABASE_USER: postgres DATABASE_PASSWORD: psw From fb7860170b3a58858b23a359f6d717710faff8ec Mon Sep 17 00:00:00 2001 From: Ruslan Forostianov Date: Sun, 8 May 2022 18:22:44 +0200 Subject: [PATCH 11/13] Use puma if it is production --- Gemfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Gemfile b/Gemfile index 1d62d385..acbccbbf 100644 --- a/Gemfile +++ b/Gemfile @@ -62,3 +62,7 @@ group :development do gem 'capistrano-rbenv', '~> 2.0' gem 'capistrano-passenger', '~> 0.2.0' end + +group :production do + gem 'puma', '~> 5.6.4' +end From d71d54bbfda45b62fd37a4642aeeed88a88d42f2 Mon Sep 17 00:00:00 2001 From: Pieter Lukasse Date: Mon, 9 May 2022 11:28:57 +0200 Subject: [PATCH 12/13] change to production config to get it working again --- config/environments/production.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index 589211f4..1251b886 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -36,12 +36,15 @@ # `config.assets.precompile` has moved to config/initializers/assets.rb + #run background tasks syncronously + config.active_job.queue_adapter = :inline + # Specifies the header that your server uses for sending files. # config.action_dispatch.x_sendfile_header = "X-Sendfile" # for apache # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for nginx # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. - config.force_ssl = true + #config.force_ssl = true # Set to :debug to see everything in the log. config.log_level = :info From 63d762cb3b1d5ce1977f9cbbde4d62759b5a5907 Mon Sep 17 00:00:00 2001 From: Ruslan Forostianov Date: Thu, 12 May 2022 15:36:01 +0200 Subject: [PATCH 13/13] Initialize database instead of full-blown migration --- docker-entrypoint.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 22a8cfe9..ea9ee438 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -17,9 +17,8 @@ else echo "Creating civic database" rake db:create - echo "Migrating civic database" - # migrate: - rake db:migrate + echo "Initializing civic database" + rake db:structure:load echo "Loading a recent database backup" # unpack ./db/data.sql.gz (required before loading data dump):