From be0a94158e2d63045e89924a52112ec36145b705 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Fri, 26 Jan 2024 15:32:27 -0500 Subject: [PATCH 01/39] docker-compose: Create api_cache volume to allow django container to cache concordances and nginx container to serve them --- docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index e4bb5aabf..b27ab6e5e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,7 @@ services: - ./:/code/ - static_volume:/resources/static - media_volume:/resources/media + - api_cache_volume:/resources/api_cache env_file: ./config/envs/dev_env ports: - 3000:3000 @@ -25,6 +26,7 @@ services: - ./config/nginx/conf.d:/etc/nginx/conf.d - static_volume:/resources/static - media_volume:/resources/media + - api_cache_volume:/resources/api_cache - ./certificates:/etc/nginx/ssl/live restart: always depends_on: @@ -42,3 +44,4 @@ volumes: postgres_data: static_volume: media_volume: + api_cache_volume: From 22a5d376cbf22ac3e1b2c161a539dc78ddd473b3 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Fri, 26 Jan 2024 15:34:25 -0500 Subject: [PATCH 02/39] nginx configuration: configure nginx to serve cached concordances --- config/nginx/conf.d/cantusdb.conf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config/nginx/conf.d/cantusdb.conf b/config/nginx/conf.d/cantusdb.conf index c61a19b31..8965125ef 100644 --- a/config/nginx/conf.d/cantusdb.conf +++ b/config/nginx/conf.d/cantusdb.conf @@ -46,7 +46,11 @@ server { location /media { alias /resources/media/; } - + + location /concordances { + alias /resources/api_cache/concordances.json; + } + location = /style.css { root /; } From 817e0d6a247cebccdbcd524538be10c4aecab9b8 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Fri, 26 Jan 2024 15:36:44 -0500 Subject: [PATCH 03/39] update update_cached_concordances command to write to api_cache volume --- .../main_app/management/commands/update_cached_concordances.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/cantusdb_project/main_app/management/commands/update_cached_concordances.py b/django/cantusdb_project/main_app/management/commands/update_cached_concordances.py index 8acbbd6fa..ccd0ed11e 100644 --- a/django/cantusdb_project/main_app/management/commands/update_cached_concordances.py +++ b/django/cantusdb_project/main_app/management/commands/update_cached_concordances.py @@ -10,7 +10,7 @@ class Command(BaseCommand): def handle(self, *args, **kwargs) -> None: - CACHE_DIR: str = "api_cache" + CACHE_DIR: str = "/resources/api_cache" FILEPATH: str = f"{CACHE_DIR}/concordances.json" start_time: str = datetime.now().isoformat() stdout.write(f"Running update_cached_concordances at {start_time}.\n") From 02e8c221c960ea9bae8b3c63c3dbf7dc89a4f4be Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Fri, 26 Jan 2024 16:06:17 -0500 Subject: [PATCH 04/39] update cron.txt to run update_cached_concordances daily --- cron/cron.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/cron/cron.txt b/cron/cron.txt index b37423519..34daef61f 100644 --- a/cron/cron.txt +++ b/cron/cron.txt @@ -6,5 +6,6 @@ # min hour day month weekday command 0 4 * * * bash /home/ubuntu/code/CantusDB/cron/postgres/db_backup.sh +10 4 * * * bash /home/ubuntu/code/CantusDB/cron/management/manage.sh update_cached_concordances 40 4 1 * * bash /home/ubuntu/code/CantusDB/cron/management/manage.sh populate_next_chant_fields; bash /home/ubuntu/code/CantusDB/cron/management/manage.sh populate_is_last_chant_in_feast 50 4 * * 7 /usr/local/bin/docker-compose -f /home/ubuntu/code/CantusDB/docker-compose.yml exec -T nginx lego --path /etc/nginx/ssl/live -d cantusdatabase.org -d www.cantusdatabase.org -d mass.cantusdatabase.org -m updateme@example.com --http --http.webroot /var/www/lego/ renew --days 45 --renew-hook "nginx -s reload" From 8d7e661c498aeb8e8d9b0582daa597c8da2d0819 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Tue, 30 Jan 2024 10:29:19 -0500 Subject: [PATCH 05/39] concordances API: Configure Nginx to add a 6h 'expires' response header --- config/nginx/conf.d/cantusdb.conf | 34 ++----------------------------- 1 file changed, 2 insertions(+), 32 deletions(-) diff --git a/config/nginx/conf.d/cantusdb.conf b/config/nginx/conf.d/cantusdb.conf index 8965125ef..83ff9c613 100644 --- a/config/nginx/conf.d/cantusdb.conf +++ b/config/nginx/conf.d/cantusdb.conf @@ -1,37 +1,6 @@ server { - listen 80; - - server_tokens off; - - location ^~ /.well-known/acme-challenge/ { - root /var/www/lego; - } - - location / { - return 301 https://$host$request_uri; - } -} - -server { - # Redirect all https traffic for mass.cantusdatabase.org to cantusdatabase.org - listen 443 ssl; - server_name mass.cantusdatabase.org; - - ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt; - ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key; - - location / { - return 301 https://cantusdatabase.org$request_uri; - } -} - -server { - - listen 443 default_server http2 ssl; - - ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt; - ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key; + listen 80; location / { proxy_pass http://django:8000; @@ -49,6 +18,7 @@ server { location /concordances { alias /resources/api_cache/concordances.json; + expires 6h; } location = /style.css { From 927a91267085f19695320955c52239f6d1209f46 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Tue, 30 Jan 2024 10:42:42 -0500 Subject: [PATCH 06/39] configure cantusdb.conf for Production --- config/nginx/conf.d/cantusdb.conf | 33 ++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/config/nginx/conf.d/cantusdb.conf b/config/nginx/conf.d/cantusdb.conf index 83ff9c613..fa91ecb1e 100644 --- a/config/nginx/conf.d/cantusdb.conf +++ b/config/nginx/conf.d/cantusdb.conf @@ -1,7 +1,38 @@ server { - listen 80; + server_tokens off; + + location ^~ /.well-known/acme-challenge/ { + root /var/www/lego; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { + # Redirect all https traffic for mass.cantusdatabase.org to cantusdatabase.org + listen 443 ssl; + + server_name mass.cantusdatabase.org; + + ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt; + ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key; + + location / { + return 301 https://cantusdatabase.org$request_uri; + } +} + +server { + + listen 443 default_server http2 ssl; + + ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt; + ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key; + location / { proxy_pass http://django:8000; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; From 93d49c86636336aad4fd7742b0be5b8eee04deed Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Tue, 30 Jan 2024 20:50:58 +0000 Subject: [PATCH 07/39] update_cached_concordances command: Add -d/--directory option to allow concordances file to be output to a different location --- .../commands/update_cached_concordances.py | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/django/cantusdb_project/main_app/management/commands/update_cached_concordances.py b/django/cantusdb_project/main_app/management/commands/update_cached_concordances.py index ccd0ed11e..5f6f885fa 100644 --- a/django/cantusdb_project/main_app/management/commands/update_cached_concordances.py +++ b/django/cantusdb_project/main_app/management/commands/update_cached_concordances.py @@ -1,6 +1,7 @@ import ujson import os from sys import stdout +from typing import Optional from datetime import datetime from collections import defaultdict from django.db.models.query import QuerySet @@ -9,9 +10,20 @@ class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument( + "-d", + "--directory", + help="Optional filepath specifying a directory to output concordances", + type=str, + ) + def handle(self, *args, **kwargs) -> None: - CACHE_DIR: str = "/resources/api_cache" - FILEPATH: str = f"{CACHE_DIR}/concordances.json" + cache_dir: Optional[str] = kwargs["directory"] + if not cache_dir: + cache_dir = "/resources/api_cache" + + filepath: str = f"{cache_dir}/concordances.json" start_time: str = datetime.now().isoformat() stdout.write(f"Running update_cached_concordances at {start_time}.\n") concordances: dict = get_concordances() @@ -23,18 +35,18 @@ def handle(self, *args, **kwargs) -> None: "data": concordances, "metadata": metadata, } - stdout.write(f"Attempting to make directory at {CACHE_DIR} to hold cache: ") + stdout.write(f"Attempting to make directory at {cache_dir} to hold cache: ") try: - os.mkdir(CACHE_DIR) - stdout.write(f"successfully created directory at {CACHE_DIR}.\n") + os.mkdir(cache_dir) + stdout.write(f"successfully created directory at {cache_dir}.\n") except FileExistsError: - stdout.write(f"directory at {CACHE_DIR} already exists.\n") - stdout.write(f"Writing concordances to {FILEPATH} at {write_time}.\n") - with open(FILEPATH, "w") as json_file: + stdout.write(f"directory at {cache_dir} already exists.\n") + stdout.write(f"Writing concordances to {filepath} at {write_time}.\n") + with open(filepath, "w") as json_file: ujson.dump(data_and_metadata, json_file) end_time = datetime.now().isoformat() stdout.write( - f"Concordances successfully written to {FILEPATH} at {end_time}.\n\n" + f"Concordances successfully written to {filepath} at {end_time}.\n\n" ) From 2206efab1577d5992b52051beee3c5849a8817fc Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Tue, 30 Jan 2024 15:57:50 -0500 Subject: [PATCH 08/39] update nginx to cache concordances for 24h after they are created --- config/nginx/conf.d/cantusdb.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/nginx/conf.d/cantusdb.conf b/config/nginx/conf.d/cantusdb.conf index fa91ecb1e..43208899b 100644 --- a/config/nginx/conf.d/cantusdb.conf +++ b/config/nginx/conf.d/cantusdb.conf @@ -49,7 +49,7 @@ server { location /concordances { alias /resources/api_cache/concordances.json; - expires 6h; + expires modified +24h; } location = /style.css { From e974ff4e3f2186b3558a1f83a7e0f51218be57ce Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Thu, 1 Feb 2024 15:03:09 +0000 Subject: [PATCH 09/39] update_cached_concordances command: add docstring and comments --- .../commands/update_cached_concordances.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/django/cantusdb_project/main_app/management/commands/update_cached_concordances.py b/django/cantusdb_project/main_app/management/commands/update_cached_concordances.py index 5f6f885fa..9ad1a28a6 100644 --- a/django/cantusdb_project/main_app/management/commands/update_cached_concordances.py +++ b/django/cantusdb_project/main_app/management/commands/update_cached_concordances.py @@ -9,6 +9,10 @@ from main_app.models import Chant +# Usage: `python manage.py update_cached_concordances` +# or `python manage.py update_cached_concordances -d "/path/to/directory/in/which/to/save/concordances"` + + class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( @@ -21,6 +25,8 @@ def add_arguments(self, parser): def handle(self, *args, **kwargs) -> None: cache_dir: Optional[str] = kwargs["directory"] if not cache_dir: + # this default directory should match the value in docker-compose.yml, + # at services:django:volumes:api_cache_volume cache_dir = "/resources/api_cache" filepath: str = f"{cache_dir}/concordances.json" @@ -51,6 +57,13 @@ def handle(self, *args, **kwargs) -> None: def get_concordances() -> dict: + """Fetch all published chants in the database, group them by Cantus ID, and return + a dictionary containing information on each of these chants. + + Returns: + dict: A dictionary where each key is a Cantus ID and each value is a list all + published chants in the database with that Cantus ID. + """ DOMAIN: str = "https://cantusdatabase.org" stdout.write("Querying database for published chants\n") From 5c58eef63d5efeb06ea3413c151dfbec93275d6e Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Thu, 1 Feb 2024 15:51:31 +0000 Subject: [PATCH 10/39] update_cached_concordances: clean up --directory flag logic --- .../management/commands/update_cached_concordances.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/django/cantusdb_project/main_app/management/commands/update_cached_concordances.py b/django/cantusdb_project/main_app/management/commands/update_cached_concordances.py index 9ad1a28a6..976115999 100644 --- a/django/cantusdb_project/main_app/management/commands/update_cached_concordances.py +++ b/django/cantusdb_project/main_app/management/commands/update_cached_concordances.py @@ -20,15 +20,13 @@ def add_arguments(self, parser): "--directory", help="Optional filepath specifying a directory to output concordances", type=str, - ) - - def handle(self, *args, **kwargs) -> None: - cache_dir: Optional[str] = kwargs["directory"] - if not cache_dir: # this default directory should match the value in docker-compose.yml, # at services:django:volumes:api_cache_volume - cache_dir = "/resources/api_cache" + default="/resources/api_cache", + ) + def handle(self, *args, **kwargs) -> None: + cache_dir: str = kwargs["directory"] filepath: str = f"{cache_dir}/concordances.json" start_time: str = datetime.now().isoformat() stdout.write(f"Running update_cached_concordances at {start_time}.\n") From 0c737219428143e65670a9fa354a45bca6e2d62e Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Thu, 1 Feb 2024 13:30:04 -0500 Subject: [PATCH 11/39] Create config/nginx/conf.d/cantusdb-development.conf --- config/nginx/conf.d/cantusdb-development.conf | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 config/nginx/conf.d/cantusdb-development.conf diff --git a/config/nginx/conf.d/cantusdb-development.conf b/config/nginx/conf.d/cantusdb-development.conf new file mode 100644 index 000000000..15b582c23 --- /dev/null +++ b/config/nginx/conf.d/cantusdb-development.conf @@ -0,0 +1,80 @@ +# This file is configured for deployment of the CantusDB project in local development. +# When building the project locally, replace the contents of cantusdb.conf with the +# contents of this file. + +# server { +# listen 80; +# +# server_tokens off; +# +# location ^~ /.well-known/acme-challenge/ { +# root /var/www/lego; +# } +# +# location / { +# return 301 https://$host$request_uri; +# } +# } + +# server { +# # Redirect all https traffic for mass.cantusdatabase.org to cantusdatabase.org +# listen 443 ssl; +# +# server_name mass.cantusdatabase.org; +# +# ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt; +# ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key; +# +# location / { +# return 301 https://cantusdatabase.org$request_uri; +# } +# } + +server { + + listen 443 default_server http2 ssl; + + # ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt; + # ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key; + + location / { + proxy_pass http://django:8000; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_redirect off; + } + + location /static { + alias /resources/static/; + } + location /media { + alias /resources/media/; + } + + location /concordances { + alias /resources/api_cache/concordances.json; + expires modified +24h; + } + + location = /style.css { + root /; + } + location = /background.jpg { + root /; + } + location = /CantusLogoSmall.gif { + root /; + } + location = /favicon.ico { + root /; + } + + error_page 502 /502.html; + location = /502.html { + root /; + } + error_page 504 /504.html; + location = /504.html { + root /; + } +} From 4a1fd3bd94cae9b72215c8b302e86b8597387385 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Thu, 1 Feb 2024 16:43:28 -0500 Subject: [PATCH 12/39] add docker-compose-development.yml --- docker-compose-development.yml | 51 ++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 docker-compose-development.yml diff --git a/docker-compose-development.yml b/docker-compose-development.yml new file mode 100644 index 000000000..702c77992 --- /dev/null +++ b/docker-compose-development.yml @@ -0,0 +1,51 @@ +# This file is configured for deployment of the CantusDB project in local development. +# When building the project locally, replace the contents of docker-compose.yml with the +# contents of this file. + +version: '3' + +services: + django: + build: + context: ./django + volumes: + - ./:/code/ + - static_volume:/resources/static + - media_volume:/resources/media + - api_cache_volume:/resources/api_cache + env_file: ./config/envs/dev_env + ports: + - 3000:3000 + restart: always + depends_on: + - postgres + + nginx: + build: + context: ./nginx + ports: + - 80:80 + # - 443:443 + volumes: + - ./config/nginx/conf.d:/etc/nginx/conf.d + - static_volume:/resources/static + - media_volume:/resources/media + - api_cache_volume:/resources/api_cache + - ./certificates:/etc/nginx/ssl/live + restart: always + depends_on: + - django + + postgres: + build: + context: ./postgres + env_file: ./config/envs/dev_env + volumes: + - postgres_data:/var/lib/postgresql/data/ + restart: always + +volumes: + postgres_data: + static_volume: + media_volume: + api_cache_volume: From 8d389517937cf705de009bda3a9a39ee7737fb69 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Fri, 2 Feb 2024 11:22:00 -0500 Subject: [PATCH 13/39] docker-compose-development.yml: Comment out line relating to certificates --- docker-compose-development.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-development.yml b/docker-compose-development.yml index 702c77992..0074182f2 100644 --- a/docker-compose-development.yml +++ b/docker-compose-development.yml @@ -31,7 +31,7 @@ services: - static_volume:/resources/static - media_volume:/resources/media - api_cache_volume:/resources/api_cache - - ./certificates:/etc/nginx/ssl/live + # - ./certificates:/etc/nginx/ssl/live restart: always depends_on: - django From 99575dabb6c37951e955223b560a746f2a952e6f Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Fri, 2 Feb 2024 11:26:44 -0500 Subject: [PATCH 14/39] conf.d/cantusdb-development.conf: replace listen 443 with listen 80 --- config/nginx/conf.d/cantusdb-development.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/nginx/conf.d/cantusdb-development.conf b/config/nginx/conf.d/cantusdb-development.conf index 15b582c23..034e62f7c 100644 --- a/config/nginx/conf.d/cantusdb-development.conf +++ b/config/nginx/conf.d/cantusdb-development.conf @@ -31,8 +31,8 @@ # } server { - - listen 443 default_server http2 ssl; + # listen 443 default_server http2 ssl; + listen 80; # ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt; # ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key; From 11e3450986576baae81057e17d3d83b59b940680 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Fri, 2 Feb 2024 11:35:09 -0500 Subject: [PATCH 15/39] Create conf.d/cantusdb-staging.conf --- config/nginx/conf.d/cantusdb-staging.conf | 83 +++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 config/nginx/conf.d/cantusdb-staging.conf diff --git a/config/nginx/conf.d/cantusdb-staging.conf b/config/nginx/conf.d/cantusdb-staging.conf new file mode 100644 index 000000000..a41c8849f --- /dev/null +++ b/config/nginx/conf.d/cantusdb-staging.conf @@ -0,0 +1,83 @@ +server { + listen 80; + + server_tokens off; + + location ^~ /.well-known/acme-challenge/ { + root /var/www/lego; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { + # Redirect all https traffic for staging-alias.cantusdatabase.org to staging.cantusdatabase.org + # (on Production, this redirects from mass.cantusdatabase.org to cantusdatabase.org. + # The Staging server's subdomain "staging-alias" is analogous to Production's "mass" subdomain.) + listen 443 ssl; + + # server_name mass.cantusdatabase.org; + server_name staging-alias.cantusdatabase.org; + + # ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt; + # ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key; + ssl_certificate /etc/nginx/ssl/live/certificates/staging.cantusdatabase.org.crt; + ssl_certificate_key /etc/nginx/ssl/live/certificates/staging.cantusdatabase.org.key; + + location / { + return 301 https://staging.cantusdatabase.org$request_uri; + } +} + +server { + + listen 443 default_server http2 ssl; + + # ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt; + # ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key; + ssl_certificate /etc/nginx/ssl/live/certificates/staging.cantusdatabase.org.crt; + ssl_certificate_key /etc/nginx/ssl/live/certificates/staging.cantusdatabase.org.key; + + location / { + proxy_pass http://django:8000; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_redirect off; + } + + location /static { + alias /resources/static/; + } + location /media { + alias /resources/media/; + } + + location /concordances { + alias /resources/api_cache/concordances.json; + expires modified +24h; + } + + location = /style.css { + root /; + } + location = /background.jpg { + root /; + } + location = /CantusLogoSmall.gif { + root /; + } + location = /favicon.ico { + root /; + } + + error_page 502 /502.html; + location = /502.html { + root /; + } + error_page 504 /504.html; + location = /504.html { + root /; + } +} From 000f7ec0793beaa3d03b3fc9c09c604fdb261ec5 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Fri, 2 Feb 2024 11:39:12 -0500 Subject: [PATCH 16/39] Create cron/cron-staging.txt --- cron/cron-staging.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 cron/cron-staging.txt diff --git a/cron/cron-staging.txt b/cron/cron-staging.txt new file mode 100644 index 000000000..de08ce2c0 --- /dev/null +++ b/cron/cron-staging.txt @@ -0,0 +1,12 @@ +###################### +# CantusDB Cron Jobs # +###################### + +# Note: This is set up to run on the production server. If you want to run it on your local machine, you will need to change the paths. + +# min hour day month weekday command +0 4 * * * bash /home/ubuntu/code/CantusDB/cron/postgres/db_backup.sh +10 4 * * * bash /home/ubuntu/code/CantusDB/cron/management/manage.sh update_cached_concordances +40 4 1 * * bash /home/ubuntu/code/CantusDB/cron/management/manage.sh populate_next_chant_fields; bash /home/ubuntu/code/CantusDB/cron/management/manage.sh populate_is_last_chant_in_feast +# 50 4 * * 7 /usr/local/bin/docker-compose -f /home/ubuntu/code/CantusDB/docker-compose.yml exec -T nginx lego --path /etc/nginx/ssl/live -d cantusdatabase.org -d www.cantusdatabase.org -d mass.cantusdatabase.org -m updateme@example.com --http --http.webroot /var/www/lego/ renew --days 45 --renew-hook "nginx -s reload" +50 4 * * 7 /usr/local/bin/docker-compose -f /home/ubuntu/code/CantusDB/docker-compose.yml exec -T nginx lego --path /etc/nginx/ssl/live -d staging.cantusdatabase.org -d staging-alias.cantusdatabase.org -m updateme@example.com --http --http.webroot /var/www/lego/ renew --days 45 --renew-hook "nginx -s reload" From 1c3875c2f4c95d8ae7ad65e848a4a12f4e7570b1 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Fri, 2 Feb 2024 13:14:02 -0500 Subject: [PATCH 17/39] rename conf.d files --- .../{cantusdb-development.conf => cantusdb.conf.development} | 0 .../nginx/conf.d/{cantusdb-staging.conf => cantusdb.conf.staging} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename config/nginx/conf.d/{cantusdb-development.conf => cantusdb.conf.development} (100%) rename config/nginx/conf.d/{cantusdb-staging.conf => cantusdb.conf.staging} (100%) diff --git a/config/nginx/conf.d/cantusdb-development.conf b/config/nginx/conf.d/cantusdb.conf.development similarity index 100% rename from config/nginx/conf.d/cantusdb-development.conf rename to config/nginx/conf.d/cantusdb.conf.development diff --git a/config/nginx/conf.d/cantusdb-staging.conf b/config/nginx/conf.d/cantusdb.conf.staging similarity index 100% rename from config/nginx/conf.d/cantusdb-staging.conf rename to config/nginx/conf.d/cantusdb.conf.staging From 7afecc9541ce72ec8b4f1acc8681315dbe84e206 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Fri, 2 Feb 2024 13:14:51 -0500 Subject: [PATCH 18/39] Rename docker-compose-development.yml -> docker-compose.yml.development --- docker-compose-development.yml => docker-compose.yml.development | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docker-compose-development.yml => docker-compose.yml.development (100%) diff --git a/docker-compose-development.yml b/docker-compose.yml.development similarity index 100% rename from docker-compose-development.yml rename to docker-compose.yml.development From 03320b682aca3fed747f9891149cbfdb81884326 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Fri, 2 Feb 2024 13:28:11 -0500 Subject: [PATCH 19/39] docker-compose.yml.development: Uncomment nginx certificates volume --- docker-compose.yml.development | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml.development b/docker-compose.yml.development index 0074182f2..702c77992 100644 --- a/docker-compose.yml.development +++ b/docker-compose.yml.development @@ -31,7 +31,7 @@ services: - static_volume:/resources/static - media_volume:/resources/media - api_cache_volume:/resources/api_cache - # - ./certificates:/etc/nginx/ssl/live + - ./certificates:/etc/nginx/ssl/live restart: always depends_on: - django From 724b0cf85f3a63ea1d0fd0c0fc619affeb3a947f Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Fri, 2 Feb 2024 16:03:00 -0500 Subject: [PATCH 20/39] Add `-T` flag to `docker-compose exec` in `manage.sh` --- cron/management/manage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron/management/manage.sh b/cron/management/manage.sh index 344b24a46..fdf9be76a 100644 --- a/cron/management/manage.sh +++ b/cron/management/manage.sh @@ -7,4 +7,4 @@ DOCKER_COMPOSE_FILE=/home/ubuntu/code/CantusDB/docker-compose.yml # This is the path to the docker-compose file. COMMAND=$1 # This is the command to execute. -/usr/local/bin/docker-compose -f $DOCKER_COMPOSE_FILE exec django python manage.py $COMMAND \ No newline at end of file +/usr/local/bin/docker-compose -f $DOCKER_COMPOSE_FILE exec -T django python manage.py $COMMAND From cc9d08e26b16c8ebe42db40430dfb64f1bbe2ee6 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Tue, 6 Feb 2024 11:08:33 -0500 Subject: [PATCH 21/39] docker-compose.yml: django container: remove 3000:3000 port mapping --- docker-compose.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b27ab6e5e..3e09c303d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,7 @@ +# This file is configured for deployment of the CantusDB project in local development. +# When building the project locally, replace the contents of docker-compose.yml with the +# contents of this file. + version: '3' services: @@ -10,8 +14,6 @@ services: - media_volume:/resources/media - api_cache_volume:/resources/api_cache env_file: ./config/envs/dev_env - ports: - - 3000:3000 restart: always depends_on: - postgres @@ -21,13 +23,14 @@ services: context: ./nginx ports: - 80:80 - - 443:443 + - ${PORT}:${PORT} volumes: - ./config/nginx/conf.d:/etc/nginx/conf.d - static_volume:/resources/static - media_volume:/resources/media - api_cache_volume:/resources/api_cache - - ./certificates:/etc/nginx/ssl/live + # - ./certificates:/etc/nginx/ssl/live + env_file: ./config/envs/dev_env restart: always depends_on: - django From 3634bb33ab538bb705904c53f31e96beaf174042 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Tue, 6 Feb 2024 11:11:27 -0500 Subject: [PATCH 22/39] docker-compose.yml: nginx container: Reset 443:443 port mapping and reset change I had made to make it run locally --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3e09c303d..157daf254 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,13 +23,13 @@ services: context: ./nginx ports: - 80:80 - - ${PORT}:${PORT} + - 443:443 volumes: - ./config/nginx/conf.d:/etc/nginx/conf.d - static_volume:/resources/static - media_volume:/resources/media - api_cache_volume:/resources/api_cache - # - ./certificates:/etc/nginx/ssl/live + - ./certificates:/etc/nginx/ssl/live env_file: ./config/envs/dev_env restart: always depends_on: From 11a4d60ef4cd9a4b46658f2e3c25d461434e7cc9 Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Fri, 2 Feb 2024 18:19:53 -0500 Subject: [PATCH 23/39] Change Postgres config location Previously, the postgres configuration file was added to the /var/lib/postgresql/data directory which was overwritten by the already-existing named data volume. When initializing a database, the placement of the configuration file in /var/lib/postgresql/data caused an error that the "PGDATA was not empty." This commit fixes this issue by placing the configuration file in a different location (/etc/postgresql) and then running the Dockerfile entrypoint with an argument showing the location of the config file. --- docker-compose.yml | 1 + postgres/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index b27ab6e5e..fadfc5186 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,6 +39,7 @@ services: volumes: - postgres_data:/var/lib/postgresql/data/ restart: always + command: -c 'config_file=/etc/postgresql/postgresql.conf' volumes: postgres_data: diff --git a/postgres/Dockerfile b/postgres/Dockerfile index fe8d210e1..1c915233d 100644 --- a/postgres/Dockerfile +++ b/postgres/Dockerfile @@ -1,4 +1,4 @@ FROM postgres:14-alpine -COPY postgresql.conf /var/lib/postgresql/data/postgresql.conf +COPY postgresql.conf /etc/postgresql/postgresql.conf COPY postgres_backup.sh /usr/local/bin/postgres_backup.sh RUN chmod +x /usr/local/bin/postgres_backup.sh From 830ffe60a5a2b7425d210ac44b5859223c897054 Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Fri, 2 Feb 2024 18:38:11 -0500 Subject: [PATCH 24/39] Add postgres to postgres container command for clarity --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index fadfc5186..3234ab576 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,7 +39,7 @@ services: volumes: - postgres_data:/var/lib/postgresql/data/ restart: always - command: -c 'config_file=/etc/postgresql/postgresql.conf' + command: postgres -c 'config_file=/etc/postgresql/postgresql.conf' volumes: postgres_data: From 94cf8ce1b2e829cf399229fb849bcae24776aea3 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Tue, 6 Feb 2024 11:17:01 -0500 Subject: [PATCH 25/39] Add production and staging versions of docker-compose.yml --- docker-compose.yml.production | 45 +++++++++++++++++++++++++++++++++ docker-compose.yml.staging | 47 +++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 docker-compose.yml.production create mode 100644 docker-compose.yml.staging diff --git a/docker-compose.yml.production b/docker-compose.yml.production new file mode 100644 index 000000000..958f61660 --- /dev/null +++ b/docker-compose.yml.production @@ -0,0 +1,45 @@ +version: '3' + +services: + django: + build: + context: ./django + volumes: + - ./:/code/ + - static_volume:/resources/static + - media_volume:/resources/media + - api_cache_volume:/resources/api_cache + env_file: ./config/envs/dev_env + restart: always + depends_on: + - postgres + + nginx: + build: + context: ./nginx + ports: + - 80:80 + - 443:443 + volumes: + - ./config/nginx/conf.d:/etc/nginx/conf.d + - static_volume:/resources/static + - media_volume:/resources/media + - api_cache_volume:/resources/api_cache + - ./certificates:/etc/nginx/ssl/live + restart: always + depends_on: + - django + + postgres: + build: + context: ./postgres + env_file: ./config/envs/dev_env + volumes: + - postgres_data:/var/lib/postgresql/data/ + restart: always + +volumes: + postgres_data: + static_volume: + media_volume: + api_cache_volume: diff --git a/docker-compose.yml.staging b/docker-compose.yml.staging new file mode 100644 index 000000000..b27ab6e5e --- /dev/null +++ b/docker-compose.yml.staging @@ -0,0 +1,47 @@ +version: '3' + +services: + django: + build: + context: ./django + volumes: + - ./:/code/ + - static_volume:/resources/static + - media_volume:/resources/media + - api_cache_volume:/resources/api_cache + env_file: ./config/envs/dev_env + ports: + - 3000:3000 + restart: always + depends_on: + - postgres + + nginx: + build: + context: ./nginx + ports: + - 80:80 + - 443:443 + volumes: + - ./config/nginx/conf.d:/etc/nginx/conf.d + - static_volume:/resources/static + - media_volume:/resources/media + - api_cache_volume:/resources/api_cache + - ./certificates:/etc/nginx/ssl/live + restart: always + depends_on: + - django + + postgres: + build: + context: ./postgres + env_file: ./config/envs/dev_env + volumes: + - postgres_data:/var/lib/postgresql/data/ + restart: always + +volumes: + postgres_data: + static_volume: + media_volume: + api_cache_volume: From e6c19061f0a1bfe1b7833383f9551e3593be7ba8 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Tue, 6 Feb 2024 11:18:01 -0500 Subject: [PATCH 26/39] .gitignore: add docker-compose.yml and also remove unnecessary line about fixtures --- .gitignore | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 4087c52d7..a28e57c6a 100644 --- a/.gitignore +++ b/.gitignore @@ -104,13 +104,11 @@ django/media/ .env venv -# Fixtures -django/cantusdb_project/main_app/fixtures - -# +# Configuration files config/envs/dev_env .devcontainer/ certificates/ +docker-compose.yml # Drupal scripts from old Cantus django/cantusdb_project/Drupal_scripts @@ -123,4 +121,3 @@ django/cantusdb_project/Drupal_scripts # cached files for APIs api_cache/ - From abf992dcbaca6b1aa0bd8675f2b785a66a27e566 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Tue, 6 Feb 2024 11:20:07 -0500 Subject: [PATCH 27/39] docker-compose.yml.staging: Remove 3000:3000 port mapping --- docker-compose.yml.staging | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yml.staging b/docker-compose.yml.staging index b27ab6e5e..958f61660 100644 --- a/docker-compose.yml.staging +++ b/docker-compose.yml.staging @@ -10,8 +10,6 @@ services: - media_volume:/resources/media - api_cache_volume:/resources/api_cache env_file: ./config/envs/dev_env - ports: - - 3000:3000 restart: always depends_on: - postgres From 7ba3ab9af5b07844599707622b85461c63958b48 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Tue, 6 Feb 2024 11:42:43 -0500 Subject: [PATCH 28/39] Create cantusdb.conf.production and add cantusdb.conf to .gitignore --- .gitignore | 1 + config/nginx/conf.d/cantusdb.conf.production | 76 ++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 config/nginx/conf.d/cantusdb.conf.production diff --git a/.gitignore b/.gitignore index a28e57c6a..15c6b4b02 100644 --- a/.gitignore +++ b/.gitignore @@ -109,6 +109,7 @@ config/envs/dev_env .devcontainer/ certificates/ docker-compose.yml +config/nginx/conf.d/cantusdb.conf # Drupal scripts from old Cantus django/cantusdb_project/Drupal_scripts diff --git a/config/nginx/conf.d/cantusdb.conf.production b/config/nginx/conf.d/cantusdb.conf.production new file mode 100644 index 000000000..43208899b --- /dev/null +++ b/config/nginx/conf.d/cantusdb.conf.production @@ -0,0 +1,76 @@ +server { + listen 80; + + server_tokens off; + + location ^~ /.well-known/acme-challenge/ { + root /var/www/lego; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { + # Redirect all https traffic for mass.cantusdatabase.org to cantusdatabase.org + listen 443 ssl; + + server_name mass.cantusdatabase.org; + + ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt; + ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key; + + location / { + return 301 https://cantusdatabase.org$request_uri; + } +} + +server { + + listen 443 default_server http2 ssl; + + ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt; + ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key; + + location / { + proxy_pass http://django:8000; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_redirect off; + } + + location /static { + alias /resources/static/; + } + location /media { + alias /resources/media/; + } + + location /concordances { + alias /resources/api_cache/concordances.json; + expires modified +24h; + } + + location = /style.css { + root /; + } + location = /background.jpg { + root /; + } + location = /CantusLogoSmall.gif { + root /; + } + location = /favicon.ico { + root /; + } + + error_page 502 /502.html; + location = /502.html { + root /; + } + error_page 504 /504.html; + location = /504.html { + root /; + } +} From 1b8bc424bee4f12db1727d9d6b4894fb84125b18 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Tue, 6 Feb 2024 11:47:12 -0500 Subject: [PATCH 29/39] run git rm on cantusdb.conf, docker-compose.yml --- config/nginx/conf.d/cantusdb.conf | 76 ------------------------------- docker-compose.yml | 50 -------------------- 2 files changed, 126 deletions(-) delete mode 100644 config/nginx/conf.d/cantusdb.conf delete mode 100644 docker-compose.yml diff --git a/config/nginx/conf.d/cantusdb.conf b/config/nginx/conf.d/cantusdb.conf deleted file mode 100644 index 43208899b..000000000 --- a/config/nginx/conf.d/cantusdb.conf +++ /dev/null @@ -1,76 +0,0 @@ -server { - listen 80; - - server_tokens off; - - location ^~ /.well-known/acme-challenge/ { - root /var/www/lego; - } - - location / { - return 301 https://$host$request_uri; - } -} - -server { - # Redirect all https traffic for mass.cantusdatabase.org to cantusdatabase.org - listen 443 ssl; - - server_name mass.cantusdatabase.org; - - ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt; - ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key; - - location / { - return 301 https://cantusdatabase.org$request_uri; - } -} - -server { - - listen 443 default_server http2 ssl; - - ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt; - ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key; - - location / { - proxy_pass http://django:8000; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_redirect off; - } - - location /static { - alias /resources/static/; - } - location /media { - alias /resources/media/; - } - - location /concordances { - alias /resources/api_cache/concordances.json; - expires modified +24h; - } - - location = /style.css { - root /; - } - location = /background.jpg { - root /; - } - location = /CantusLogoSmall.gif { - root /; - } - location = /favicon.ico { - root /; - } - - error_page 502 /502.html; - location = /502.html { - root /; - } - error_page 504 /504.html; - location = /504.html { - root /; - } -} diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 157daf254..000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,50 +0,0 @@ -# This file is configured for deployment of the CantusDB project in local development. -# When building the project locally, replace the contents of docker-compose.yml with the -# contents of this file. - -version: '3' - -services: - django: - build: - context: ./django - volumes: - - ./:/code/ - - static_volume:/resources/static - - media_volume:/resources/media - - api_cache_volume:/resources/api_cache - env_file: ./config/envs/dev_env - restart: always - depends_on: - - postgres - - nginx: - build: - context: ./nginx - ports: - - 80:80 - - 443:443 - volumes: - - ./config/nginx/conf.d:/etc/nginx/conf.d - - static_volume:/resources/static - - media_volume:/resources/media - - api_cache_volume:/resources/api_cache - - ./certificates:/etc/nginx/ssl/live - env_file: ./config/envs/dev_env - restart: always - depends_on: - - django - - postgres: - build: - context: ./postgres - env_file: ./config/envs/dev_env - volumes: - - postgres_data:/var/lib/postgresql/data/ - restart: always - -volumes: - postgres_data: - static_volume: - media_volume: - api_cache_volume: From 9a131dd8bc9b29bbcb68084bf6af1f9385dcd5bb Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Tue, 6 Feb 2024 11:54:06 -0500 Subject: [PATCH 30/39] Add comment to docker-compose.yml.development explaining 3000:3000 port mapping --- docker-compose.yml.development | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml.development b/docker-compose.yml.development index 702c77992..c4d1fac65 100644 --- a/docker-compose.yml.development +++ b/docker-compose.yml.development @@ -15,7 +15,7 @@ services: - api_cache_volume:/resources/api_cache env_file: ./config/envs/dev_env ports: - - 3000:3000 + - 3000:3000 # this mapping allows the VSCode Dev Containers extension to work restart: always depends_on: - postgres From dee9b1b86cf99510d93e9af3fbf33ca928cb2201 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Tue, 6 Feb 2024 11:58:20 -0500 Subject: [PATCH 31/39] docker-compose.yml.development: Re-add 443:443 port mapping --- docker-compose.yml.development | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml.development b/docker-compose.yml.development index c4d1fac65..93ff5a60b 100644 --- a/docker-compose.yml.development +++ b/docker-compose.yml.development @@ -25,7 +25,7 @@ services: context: ./nginx ports: - 80:80 - # - 443:443 + - 443:443 volumes: - ./config/nginx/conf.d:/etc/nginx/conf.d - static_volume:/resources/static From d5321bb22c7c57b3e692fa2b37089fb592e48501 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Tue, 6 Feb 2024 14:30:51 -0500 Subject: [PATCH 32/39] rename cron-staging.txt -> cron.txt.staging --- cron/{cron-staging.txt => cron.txt.staging} | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) rename cron/{cron-staging.txt => cron.txt.staging} (63%) diff --git a/cron/cron-staging.txt b/cron/cron.txt.staging similarity index 63% rename from cron/cron-staging.txt rename to cron/cron.txt.staging index de08ce2c0..721996d45 100644 --- a/cron/cron-staging.txt +++ b/cron/cron.txt.staging @@ -8,5 +8,4 @@ 0 4 * * * bash /home/ubuntu/code/CantusDB/cron/postgres/db_backup.sh 10 4 * * * bash /home/ubuntu/code/CantusDB/cron/management/manage.sh update_cached_concordances 40 4 1 * * bash /home/ubuntu/code/CantusDB/cron/management/manage.sh populate_next_chant_fields; bash /home/ubuntu/code/CantusDB/cron/management/manage.sh populate_is_last_chant_in_feast -# 50 4 * * 7 /usr/local/bin/docker-compose -f /home/ubuntu/code/CantusDB/docker-compose.yml exec -T nginx lego --path /etc/nginx/ssl/live -d cantusdatabase.org -d www.cantusdatabase.org -d mass.cantusdatabase.org -m updateme@example.com --http --http.webroot /var/www/lego/ renew --days 45 --renew-hook "nginx -s reload" -50 4 * * 7 /usr/local/bin/docker-compose -f /home/ubuntu/code/CantusDB/docker-compose.yml exec -T nginx lego --path /etc/nginx/ssl/live -d staging.cantusdatabase.org -d staging-alias.cantusdatabase.org -m updateme@example.com --http --http.webroot /var/www/lego/ renew --days 45 --renew-hook "nginx -s reload" +50 4 * * 7 /usr/local/bin/docker-compose -f /home/ubuntu/code/CantusDB/docker-compose.yml exec -T nginx lego --path /etc/nginx/ssl/live -d staging.cantusdatabase.org -d staging-alias.cantusdatabase.org -m UPDATE.ME@example.com --http --http.webroot /var/www/lego/ renew --days 45 --renew-hook "nginx -s reload" From 9197b838ec11fb2c4bab5cf61b00498cc42fdaa7 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Tue, 6 Feb 2024 14:31:35 -0500 Subject: [PATCH 33/39] rename cron.txt -> cron.txt.production --- cron/{cron.txt => cron.txt.production} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename cron/{cron.txt => cron.txt.production} (70%) diff --git a/cron/cron.txt b/cron/cron.txt.production similarity index 70% rename from cron/cron.txt rename to cron/cron.txt.production index 34daef61f..0d10638e5 100644 --- a/cron/cron.txt +++ b/cron/cron.txt.production @@ -2,10 +2,10 @@ # CantusDB Cron Jobs # ###################### -# Note: This is set up to run on the production server. If you want to run it on your local machine, you will need to change the paths. +# Note: This is set up to run on the production and staging servers. If you want to run it on your local machine, you will need to change the paths. # min hour day month weekday command 0 4 * * * bash /home/ubuntu/code/CantusDB/cron/postgres/db_backup.sh 10 4 * * * bash /home/ubuntu/code/CantusDB/cron/management/manage.sh update_cached_concordances 40 4 1 * * bash /home/ubuntu/code/CantusDB/cron/management/manage.sh populate_next_chant_fields; bash /home/ubuntu/code/CantusDB/cron/management/manage.sh populate_is_last_chant_in_feast -50 4 * * 7 /usr/local/bin/docker-compose -f /home/ubuntu/code/CantusDB/docker-compose.yml exec -T nginx lego --path /etc/nginx/ssl/live -d cantusdatabase.org -d www.cantusdatabase.org -d mass.cantusdatabase.org -m updateme@example.com --http --http.webroot /var/www/lego/ renew --days 45 --renew-hook "nginx -s reload" +50 4 * * 7 /usr/local/bin/docker-compose -f /home/ubuntu/code/CantusDB/docker-compose.yml exec -T nginx lego --path /etc/nginx/ssl/live -d cantusdatabase.org -d www.cantusdatabase.org -d mass.cantusdatabase.org -m UPDATE.ME@example.com --http --http.webroot /var/www/lego/ renew --days 45 --renew-hook "nginx -s reload" From 36f9b75783bc2e8867356bf0fc723bcc0722bd24 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Tue, 6 Feb 2024 14:32:53 -0500 Subject: [PATCH 34/39] add cron.txt to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 15c6b4b02..4197fbb89 100644 --- a/.gitignore +++ b/.gitignore @@ -110,6 +110,7 @@ config/envs/dev_env certificates/ docker-compose.yml config/nginx/conf.d/cantusdb.conf +cron/cron.txt # Drupal scripts from old Cantus django/cantusdb_project/Drupal_scripts From aa920f3819219446e7a8db78ae3a99ae0d7b53dc Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Tue, 6 Feb 2024 18:24:24 -0500 Subject: [PATCH 35/39] Add postgres command with path to config file to docker-compose staging --- docker-compose.yml.staging | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml.staging b/docker-compose.yml.staging index 958f61660..1bbaa5810 100644 --- a/docker-compose.yml.staging +++ b/docker-compose.yml.staging @@ -37,6 +37,7 @@ services: volumes: - postgres_data:/var/lib/postgresql/data/ restart: always + command: postgres -c 'config_file=/etc/postgresql/postgresql.conf' volumes: postgres_data: From bad5c146ca885ab0dad0c2857775b5b95bbf1645 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Feb 2024 14:09:27 +0000 Subject: [PATCH 36/39] Bump django from 4.2.3 to 4.2.10 in /django/cantusdb_project Bumps [django](https://github.com/django/django) from 4.2.3 to 4.2.10. - [Commits](https://github.com/django/django/compare/4.2.3...4.2.10) --- updated-dependencies: - dependency-name: django dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- django/cantusdb_project/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/cantusdb_project/requirements.txt b/django/cantusdb_project/requirements.txt index b052dc46d..1b09ab55b 100644 --- a/django/cantusdb_project/requirements.txt +++ b/django/cantusdb_project/requirements.txt @@ -9,7 +9,7 @@ chardet==3.0.4 charset-normalizer==2.0.12 click==7.1.2 coverage==5.3.1 -Django==4.2.3 +Django==4.2.10 django-autocomplete-light==3.9.4 django-extra-views==0.13.0 django-quill-editor==0.1.40 From c2891b3714745eacdbcec3f1ccc983f26fb0e749 Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Thu, 8 Feb 2024 09:31:17 -0500 Subject: [PATCH 37/39] Update docker-compose.yml.development with postgres command including config file path --- docker-compose.yml.development | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml.development b/docker-compose.yml.development index 93ff5a60b..afdd50785 100644 --- a/docker-compose.yml.development +++ b/docker-compose.yml.development @@ -43,6 +43,7 @@ services: volumes: - postgres_data:/var/lib/postgresql/data/ restart: always + command: postgres -c 'config_file=/etc/postgresql/postgresql.conf' volumes: postgres_data: From 86616b0d0e861a69766088170b3c170e838c4078 Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Thu, 8 Feb 2024 18:43:46 +0000 Subject: [PATCH 38/39] Ensure CSRF_TRUSTED_ORIGINS and ALLOWED_HOSTS are complete and up-to-date --- django/cantusdb_project/cantusdb/settings.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/django/cantusdb_project/cantusdb/settings.py b/django/cantusdb_project/cantusdb/settings.py index 83021a2c0..8487aefd3 100644 --- a/django/cantusdb_project/cantusdb/settings.py +++ b/django/cantusdb_project/cantusdb/settings.py @@ -200,7 +200,21 @@ "127.0.0.1", ] -CSRF_TRUSTED_ORIGINS = ["https://cantusdatabase.org", "https://www.cantusdatabase.org"] +CSRF_TRUSTED_ORIGINS = [ + "https://cantusdatabase.org", + "https://www.cantusdatabase.org", + "https://staging.cantusdatabase.org", + # "https://mass.cantusdatabase.org", + # "https://staging-alias.cantusdatabase.org", + # traffic to the two subdomains above should be redirected by nginx, and + # should never reach django +] + +ALLOWED_HOSTS = [ + ".cantusdatabase.org", # leading '.': include all subdomains + ".localhost", + "127.0.0.1", +] if DEBUG: INSTALLED_APPS.append("debug_toolbar") From deb89106c8982499e2930544d2276f4d2c22525e Mon Sep 17 00:00:00 2001 From: Jacob deGroot-Maggetti <58090591+jacobdgm@users.noreply.github.com> Date: Thu, 8 Feb 2024 20:16:06 +0000 Subject: [PATCH 39/39] Reconfigure settings.py to read allowed hosts and trusted origins from environment variables --- django/cantusdb_project/cantusdb/settings.py | 32 ++++++++------------ 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/django/cantusdb_project/cantusdb/settings.py b/django/cantusdb_project/cantusdb/settings.py index 8487aefd3..5510b2324 100644 --- a/django/cantusdb_project/cantusdb/settings.py +++ b/django/cantusdb_project/cantusdb/settings.py @@ -35,11 +35,21 @@ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = os.getenv("CANTUSDB_SECRET_KEY") +PROJECT_ENVIRONMENT = os.getenv("PROJECT_ENVIRONMENT") + # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = bool(strtobool(os.getenv("CANTUSDB_DEBUG", "False"))) -# need to set this to false so that we can display the custom 404 page +DEBUG = False # this is switched to True below when PROJECT_ENVIRONMENT=="DEVELOPMENT" -ALLOWED_HOSTS = [os.getenv("CANTUSDB_HOSTS")] +if PROJECT_ENVIRONMENT == "DEVELOPMENT": + ALLOWED_HOSTS = os.getenv("CANTUSDB_HOSTS_DEVELOPMENT").split(" ") + CSRF_TRUSTED_ORIGINS = os.getenv("CANTUSDB_ORIGINS_DEVELOPMENT").split(" ") + DEBUG = True +if PROJECT_ENVIRONMENT == "STAGING": + ALLOWED_HOSTS = os.getenv("CANTUSDB_HOSTS_STAGING").split(" ") + CSRF_TRUSTED_ORIGINS = os.getenv("CANTUSDB_ORIGINS_STAGING").split(" ") +if PROJECT_ENVIRONMENT == "PRODUCTION": + ALLOWED_HOSTS = os.getenv("CANTUSDB_HOSTS_PRODUCTION").split(" ") + CSRF_TRUSTED_ORIGINS = os.getenv("CANTUSDB_ORIGINS_PRODUCTION").split(" ") # Application definition @@ -200,22 +210,6 @@ "127.0.0.1", ] -CSRF_TRUSTED_ORIGINS = [ - "https://cantusdatabase.org", - "https://www.cantusdatabase.org", - "https://staging.cantusdatabase.org", - # "https://mass.cantusdatabase.org", - # "https://staging-alias.cantusdatabase.org", - # traffic to the two subdomains above should be redirected by nginx, and - # should never reach django -] - -ALLOWED_HOSTS = [ - ".cantusdatabase.org", # leading '.': include all subdomains - ".localhost", - "127.0.0.1", -] - if DEBUG: INSTALLED_APPS.append("debug_toolbar") # debug toolbar must be inserted as early in the middleware as possible