Merge branch 'develop' into issue-1202

.gitignore

@@ -104,13 +104,13 @@ django/media/
 .env
 venv
 
-# Fixtures
-django/cantusdb_project/main_app/fixtures
-
-#
+# Configuration files
 config/envs/dev_env
 .devcontainer/
 certificates/
+docker-compose.yml
+config/nginx/conf.d/cantusdb.conf
+cron/cron.txt
 
 # Drupal scripts from old Cantus
 django/cantusdb_project/Drupal_scripts
@@ -123,4 +123,3 @@ django/cantusdb_project/Drupal_scripts
 
 # cached files for APIs
 api_cache/
-

config/nginx/conf.d/cantusdb.conf.development

@@ -0,0 +1,80 @@
+# This file is configured for deployment of the CantusDB project in local development.
+# When building the project locally, replace the contents of cantusdb.conf with the
+# contents of this file.
+
+# server {
+#     listen 80;
+# 
+#     server_tokens off;
+# 
+#     location ^~ /.well-known/acme-challenge/ {
+#         root /var/www/lego;
+#     }
+# 
+#     location / {
+#         return 301 https://$host$request_uri;
+#     }
+# }
+
+# server {
+#     # Redirect all https traffic for mass.cantusdatabase.org to cantusdatabase.org
+#     listen 443 ssl;
+# 
+#     server_name mass.cantusdatabase.org;
+# 
+#     ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt;
+#     ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key;
+# 
+#     location / {
+#         return 301 https://cantusdatabase.org$request_uri;
+#     }
+# }
+
+server {
+    # listen 443 default_server http2 ssl;
+    listen 80;
+
+    # ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt;
+    # ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key;
+
+    location / {
+        proxy_pass http://django:8000;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+    }
+
+    location /static {
+        alias /resources/static/;
+    }
+    location /media {
+        alias /resources/media/;
+    }
+
+    location /concordances {
+        alias /resources/api_cache/concordances.json;
+        expires modified +24h;
+    }
+
+    location = /style.css {
+        root /;
+    }
+    location = /background.jpg {
+        root /;
+    }
+    location = /CantusLogoSmall.gif {
+        root /;
+    }
+    location = /favicon.ico {
+        root /;
+    }
+
+    error_page 502 /502.html;
+    location = /502.html {
+        root /;
+    }
+    error_page 504 /504.html;
+    location = /504.html {
+        root /;
+    }
+}

config/nginx/conf.d/cantusdb.conf → config/nginx/conf.d/cantusdb.conf.production

@@ -46,7 +46,12 @@ server {
     location /media {
         alias /resources/media/;
     }
-
+
+    location /concordances {
+        alias /resources/api_cache/concordances.json;
+        expires modified +24h;
+    }
+
     location = /style.css {
         root /;
     }

config/nginx/conf.d/cantusdb.conf.staging

@@ -0,0 +1,83 @@
+server {
+    listen 80;
+
+    server_tokens off;
+
+    location ^~ /.well-known/acme-challenge/ {
+        root /var/www/lego;
+    }
+
+    location / {
+        return 301 https://$host$request_uri;
+    }
+}
+
+server {
+    # Redirect all https traffic for staging-alias.cantusdatabase.org to staging.cantusdatabase.org
+    # (on Production, this redirects from mass.cantusdatabase.org to cantusdatabase.org.
+    # The Staging server's subdomain "staging-alias" is analogous to Production's "mass" subdomain.)
+    listen 443 ssl;
+
+    # server_name mass.cantusdatabase.org;
+    server_name staging-alias.cantusdatabase.org;
+
+    # ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt;
+    # ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key;
+    ssl_certificate /etc/nginx/ssl/live/certificates/staging.cantusdatabase.org.crt;
+    ssl_certificate_key /etc/nginx/ssl/live/certificates/staging.cantusdatabase.org.key;
+
+    location / {
+        return 301 https://staging.cantusdatabase.org$request_uri;
+    }
+}
+
+server {
+
+    listen 443 default_server http2 ssl;
+
+    # ssl_certificate /etc/nginx/ssl/live/certificates/cantusdatabase.org.crt;
+    # ssl_certificate_key /etc/nginx/ssl/live/certificates/cantusdatabase.org.key;
+    ssl_certificate /etc/nginx/ssl/live/certificates/staging.cantusdatabase.org.crt;
+    ssl_certificate_key /etc/nginx/ssl/live/certificates/staging.cantusdatabase.org.key;
+
+    location / {
+        proxy_pass http://django:8000;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+    }
+
+    location /static {
+        alias /resources/static/;
+    }
+    location /media {
+        alias /resources/media/;
+    }
+
+    location /concordances {
+        alias /resources/api_cache/concordances.json;
+        expires modified +24h;
+    }
+
+    location = /style.css {
+        root /;
+    }
+    location = /background.jpg {
+        root /;
+    }
+    location = /CantusLogoSmall.gif {
+        root /;
+    }
+    location = /favicon.ico {
+        root /;
+    }
+
+    error_page 502 /502.html;
+    location = /502.html {
+        root /;
+    }
+    error_page 504 /504.html;
+    location = /504.html {
+        root /;
+    }
+}

cron/cron.txt.production

@@ -0,0 +1,11 @@
+######################
+# CantusDB Cron Jobs #
+######################
+
+# Note: This is set up to run on the production and staging servers. If you want to run it on your local machine, you will need to change the paths.
+
+# min   hour    day     month   weekday command
+0       4       *       *       *       bash /home/ubuntu/code/CantusDB/cron/postgres/db_backup.sh
+10      4       *       *       *       bash /home/ubuntu/code/CantusDB/cron/management/manage.sh update_cached_concordances
+40      4       1       *       *       bash /home/ubuntu/code/CantusDB/cron/management/manage.sh populate_next_chant_fields; bash /home/ubuntu/code/CantusDB/cron/management/manage.sh populate_is_last_chant_in_feast
+50      4       *       *       7       /usr/local/bin/docker-compose -f /home/ubuntu/code/CantusDB/docker-compose.yml exec -T nginx lego --path /etc/nginx/ssl/live -d cantusdatabase.org -d www.cantusdatabase.org -d mass.cantusdatabase.org -m [email protected] --http --http.webroot /var/www/lego/ renew --days 45 --renew-hook "nginx -s reload" 

cron/cron.txt → cron/cron.txt.staging

@@ -6,5 +6,6 @@
 
 # min   hour    day     month   weekday command
 0       4       *       *       *       bash /home/ubuntu/code/CantusDB/cron/postgres/db_backup.sh
+10      4       *       *       *       bash /home/ubuntu/code/CantusDB/cron/management/manage.sh update_cached_concordances
 40      4       1       *       *       bash /home/ubuntu/code/CantusDB/cron/management/manage.sh populate_next_chant_fields; bash /home/ubuntu/code/CantusDB/cron/management/manage.sh populate_is_last_chant_in_feast
-50      4       *       *       7       /usr/local/bin/docker-compose -f /home/ubuntu/code/CantusDB/docker-compose.yml exec -T nginx lego --path /etc/nginx/ssl/live -d cantusdatabase.org -d www.cantusdatabase.org -d mass.cantusdatabase.org -m updateme@example.com --http --http.webroot /var/www/lego/ renew --days 45 --renew-hook "nginx -s reload" 
+50      4       *       *       7       /usr/local/bin/docker-compose -f /home/ubuntu/code/CantusDB/docker-compose.yml exec -T nginx lego --path /etc/nginx/ssl/live -d staging.cantusdatabase.org -d staging-alias.cantusdatabase.org -m UPDATE.ME@example.com --http --http.webroot /var/www/lego/ renew --days 45 --renew-hook "nginx -s reload" 

cron/management/manage.sh

@@ -7,4 +7,4 @@
 DOCKER_COMPOSE_FILE=/home/ubuntu/code/CantusDB/docker-compose.yml   # This is the path to the docker-compose file.
 COMMAND=$1                                                          # This is the command to execute.
 
-/usr/local/bin/docker-compose -f $DOCKER_COMPOSE_FILE exec django python manage.py $COMMAND
+/usr/local/bin/docker-compose -f $DOCKER_COMPOSE_FILE exec -T django python manage.py $COMMAND

django/cantusdb_project/cantusdb/settings.py

@@ -35,11 +35,21 @@
 # SECURITY WARNING: keep the secret key used in production secret!
 SECRET_KEY = os.getenv("CANTUSDB_SECRET_KEY")
 
+PROJECT_ENVIRONMENT = os.getenv("PROJECT_ENVIRONMENT")
+
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = bool(strtobool(os.getenv("CANTUSDB_DEBUG", "False")))
-# need to set this to false so that we can display the custom 404 page
+DEBUG = False  # this is switched to True below when PROJECT_ENVIRONMENT=="DEVELOPMENT"
 
-ALLOWED_HOSTS = [os.getenv("CANTUSDB_HOSTS")]
+if PROJECT_ENVIRONMENT == "DEVELOPMENT":
+    ALLOWED_HOSTS = os.getenv("CANTUSDB_HOSTS_DEVELOPMENT").split(" ")
+    CSRF_TRUSTED_ORIGINS = os.getenv("CANTUSDB_ORIGINS_DEVELOPMENT").split(" ")
+    DEBUG = True
+if PROJECT_ENVIRONMENT == "STAGING":
+    ALLOWED_HOSTS = os.getenv("CANTUSDB_HOSTS_STAGING").split(" ")
+    CSRF_TRUSTED_ORIGINS = os.getenv("CANTUSDB_ORIGINS_STAGING").split(" ")
+if PROJECT_ENVIRONMENT == "PRODUCTION":
+    ALLOWED_HOSTS = os.getenv("CANTUSDB_HOSTS_PRODUCTION").split(" ")
+    CSRF_TRUSTED_ORIGINS = os.getenv("CANTUSDB_ORIGINS_PRODUCTION").split(" ")
 
 
 # Application definition
@@ -200,8 +210,6 @@
     "127.0.0.1",
 ]
 
-CSRF_TRUSTED_ORIGINS = ["https://cantusdatabase.org", "https://www.cantusdatabase.org"]
-
 if DEBUG:
     INSTALLED_APPS.append("debug_toolbar")
     # debug toolbar must be inserted as early in the middleware as possible

django/cantusdb_project/main_app/management/commands/update_cached_concordances.py

@@ -1,17 +1,33 @@
 import ujson
 import os
 from sys import stdout
+from typing import Optional
 from datetime import datetime
 from collections import defaultdict
 from django.db.models.query import QuerySet
 from django.core.management.base import BaseCommand
 from main_app.models import Chant
 
 
+# Usage: `python manage.py update_cached_concordances`
+# or `python manage.py update_cached_concordances -d "/path/to/directory/in/which/to/save/concordances"`
+
+
 class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "-d",
+            "--directory",
+            help="Optional filepath specifying a directory to output concordances",
+            type=str,
+            # this default directory should match the value in docker-compose.yml,
+            # at services:django:volumes:api_cache_volume
+            default="/resources/api_cache",
+        )
+
     def handle(self, *args, **kwargs) -> None:
-        CACHE_DIR: str = "api_cache"
-        FILEPATH: str = f"{CACHE_DIR}/concordances.json"
+        cache_dir: str = kwargs["directory"]
+        filepath: str = f"{cache_dir}/concordances.json"
         start_time: str = datetime.now().isoformat()
         stdout.write(f"Running update_cached_concordances at {start_time}.\n")
         concordances: dict = get_concordances()
@@ -23,22 +39,29 @@ def handle(self, *args, **kwargs) -> None:
             "data": concordances,
             "metadata": metadata,
         }
-        stdout.write(f"Attempting to make directory at {CACHE_DIR} to hold cache: ")
+        stdout.write(f"Attempting to make directory at {cache_dir} to hold cache: ")
         try:
-            os.mkdir(CACHE_DIR)
-            stdout.write(f"successfully created directory at {CACHE_DIR}.\n")
+            os.mkdir(cache_dir)
+            stdout.write(f"successfully created directory at {cache_dir}.\n")
         except FileExistsError:
-            stdout.write(f"directory at {CACHE_DIR} already exists.\n")
-        stdout.write(f"Writing concordances to {FILEPATH} at {write_time}.\n")
-        with open(FILEPATH, "w") as json_file:
+            stdout.write(f"directory at {cache_dir} already exists.\n")
+        stdout.write(f"Writing concordances to {filepath} at {write_time}.\n")
+        with open(filepath, "w") as json_file:
             ujson.dump(data_and_metadata, json_file)
         end_time = datetime.now().isoformat()
         stdout.write(
-            f"Concordances successfully written to {FILEPATH} at {end_time}.\n\n"
+            f"Concordances successfully written to {filepath} at {end_time}.\n\n"
         )
 
 
 def get_concordances() -> dict:
+    """Fetch all published chants in the database, group them by Cantus ID, and return
+    a dictionary containing information on each of these chants.
+
+    Returns:
+        dict: A dictionary where each key is a Cantus ID and each value is a list all
+          published chants in the database with that Cantus ID.
+    """
     DOMAIN: str = "https://cantusdatabase.org"
 
     stdout.write("Querying database for published chants\n")

django/cantusdb_project/requirements.txt

@@ -9,7 +9,7 @@ chardet==3.0.4
 charset-normalizer==2.0.12
 click==7.1.2
 coverage==5.3.1
-Django==4.2.3
+Django==4.2.10
 django-autocomplete-light==3.9.4
 django-extra-views==0.13.0
 django-quill-editor==0.1.40

docker-compose.yml.development

@@ -0,0 +1,52 @@
+# This file is configured for deployment of the CantusDB project in local development.
+# When building the project locally, replace the contents of docker-compose.yml with the
+# contents of this file.
+
+version: '3'
+
+services:
+  django:
+    build:
+      context: ./django
+    volumes:
+      - ./:/code/
+      - static_volume:/resources/static
+      - media_volume:/resources/media
+      - api_cache_volume:/resources/api_cache
+    env_file: ./config/envs/dev_env
+    ports:
+      - 3000:3000 # this mapping allows the VSCode Dev Containers extension to work
+    restart: always
+    depends_on:
+      - postgres
+
+  nginx:
+    build:
+      context: ./nginx
+    ports:
+      - 80:80
+      - 443:443
+    volumes:
+      - ./config/nginx/conf.d:/etc/nginx/conf.d
+      - static_volume:/resources/static
+      - media_volume:/resources/media
+      - api_cache_volume:/resources/api_cache
+      - ./certificates:/etc/nginx/ssl/live
+    restart: always
+    depends_on:
+      - django
+
+  postgres:
+    build:
+      context: ./postgres
+    env_file: ./config/envs/dev_env
+    volumes:
+      - postgres_data:/var/lib/postgresql/data/
+    restart: always
+    command: postgres -c 'config_file=/etc/postgresql/postgresql.conf'
+
+volumes:
+  postgres_data:
+  static_volume:
+  media_volume:
+  api_cache_volume: