Merge develop into staging, 2024-Jan-30 edition

.github/workflows/broken-link-checker.yaml

@@ -0,0 +1,44 @@
+name: Link checker for articles and flatpages on CantusDB
+
+on:
+  schedule:
+    - cron: "8 8 * * 0" # Cron job will run at 08h08 UTC time every Sunday
+
+jobs:
+  get-all-links:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+    - id: set-matrix
+      run: |
+        BASE_URL="https://cantusdatabase.org"
+        flatpages=$(curl "$BASE_URL/flatpages-list/" | awk '{ gsub (" ", "\",\"", $0); print}')
+        articles=$(curl "$BASE_URL/articles-list/" | awk '{ gsub (" ", "\",\"", $0); print}')
+        list="{\"links\": [\"${flatpages}\",\"${articles}\"]}"
+        echo $list
+        echo "matrix=$list" >> $GITHUB_OUTPUT
+  link-Checker:
+    runs-on: ubuntu-latest
+    needs: get-all-links
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:  ${{fromJson(needs.get-all-links.outputs.matrix)}}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Link Checker
+        id: lychee
+        uses: lycheeverse/[email protected]
+        with:
+          args: --exclude http:\/\/cantus\.sk.* ${{ matrix.links }}
+          format: json
+          output: /tmp/link-checker-output.txt
+      - name: Curating Link Checker Output
+        run: |
+          echo "***Python Version***"
+          python --version
+          echo "***Invoking Parsing Script***"
+          python "$GITHUB_WORKSPACE/scripts/parse_link_checker_output.py" >> $GITHUB_STEP_SUMMARY
+          echo "***Printing Summary***"
+          cat $GITHUB_STEP_SUMMARY

.gitignore

@@ -120,3 +120,7 @@ django/cantusdb_project/Drupal_scripts
 
 # in case you're working on importing data from a database dump
 *.sql
+
+# cached files for APIs
+api_cache/
+

cron/cron.txt

@@ -7,4 +7,4 @@
 # min   hour    day     month   weekday command
 0       4       *       *       *       bash /home/ubuntu/code/CantusDB/cron/postgres/db_backup.sh
 40      4       1       *       *       bash /home/ubuntu/code/CantusDB/cron/management/manage.sh populate_next_chant_fields; bash /home/ubuntu/code/CantusDB/cron/management/manage.sh populate_is_last_chant_in_feast
-50      4       *       *       7       /usr/local/bin/docker-compose exec nginx lego --path /etc/nginx/ssl/live -d cantusdatabase.org -d www.cantusdatabase.org -d mass.cantusdatabase.org --http --http.webroot /var/www/lego/ renew --days 45 --renew-hook "nginx -s reload" 
+50      4       *       *       7       /usr/local/bin/docker-compose -f /home/ubuntu/code/CantusDB/docker-compose.yml exec -T nginx lego --path /etc/nginx/ssl/live -d cantusdatabase.org -d www.cantusdatabase.org -d mass.cantusdatabase.org -m [email protected] --http --http.webroot /var/www/lego/ renew --days 45 --renew-hook "nginx -s reload" 

django/cantusdb_project/main_app/management/commands/update_cached_concordances.py

@@ -0,0 +1,100 @@
+import ujson
+import os
+from sys import stdout
+from datetime import datetime
+from collections import defaultdict
+from django.db.models.query import QuerySet
+from django.core.management.base import BaseCommand
+from main_app.models import Chant
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **kwargs) -> None:
+        CACHE_DIR: str = "api_cache"
+        FILEPATH: str = f"{CACHE_DIR}/concordances.json"
+        start_time: str = datetime.now().isoformat()
+        stdout.write(f"Running update_cached_concordances at {start_time}.\n")
+        concordances: dict = get_concordances()
+        write_time: str = datetime.now().isoformat()
+        metadata: dict = {
+            "last_updated": write_time,
+        }
+        data_and_metadata: dict = {
+            "data": concordances,
+            "metadata": metadata,
+        }
+        stdout.write(f"Attempting to make directory at {CACHE_DIR} to hold cache: ")
+        try:
+            os.mkdir(CACHE_DIR)
+            stdout.write(f"successfully created directory at {CACHE_DIR}.\n")
+        except FileExistsError:
+            stdout.write(f"directory at {CACHE_DIR} already exists.\n")
+        stdout.write(f"Writing concordances to {FILEPATH} at {write_time}.\n")
+        with open(FILEPATH, "w") as json_file:
+            ujson.dump(data_and_metadata, json_file)
+        end_time = datetime.now().isoformat()
+        stdout.write(
+            f"Concordances successfully written to {FILEPATH} at {end_time}.\n\n"
+        )
+
+
+def get_concordances() -> dict:
+    DOMAIN: str = "https://cantusdatabase.org"
+
+    stdout.write("Querying database for published chants\n")
+    published_chants: QuerySet[Chant] = Chant.objects.filter(source__published=True)
+    values: QuerySet[dict] = published_chants.select_related(
+        "source",
+        "feast",
+        "genre",
+        "office",
+    ).values(
+        "id",
+        "source_id",
+        "source__siglum",
+        "folio",
+        "c_sequence",
+        "incipit",
+        "feast__name",
+        "genre__name",
+        "office__name",
+        "position",
+        "cantus_id",
+        "image_link",
+        "mode",
+        "manuscript_full_text_std_spelling",
+        "volpiano",
+    )
+
+    stdout.write("Processing chants\n")
+    concordances: defaultdict = defaultdict(list)
+    for chant in values:
+        source_id: int = chant["source_id"]
+        source_absolute_url: str = f"{DOMAIN}/source/{source_id}/"
+        chant_id: int = chant["id"]
+        chant_absolute_url: str = f"{DOMAIN}/chant/{chant_id}/"
+
+        concordances[chant["cantus_id"]].append(
+            {
+                "siglum": chant["source__siglum"],
+                "srclink": source_absolute_url,
+                "chantlink": chant_absolute_url,
+                "folio": chant["folio"],
+                "sequence": chant["c_sequence"],
+                "incipit": chant["incipit"],
+                "feast": chant["feast__name"],
+                "genre": chant["genre__name"],
+                "office": chant["office__name"],
+                "position": chant["position"],
+                "cantus_id": chant["cantus_id"],
+                "image": chant["image_link"],
+                "mode": chant["mode"],
+                "full_text": chant["manuscript_full_text_std_spelling"],
+                "melody": chant["volpiano"],
+                "db": "CD",
+            }
+        )
+
+    stdout.write(f"All chants processed - found {len(concordances)} Cantus IDs\n")
+
+    return dict(concordances)