Ckan 211 (#158)

* chore(deps): update dependencies and settings for CKAN 2.11 compatibility * Explicitly add linux/amd64 platform to docker as there are no other images * Clarify and clean up Dockerfile * Current status of 2.11.1 upgrade * Adjust dockerfile * Add license * Correct readme * fix permission issues * Revert x64 * Sonarcloud recomendations * Applied review comments * fix Civity reference to scheming --------- Co-authored-by: Mark Janse <[email protected]> Co-authored-by: Hans-Chrstian <[email protected]> Co-authored-by: kburger <[email protected]>
GenomicDataInfrastructure · Jan 20, 2025 · 4089575 · 4089575
1 parent 24920cc
commit 4089575
Show file tree

Hide file tree

Showing 18 changed files with 228 additions and 2,687 deletions.
diff --git a/.env b/.env
@@ -3,44 +3,52 @@
 #
 # SPDX-License-Identifier: AGPL-3.0-only
 
-# Host Ports
-CKAN_PORT_HOST=5500
-
-# Postgres
+# CKAN databases
 POSTGRES_USER=postgres
 POSTGRES_PASSWORD=postgres
 POSTGRES_DB=postgres
-POSTGRES_HOST=postgres
+POSTGRES_HOST=db
 CKAN_DB_USER=ckandbuser
 CKAN_DB_PASSWORD=ckandbpassword
 CKAN_DB=ckandb
+CKAN_SQLALCHEMY_URL=postgresql://ckandbuser:ckandbpassword@db/ckandb
+
+# Dev settings
+USE_HTTPS_FOR_DEV=false
+
+# Host Ports
+CKAN_PORT_HOST=5500
 
 # CKAN core
-CKAN_VERSION=2.10.5
+CKAN_VERSION=2.11.1
 CKAN_SITE_ID=default
 CKAN_SITE_URL=http://localhost:5500
-CKAN__CORS__ORIGIN_WHITELIST=http://localhost:3000
+
 CKAN___BEAKER__SESSION__SECRET=CHANGE_ME
 # See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings
 CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME
 CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME
 CKAN_SYSADMIN_NAME=ckan_admin
 CKAN_SYSADMIN_PASSWORD=test1234
 CKAN_SYSADMIN_EMAIL=[email protected]
+CKAN_STORAGE_PATH=/var/lib/ckan
 CKAN_SMTP_SERVER=smtp.corporateict.domain:25
 CKAN_SMTP_STARTTLS=True
 CKAN_SMTP_USER=user
 CKAN_SMTP_PASSWORD=pass
 CKAN_SMTP_MAIL_FROM=ckan@localhost
 CKAN_MAX_UPLOAD_SIZE_MB=10
-CKAN_SQLALCHEMY_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_DB}
-TEST_CKAN_SQLALCHEMY_URL=postgres://{CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/ckan_test
-CKAN_SOLR_URL=http://solr:8983/solr/ckan
-TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan
+TZ=Europe/Amsterdam
+
+# Redis
+REDIS_VERSION=7.4.2
 CKAN_REDIS_URL=redis://redis:6379/1
-TEST_CKAN_REDIS_URL=redis://redis:6379/1
 
-CKAN__HARVEST__MQ__TYPE=redis
+# Solr 
+CKAN_SOLR_URL=http://solr:8983/solr/ckan
+
+# Extensions
+CKAN__PLUGINS="envvars scheming_datasets scheming_organizations gdi_userportal dcat harvest dcat_rdf_harvester fairdatapointharvester activity"
 CKAN__HARVEST__MQ__HOSTNAME=redis
 CKAN__HARVEST__MQ__PORT=6379
 CKAN__HARVEST__MQ__REDIS_DB=1
@@ -52,3 +60,5 @@ CKAN___LOGGER_CKANEXT__LEVEL=DEBUG
 
 #OTEL
 OTEL_SDK_DISABLED=true
+
+
diff --git a/README.md b/README.md
@@ -42,18 +42,30 @@ CKAN and all the components are configured using environment variables that you
 ## 3. Useful commands
 
 ### 3.1. Build and start dev environment
+Use this mode if you are making code changes to CKAN and either creating new extensions or making code changes to existing extensions. This mode also uses the `.env` file for config options.
+
+To develop local extensions use the `docker-compose.yml` file with help from the scripts under `bin`:
+
+To build the images:
 ```bash
-docker compose up -d --build
+	bin/compose build
+```
+To install extensions from the `src` directory:
+```bash
+	bin/install_src
+```
+To start the containers:
+```bash
+	bin/compose up
 ```
-
 ### 3.2. Remove images and volumes
 ```bash
-docker compose down -v
+  bin/compose down -v
 ```
 
 ### 3.3. Logs
 ```bash
-docker compose logs -f
+  bin/compose logs -f
 ```
 
 ### 3.4. Re-index SOLR

diff --git a/bin/ckan b/bin/ckan
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# SPDX-FileCopyrightText: 2006-2024 Open Knowledge Foundation and contributors
+#
+# SPDX-License-Identifier: AGPL-3.0-only
+
+set -e
+ROOT="$(dirname ${BASH_SOURCE[0]})/.."
+
+docker compose -f "${ROOT}/docker-compose.dev.yml" exec ckan-dev ckan "$@"
diff --git a/bin/compose b/bin/compose
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# SPDX-FileCopyrightText: 2006-2024 Open Knowledge Foundation and contributors
+#
+# SPDX-License-Identifier: AGPL-3.0-only
+
+set -e
+ROOT="$(dirname ${BASH_SOURCE[0]})/.."
+
+docker compose -f "${ROOT}/docker-compose.yml" "$@"
diff --git a/bin/install_src b/bin/install_src
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# SPDX-FileCopyrightText: 2006-2024 Open Knowledge Foundation and contributors
+#
+# SPDX-License-Identifier: AGPL-3.0-only
+
+set -e
+ROOT="$(dirname ${BASH_SOURCE[0]})/.."
+
+docker compose -f "${ROOT}/docker-compose.yml" run -u root ckan-dev ./install_src.sh
diff --git a/ckan/Dockerfile b/ckan/Dockerfile
@@ -4,56 +4,57 @@
 #
 # SPDX-License-Identifier: AGPL-3.0-only
 
-FROM docker.io/ckan/ckan-base:2.10.6
+FROM ckan/ckan-base:2.11.1
 
-RUN  pip3 install -e git+https://github.com/GenomicDataInfrastructure/[email protected]#egg=ckanext-gdi-userportal && \
+USER root
+RUN chown -R ckan:ckan-sys ${APP_DIR}
+USER ckan
+
+RUN pip3 install -e git+https://github.com/GenomicDataInfrastructure/[email protected]#egg=ckanext-gdi-userportal && \
     pip3 install -r ${APP_DIR}/src/ckanext-gdi-userportal/requirements.txt
 
-RUN  pip3 install -e git+https://github.com/ckan/[email protected]#egg=ckanext-dcat && \
+RUN pip3 install -e git+https://github.com/ckan/[email protected]#egg=ckanext-dcat && \
     pip3 install -r ${APP_DIR}/src/ckanext-dcat/requirements.txt
 
 RUN pip3 install -e git+https://github.com/ckan/[email protected]#egg=ckanext-harvest && \
     pip3 install -r ${APP_DIR}/src/ckanext-harvest/requirements.txt
 
-RUN pip3 install -e git+https://github.com/CivityNL/[email protected]#egg=ckanext-scheming[requirements]
+RUN pip3 install -e git+https://github.com/CivityNL/[email protected]-1#egg=ckanext-scheming[requirements]
 
-RUN  pip3 install -e git+https://github.com/GenomicDataInfrastructure/[email protected]#egg=ckanext-fairdatapoint && \
+RUN pip3 install -e git+https://github.com/GenomicDataInfrastructure/[email protected]#egg=ckanext-fairdatapoint && \
     pip3 install -r ${APP_DIR}/src/ckanext-fairdatapoint/requirements.txt
 
-# Create log directories for CKAN harvester
-RUN mkdir -p /var/log/ckan/std && chown -R ckan:ckan /var/log/ckan
+USER root
 
-# Copy custom initialization scripts
-COPY docker-entrypoint.d/* /docker-entrypoint.d/
+### Install everything we need for the harvester
+# Install cron and supervisor
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    cron \
+    supervisor \
+    && rm -rf /var/lib/apt/lists/* \
+    && mkdir -p /var/log/ckan/std /var/log/supervisor /etc/supervisor/conf.d \
+    && chown -R ckan:ckan-sys /var/log/ckan /var/log/supervisor /etc/supervisor/conf.d
 
 # Config cron job for harvesting
 COPY config/supervisord.conf /etc/supervisord.d/ckan.conf
-COPY config/crontab ${APP_DIR}/crontab
-RUN crontab ${APP_DIR}/crontab
-
-# Apply any patches needed to CKAN core or any of the built extensions (not the
-# runtime mounted ones)
-COPY patches ${APP_DIR}/patches
+COPY config/crontab /etc/cron.d/ckan-cron
 
-RUN for d in $APP_DIR/patches/*; do \
-    if [ -d $d ]; then \
-    for f in `ls $d/*.patch | sort -g`; do \
-    cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \
-    done ; \
-    fi ; \
-    done
+# Running cron as non-root user and give permission to ckan user
+RUN chmod gu+rw /var/run && \
+    chmod gu+s /usr/sbin/cron && \
+    crontab -u ckan /etc/cron.d/ckan-cron
 
-ENV CKAN__PLUGINS="image_view text_view datatables_view envvars scheming_datasets scheming_organizations dcat dcat_json_interface harvest ckan_harvester dcat_rdf_harvester dcat_json_harvester fairdatapointharvester gdi_userportal activity"
+COPY --chown=ckan-sys:ckan-sys docker-entrypoint.d/* /docker-entrypoint.d/
+COPY --chown=ckan-sys:ckan-sys patches ${APP_DIR}/patches
+COPY --chown=ckan-sys:ckan-sys setup/prerun.py ${APP_DIR}
+COPY --chown=ckan-sys:ckan-sys setup/start_ckan_development.sh ${APP_DIR}
 
-COPY  --chown=ckan:ckan setup/prerun.py ${APP_DIR}
+USER ckan
 
-# TODO 
-# 1. Remove this command once the issue is fixed
-# https://github.com/ckan/ckan-docker-base/issues/41
-RUN mkdir -p /var/lib/ckan/storage/uploads/group && \
-    mkdir -p /var/lib/ckan/storage/uploads/user && \
-    chmod -R u+rwx "/var/lib/ckan"
-
-# https://github.com/ckan/ckan-docker-base/issues/82
-# CVE-2024-32002 was found on git and related packages
-RUN apk del git git-perl perl-git
+RUN for d in $APP_DIR/patches/*; do \
+        if [ -d $d ]; then \
+            for f in `ls $d/*.patch | sort -g`; do \
+                cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \
+            done ; \
+        fi ; \
+    done
diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev
@@ -4,51 +4,59 @@
 #
 # SPDX-License-Identifier: AGPL-3.0-only
 
-FROM docker.io/ckan/ckan-dev:2.10.6
+FROM ckan/ckan-dev:2.11.1
 
-RUN  pip3 install -e git+https://github.com/ckan/[email protected]#egg=ckanext-dcat && \
+USER root
+RUN chown -R ckan:ckan-sys ${APP_DIR}
+USER ckan
+
+
+RUN pip3 install -e git+https://github.com/ckan/[email protected]#egg=ckanext-dcat && \
     pip3 install -r ${APP_DIR}/src/ckanext-dcat/requirements.txt
 
 RUN pip3 install -e git+https://github.com/ckan/[email protected]#egg=ckanext-harvest && \
     pip3 install -r ${APP_DIR}/src/ckanext-harvest/requirements.txt
 
-RUN pip3 install -e git+https://github.com/CivityNL/[email protected]#egg=ckanext-scheming[requirements]    
+RUN pip3 install -e git+https://github.com/CivityNL/[email protected]-1#egg=ckanext-scheming[requirements]    
 
-RUN  pip3 install -e git+https://github.com/GenomicDataInfrastructure/[email protected]#egg=ckanext-fairdatapoint && \
+RUN pip3 install -e git+https://github.com/GenomicDataInfrastructure/[email protected]#egg=ckanext-fairdatapoint && \
     pip3 install -r ${APP_DIR}/src/ckanext-fairdatapoint/requirements.txt
 
-# Create log directories for CKAN harvester
-RUN mkdir -p /var/log/ckan/std && chown -R ckan:ckan /var/log/ckan
+USER root
 
-# Copy custom initialization scripts
-COPY docker-entrypoint.d/* /docker-entrypoint.d/
+### Install everything we need for the harvester
+# Install cron and supervisor
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    cron \
+    supervisor \
+    && rm -rf /var/lib/apt/lists/* \
+    && mkdir -p /var/log/ckan/std /var/log/supervisor /etc/supervisor/conf.d \
+    && chown -R ckan:ckan-sys /var/log/ckan /var/log/supervisor /etc/supervisor/conf.d
 
 # Config cron job for harvesting
 COPY config/supervisord.conf /etc/supervisord.d/ckan.conf
-COPY config/crontab ${APP_DIR}/crontab
-RUN crontab ${APP_DIR}/crontab
+COPY config/crontab /etc/cron.d/ckan-cron
 
-# Apply any patches needed to CKAN core or any of the built extensions (not the
-# runtime mounted ones)
-COPY patches ${APP_DIR}/patches
+# Running cron as non-root user and give permission to ckan user
+RUN chmod gu+rw /var/run && \
+    chmod gu+s /usr/sbin/cron && \
+    crontab -u ckan /etc/cron.d/ckan-cron
 
-RUN for d in $APP_DIR/patches/*; do \
-    if [ -d $d ]; then \
-    for f in `ls $d/*.patch | sort -g`; do \
-    cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \
-    done ; \
-    fi ; \
-    done
+COPY --chown=ckan-sys:ckan-sys docker-entrypoint.d/* /docker-entrypoint.d/
+COPY --chown=ckan-sys:ckan-sys patches ${APP_DIR}/patches
+COPY --chown=ckan-sys:ckan-sys setup/prerun.py ${APP_DIR}
+COPY --chown=ckan-sys:ckan-sys setup/start_ckan_development.sh ${APP_DIR}
 
-ENV CKAN__PLUGINS="image_view text_view datatables_view envvars scheming_datasets scheming_organizations gdi_userportal dcat harvest ckan_harvester dcat_rdf_harvester dcat_json_harvester dcat_json_interface fairdatapointharvester activity"
+USER ckan
 
-COPY  --chown=ckan:ckan setup/prerun.py ${APP_DIR}
-COPY  --chown=ckan:ckan setup/start_ckan_development.sh ${APP_DIR}
+RUN for d in $APP_DIR/patches/*; do \
+        if [ -d $d ]; then \
+            for f in `ls $d/*.patch | sort -g`; do \
+                cd $SRC_DIR/`basename "$d"` && echo "$0: Applying patch $f to $SRC_DIR/`basename $d`"; patch -p1 < "$f" ; \
+            done ; \
+        fi ; \
+    done
 
-RUN mkdir -p /var/lib/ckan/storage/uploads/group && \
-    mkdir -p /var/usr/ckan/storage/uploads/user && \
-    chmod -R u+rwx "/var/lib/ckan"
 
-# https://github.com/ckan/ckan-docker-base/issues/82
-# CVE-2024-32002 was found on git and related packages
-RUN apk del git git-perl perl-git
+
+
diff --git a/ckan/config/crontab b/ckan/config/crontab
@@ -2,6 +2,6 @@
 #
 # SPDX-License-Identifier: AGPL-3.0-only
 # run job every hour
-*/15 * * * * /usr/bin/ckan -c /srv/app/ckan.ini harvester run
-# run cean harvest log every day, at 5 AM
-0 5 * * * /usr/bin/ckan -c /srv/app/ckan.ini harvester clean-harvest-log
+*/15 * * * * /usr/local/bin/ckan -c /srv/app/ckan.ini harvester run
+# run harvest log cleanup weekly, every Sunday at 5 AM
+0 5 * * 0 /usr/local/bin/ckan -c /srv/app/ckan.ini harvester clean-harvest-log
diff --git a/ckan/config/supervisord.conf b/ckan/config/supervisord.conf
@@ -2,12 +2,22 @@
 #
 # SPDX-License-Identifier: AGPL-3.0-only
 
-[program:ckan_gather_consumer]
+# SPDX-FileCopyrightText: Stichting Health-RI
+#
+# SPDX-License-Identifier: AGPL-3.0-only
 
-; Full Path to executable, should be path to virtural environment,
-; Full path to config file too.
+[supervisord]
+; Directory where Supervisor will store its state
+logfile=/var/log/supervisor/supervisord.log
+logfile_maxbytes=50MB
+logfile_backups=10
+loglevel=info
+nodaemon=false
+minfds=1024
+minprocs=200
 
-command=/usr/bin/ckan --config=/srv/app/ckan.ini harvester gather-consumer 
+[program:ckan_gather_consumer]
+command=/usr/local/bin/ckan --config=/srv/app/ckan.ini harvester gather-consumer
 numprocs=1
 stdout_logfile=/var/log/ckan/std/gather_consumer.log
 stderr_logfile=/var/log/ckan/std/gather_consumer.log
@@ -16,30 +26,23 @@ autorestart=true
 startsecs=10
 
 [program:ckan_fetch_consumer]
-
-; Full Path to executable, should be path to virtural environment,
-; Full path to config file too.
-
-command=/usr/bin/ckan --config=/srv/app/ckan.ini harvester fetch-consumer
-
+command=/usr/local/bin/ckan --config=/srv/app/ckan.ini harvester fetch-consumer
 numprocs=1
 stdout_logfile=/var/log/ckan/std/fetch_consumer.log
 stderr_logfile=/var/log/ckan/std/fetch_consumer.log
 autostart=true
 autorestart=true
 startsecs=10
 
-
-[program:crond]
-
-; Full Path to executable, should be path to virtual environment,
-; Full path to config file too.
-
-command=/usr/sbin/crond -f -c /etc/crontabs/
-
+[program:cron]
+command=/usr/sbin/cron -f
 numprocs=1
 stdout_logfile=/var/log/ckan/std/cron.log
 stderr_logfile=/var/log/ckan/std/cron.log
 autostart=true
 autorestart=true
 startsecs=10
+
+[include]
+files = /etc/supervisor/conf.d/*.conf
+