Merge pull request #106 from qld-gov-au/QOLDEV-1011-ckan-2.11

QOLDEV-1011 add CKAN 2.11 testing

.ahoy.yml

@@ -38,7 +38,7 @@ commands:
       ahoy title "Building and starting Docker containers"
       sh bin/docker-compose.sh up -d "$@"
       echo "Initialising database schema"
-      ahoy cli '$APP_DIR/bin/init.sh'
+      ahoy cli '"${APP_DIR}"/bin/init.sh'
       echo "Waiting for containers to start listening..."
       ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m"
       if sh bin/docker-compose.sh logs | grep -q "\[Error\]"; then exit 1; fi
@@ -79,9 +79,9 @@ commands:
     cmd: |
       CKAN_CONTAINER=$(sh bin/docker-compose.sh ps -q ckan)
       if [ "${#}" -ne 0 \]; then
-        docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"
+        docker exec $CKAN_CONTAINER sh -c '. "${APP_DIR}"/bin/activate; cd $APP_DIR;'" $*"
       else
-        docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'
+        docker exec $CKAN_CONTAINER sh -c '. "${APP_DIR}"/bin/activate && cd $APP_DIR && sh'
       fi
 
   doctor:
@@ -92,7 +92,7 @@ commands:
     usage: Install test site data.
     cmd: |
       ahoy title "Installing a fresh site"
-      ahoy cli '$APP_DIR/bin/init.sh && $APP_DIR/bin/create-test-data.sh'
+      ahoy cli '"${APP_DIR}"/bin/init.sh && "${APP_DIR}"/bin/create-test-data.sh'
 
   clean:
     usage: Remove containers and all build files.
@@ -128,13 +128,13 @@ commands:
     cmd: |
       docker cp . $(sh bin/docker-compose.sh ps -q ckan):/srv/app/
       docker cp bin/ckan_cli $(sh bin/docker-compose.sh ps -q ckan):/usr/bin/
-      ahoy cli 'chmod -v u+x /usr/bin/ckan_cli $APP_DIR/bin/*; cp -v .docker/test.ini $CKAN_INI; $APP_DIR/bin/process-config.sh'
+      ahoy cli 'chmod -v u+x /usr/bin/ckan_cli "${APP_DIR}"/bin/*; cp -v .docker/test.ini $CKAN_INI; "${APP_DIR}"/bin/process-config.sh'
 
   test-unit:
     usage: Run unit tests.
     cmd: |
       ahoy title 'Run unit tests'
-      ahoy cli 'pytest --ckan-ini=${CKAN_INI} $APP_DIR/ckanext' || \
+      ahoy cli 'pytest --ckan-ini=${CKAN_INI} "${APP_DIR}"/ckanext --junit-xml=test/junit/results.xml' || \
       [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ]
 
   test-bdd:
@@ -145,13 +145,14 @@ commands:
       ahoy start-ckan-job-worker
       ahoy start-mailmock &
       sleep 5
+      OUTPUT="--junit-directory=test/junit/"
       if [ "$BEHAVE_TAG" = "" ]; then
-        (ahoy cli "behave -k ${*:-test/features} --tags=smoke" \
-         && ahoy cli "behave -k ${*:-test/features} --tags=-smoke" \
+        (ahoy cli "behave -k ${*:-test/features} $OUTPUT --tags=smoke" \
+         && ahoy cli "behave -k ${*:-test/features} $OUTPUT --tags=-smoke" \
         ) || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
       else
         # run tests with the specified tag
-        ahoy cli "behave -k ${*:-test/features} --tags=$BEHAVE_TAG" \
+        ahoy cli "behave -k ${*:-test/features} $OUTPUT --tags=$BEHAVE_TAG" \
           || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
       fi
       ahoy stop-mailmock
@@ -161,7 +162,7 @@ commands:
     usage: Starts email mock server used for email BDD tests
     cmd: |
       ahoy title 'Starting mailmock'
-      ahoy cli 'mailmock -p 8025 -o ${APP_DIR}/test/emails' # for debugging mailmock email output remove --no-stdout
+      ahoy cli 'mailmock -p 8025 -o "${APP_DIR}"/test/emails' # for debugging mailmock email output remove --no-stdout
 
   stop-mailmock:
     usage: Stops email mock server used for email BDD tests
@@ -180,7 +181,8 @@ commands:
     usage: Stops CKAN background job worker
     cmd: |
       ahoy title 'Stopping CKAN background job worker'
-      ahoy cli "pkill -f 'jobs worker'"
+      # ignore failures from pkill killing itself
+      ahoy cli 'pkill -f "jobs worker"' || true
 
   # Utilities.
   title:

.docker/Dockerfile-template.ckan

@@ -1,4 +1,8 @@
-FROM openknowledge/ckan-dev:{CKAN_VERSION}
+FROM ckan/ckan-dev:{CKAN_VERSION}
+
+# swap between root and unprivileged user
+ARG ORIGINAL_USER
+RUN ORIGINAL_USER=$(id -un)
 
 ARG SITE_URL=http://ckan:5000/
 ENV PYTHON_VERSION={PYTHON_VERSION}
@@ -8,20 +12,25 @@ ENV PYTHON={PYTHON}
 
 WORKDIR "${APP_DIR}"
 
-ENV DOCKERIZE_VERSION v0.6.1
-RUN apk add --no-cache build-base \
-    && curl -sL https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
-        | tar -C /usr/local/bin -xzvf -
-
 COPY .docker/test.ini $CKAN_INI
 
-COPY . ${APP_DIR}/
+COPY . "${APP_DIR}"/
+
+USER root
 
 COPY bin/ckan_cli /usr/bin/
 
-RUN chmod +x ${APP_DIR}/bin/*.sh /usr/bin/ckan_cli
+RUN chmod +x "${APP_DIR}"/bin/*.sh /usr/bin/ckan_cli
+
+ENV DOCKERIZE_VERSION=v0.6.1
+RUN wget -O - https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+        | tar -C /usr/local/bin -xzvf -
+
+RUN which ps || apt-get install -y procps
+
+USER "$ORIGINAL_USER"
 
 # Init current extension.
-RUN ${APP_DIR}/bin/init-ext.sh
+RUN "${APP_DIR}"/bin/init-ext.sh
 
 CMD ["/srv/app/bin/serve.sh"]

.docker/test.ini

@@ -1,16 +1,3 @@
-#
-# CKAN - Pylons configuration
-#
-# These are some of the configuration options available for your CKAN
-# instance. Check the documentation in 'doc/configuration.rst' or at the
-# following URL for a description of what they do and the full list of
-# available options:
-#
-# http://docs.ckan.org/en/latest/maintaining/configuration.html
-#
-# The %(here)s variable will be replaced with the parent directory of this file
-#
-
 [DEFAULT]
 debug = false
 smtp_server = localhost:8025
@@ -33,6 +20,7 @@ beaker.session.key = ckan
 # This is the secret token that the beaker library uses to hash the cookie sent
 # to the client. `paster make-config` generates a unique value for this each
 # time it generates a config file.
+SECRET_KEY = bSmgPpaxg2M+ZRes3u1TXwIcE
 beaker.session.secret = bSmgPpaxg2M+ZRes3u1TXwIcE
 
 # `paster make-config` generates a unique value for this each time it generates
@@ -66,7 +54,6 @@ ckan.auth.user_delete_organizations = true
 ckan.auth.create_user_via_api = false
 ckan.auth.create_user_via_web = true
 ckan.auth.roles_that_cascade_to_sub_groups = admin
-ckan.auth.public_user_details = False
 
 
 ## Search Settings
@@ -92,15 +79,7 @@ ckan.redis.url = redis://redis:6379
 
 ## Plugins Settings
 
-# Note: Add ``datastore`` to enable the CKAN DataStore
-#       Add ``datapusher`` to enable DataPusher
-#       Add ``resource_proxy`` to enable resource proxying and get around the
-#       same origin policy
-ckan.plugins = stats datarequests
-
-# Define which views should be created by default
-# (plugins must be loaded in ckan.plugins)
-ckan.views.default_views = image_view text_view recline_view
+ckan.plugins = datarequests
 
 # Customize which text formats the text_view plugin will show
 #ckan.preview.json_formats = json
@@ -129,20 +108,6 @@ ckan.storage_path = /app/filestore
 #ckan.max_resource_size = 10
 #ckan.max_image_size = 2
 
-## Datapusher settings
-
-# Make sure you have set up the DataStore
-
-#ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
-#ckan.datapusher.url = http://127.0.0.1:8800/
-#ckan.datapusher.assume_task_stale_after = 3600
-
-# Resource Proxy settings
-# Preview size limit, default: 1MB
-#ckan.resource_proxy.max_file_size = 1048576
-# Size of chunks to read/write.
-#ckan.resource_proxy.chunk_size = 4096
-
 ## Activity Streams Settings
 
 #ckan.activity_streams_enabled = true
@@ -182,7 +147,7 @@ ckan.datarequests.enable_closing_circumstances = True
 ckan.comments.check_for_profanity = True
 ckan.comments.bad_words_file = /srv/app/ckanext/datarequests/bad_words.txt
 
-# Logging configuration
+## Logging configuration
 [loggers]
 keys = root, ckan, ckanext
 

.github/workflows/test.yml

@@ -7,20 +7,39 @@ on:
       - master
 
 jobs:
+  # Quick check so we don't waste minutes if there's a Flake8 error
+  lint:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+      - name: Install requirements
+        run: pip install flake8 pycodestyle
+      - name: Check syntax
+        run: flake8
+
   test:
+    needs: lint
     strategy:
       fail-fast: false
       matrix:
-        ckan-version: ["2.10", 2.9, 2.9-py2]
+        ckan-version: ["2.11", "2.10", 2.9]
 
     name: Test on CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
-    container: drevops/ci-builder:23.7.0
+    container: drevops/ci-runner:23.12.0
     env:
       CKAN_VERSION: ${{ matrix.ckan-version }}
 
     steps:
-      - uses: actions/checkout@v3
+      # Patch https://github.com/actions/runner/issues/863
+      - name: Preserve $HOME set in the container
+        run: echo HOME=/root >> "$GITHUB_ENV"
+
+      - uses: actions/checkout@v4
         timeout-minutes: 2
 
       - name: Build
@@ -34,16 +53,22 @@ jobs:
       - name: Retrieve logs
         if: always()
         run: ahoy logs
-        timeout-minutes: 5
+        timeout-minutes: 1
 
-      - name: Retrieve screenshots
-        if: failure()
+      - name: Retrieve results
+        if: always()
         run: bin/process-artifacts.sh
         timeout-minutes: 1
 
+      - name: Test Summary
+        uses: test-summary/action@v2
+        with:
+          paths: "/tmp/artifacts/junit/*.xml"
+        if: always()
+
       - name: Upload screenshots
-        if: failure()
-        uses: actions/upload-artifact@v3
+        if: always()
+        uses: actions/upload-artifact@v4
         with:
           name: CKAN ${{ matrix.ckan-version }} screenshots
           path: /tmp/artifacts/behave/screenshots

bin/build.sh

@@ -14,13 +14,8 @@ sed -i -e "s/##//" docker-compose.yml
 # Pull the latest images.
 ahoy pull
 
-PYTHON=python
-if [ "$CKAN_VERSION" = "2.9-py2" ]; then
-    PYTHON_VERSION=py2
-else
-    PYTHON_VERSION=py3
-    PYTHON="${PYTHON}3"
-fi
+PYTHON_VERSION=py3
+PYTHON="python3"
 
 sed "s|{CKAN_VERSION}|$CKAN_VERSION|g" .docker/Dockerfile-template.ckan \
     | sed "s|{PYTHON_VERSION}|$PYTHON_VERSION|g" \