KK-1417 | Fix SonarCloud security issues & some of the hotspots

.dockerignore

@@ -1,11 +1,59 @@
-node_modules
-Dockerfile*
-docker-compose*
-compose.*
-.dockerignore
-.git
-.gitignore
-README.md
-LICENSE
-.vscode
-build
+# Using "**/" prefix to make ignores work in root and in subdirectories at any level.
+#
+# From https://docs.docker.com/build/concepts/context/#dockerignore-files
+# "Docker also supports a special wildcard string ** that matches any
+# number of directories (including zero). For example, **/*.go excludes
+# all files that end with .go found anywhere in the build context."
+#
+# NOTE:
+# You can add an exception for a specific file by prefixing the line with an !
+# if you need to include a file that would otherwise be ignored.
+
+# Environment variable files
+**/*.env
+**/*.env.example
+**/.env
+**/.env.*
+
+# Certificate/keystore/key files
+**/*.ca-bundle
+**/*.cer
+**/*.cert
+**/*.crt
+**/*.jks
+**/*.key
+**/*.keystore
+**/*.p7b
+**/*.p7c
+**/*.p7s
+**/*.p12
+**/*.pem
+**/*.pfx
+**/*.ppk
+**/*.pvk
+
+# Build files
+**/build
+**/dist
+**/target
+
+# Miscellaneous
+**/*.lock
+**/*.log
+**/*.temp
+**/*.tmp
+**/.DS_Store
+**/.git
+**/.gitignore
+**/.idea
+**/.pytest_cache
+**/.ruff_cache
+**/.ssh
+**/.venv
+**/.vscode
+**/__pycache__
+**/compose.*
+**/node_modules
+**/temp
+**/tmp
+**/venv

.env.keycloak.example → .env.example

@@ -1,9 +1,32 @@
+SECRET_KEY=
+CORS_ORIGIN_ALLOW_ALL=1
+APPLY_MIGRATIONS=1
+ADD_DEFAULT_LANGUAGES=1
 DEBUG=1
-DATABASE_URL=postgres://kukkuu:kukkuu@localhost:5434/kukkuu
+# Values in DATABASE_* and POSTGRES_* variables must match!
+# DATABASE_URL is used by Django
+# DATABASE_HOST is used by docker-entrypoint.sh
+# POSTGRES_* variables are used by the Postgres Docker container
+# i.e. DATABASE_URL should be:
+# postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DATABASE_HOST}/${POSTGRES_DB}
+# or optionally contain a port number e.g. ":5434" after the host
+# Database settings for Docker + Docker compose based local development:
+DATABASE_URL=postgres://kukkuu:[email protected]/kukkuu
+DATABASE_HOST=kukkuu-db.helsinki
+# Database settings for non-Docker local development:
+# DATABASE_URL=postgres://kukkuu:kukkuu@localhost/kukkuu
+# DATABASE_HOST=localhost
+POSTGRES_USER=kukkuu
+POSTGRES_PASSWORD=kukkuu
+POSTGRES_DB=kukkuu
+SKIP_DATABASE_CHECK=0
+ALLOWED_HOSTS=*
 CORS_ALLOWED_ORIGINS=http://localhost:3000,http://localhost:3001,http://localhost:3002
 CORS_ORIGIN_ALLOW_ALL=True
-ALLOWED_HOSTS=*
+# For Keycloak test environment authentication service:
 TOKEN_AUTH_AUTHSERVER_URL=https://tunnistus.test.hel.ninja/auth/realms/helsinki-tunnistus
+# For local Tunnistamo authentication service:
+# TOKEN_AUTH_AUTHSERVER_URL=http://tunnistamo-backend:8000/openid
 # For local Kukkuu API:
 TOKEN_AUTH_ACCEPTED_AUDIENCE=kukkuu-api-dev,profile-api-test
 # For test env Kukkuu API:
@@ -14,9 +37,10 @@ GDPR_API_QUERY_SCOPE=gdprquery
 GDPR_API_DELETE_SCOPE=gdprdelete
 GDPR_API_AUTHORIZATION_FIELD=authorization.permissions.scopes
 HELUSERS_BACK_CHANNEL_LOGOUT_ENABLED=True
+HELUSERS_PASSWORD_LOGIN_DISABLED=False
 KUKKUU_HASHID_SALT=abcdefg123456
 KUKKUU_TICKET_VERIFICATION_URL=http://localhost:3000/ticket-verification-endpoint/{reference_id}
-MAIL_MAILGUN_KEY
+MAIL_MAILGUN_KEY=
 MAIL_MAILGUN_DOMAIN=hel.fi
 MAIL_MAILGUN_API=https://api.eu.mailgun.net/v3
 KUKKUU_NOTIFICATIONS_SHEET_ID=1TkdQsO50DHOg5pi1JhzudOL1GKpiK-V2DCIoAipKj-M
@@ -26,7 +50,6 @@ TOKEN_AUTH_BROWSER_TEST_JWT_ISSUER=https://kukkuu-ui.test.hel.ninja,https://kukk
 
 # Django-admin Keycloak login related variables:
 SOCIAL_AUTH_TUNNISTAMO_KEY=kukkuu-django-admin-dev
-# Get secret from development-kv library → hki-CpsLjsaY-dev-kv keyvault → SOCIAL-AUTH-TUNNISTAMO-SECRET:
-# https://portal.azure.com/#@helsinginkaupunki.onmicrosoft.com/asset/Microsoft_Azure_KeyVault/Secret/https://hki-cpsljsay-dev-kv.vault.azure.net/secrets/SOCIAL-AUTH-TUNNISTAMO-SECRET
-SOCIAL_AUTH_TUNNISTAMO_SECRET=please-get-secret-from-keyvault
+# Get secret from keyvault, see README.md for instructions:
+SOCIAL_AUTH_TUNNISTAMO_SECRET=
 SOCIAL_AUTH_TUNNISTAMO_OIDC_ENDPOINT=https://tunnistus.test.hel.ninja/auth/realms/helsinki-tunnistus

.gitignore

@@ -24,6 +24,13 @@
 .DS_Store
 .eggs/
 .env
+.env.development
+.env.development.local
+.env.local
+.env.production
+.env.production.local
+.env.test
+.env.test.local
 .grunt
 .hypothesis/
 .idea

.pre-commit-config.yaml

@@ -12,7 +12,7 @@ repos:
     # the ruff's version line, it is used by test_pre_commit_ruff_version.py
     # test case in order not to have to add a YAML library dependency just
     # to test this version:
-    rev: v0.9.6 # ruff-pre-commit version
+    rev: v0.9.9 # ruff-pre-commit version
     hooks:
       # Run the linter
       - id: ruff

Dockerfile

@@ -7,44 +7,41 @@ WORKDIR /app
 
 RUN mkdir /entrypoint
 
-COPY --chown=default:root requirements.txt /app/requirements.txt
-COPY --chown=default:root requirements-prod.txt /app/requirements-prod.txt
+# chmod=755 = rwxr-xr-x i.e. owner can read, write and execute, group and others can read and execute.
+#
+# Related to SonarCloud security hotspot docker:S6470 i.e.
+# "Recursively copying context directories is security-sensitive" i.e.
+# https://rules.sonarsource.com/docker/RSPEC-6470/
+# see .dockerignore for info on what is not copied here:
+COPY --chown=root:root --chmod=755 . /app/
 
 RUN yum update -y && yum install -y \
     nc \
     && pip install -U pip \
     && pip install --no-cache-dir -r /app/requirements.txt \
     && pip install --no-cache-dir  -r /app/requirements-prod.txt
 
-COPY --chown=default:root docker-entrypoint.sh /entrypoint/docker-entrypoint.sh
+# fatal: detected dubious ownership in repository at '/app'
+RUN git config --system --add safe.directory /app
+
+COPY --chown=root:root --chmod=755 docker-entrypoint.sh /entrypoint/docker-entrypoint.sh
 ENTRYPOINT ["/entrypoint/docker-entrypoint.sh"]
 
 # ==============================
 FROM appbase AS development
 # ==============================
 
-COPY --chown=default:root requirements-dev.txt /app/requirements-dev.txt
 RUN pip install --no-cache-dir -r /app/requirements-dev.txt
 
 ENV DEV_SERVER=1
 
-COPY --chown=default:root . /app/
-
-# fatal: detected dubious ownership in repository at '/app'
-RUN git config --system --add safe.directory /app
-
 USER default
 EXPOSE 8081/tcp
 
 # ==============================
 FROM appbase AS production
 # ==============================
 
-COPY --chown=default:root . /app/
-
-# fatal: detected dubious ownership in repository at '/app'
-RUN git config --system --add safe.directory /app
-
 RUN SECRET_KEY="only-used-for-collectstatic" KUKKUU_HASHID_SALT="only-used-for-collectstatic" python manage.py collectstatic
 
 USER default

README.md

@@ -15,6 +15,8 @@
     - [Database](#database)
     - [Notification import](#notification-import)
     - [Daily running, Debugging](#daily-running-debugging)
+  - [Generating secret key for Django](#generating-secret-key-for-django)
+  - [Getting secret for django-admin login](#getting-secret-for-django-admin-login)
   - [Keeping Python requirements up to date](#keeping-python-requirements-up-to-date)
   - [Code linting & formatting](#code-linting--formatting)
   - [Pre-commit hooks](#pre-commit-hooks)
@@ -101,9 +103,10 @@ Optionally if you want to use pre-commit hooks:
 
 ### Development with Docker
 
-1. Copy `docker-compose.env.example` to `docker-compose.env`, then modify it as needed. At a minimum, ensure that `SOCIAL_AUTH_TUNNISTAMO_SECRET` is set using the secret from keyvault.
-
-2. Run `docker compose up`
+1. Copy `.env.example` to `.env`
+2. Set value for `SECRET_KEY` to `.env` with [Generating secret key for Django](#generating-secret-key-for-django) instructions
+3. Set value for `SOCIAL_AUTH_TUNNISTAMO_SECRET` to `.env` with [Getting secret for django admin login](#getting-secret-for-django-admin-login) instructions
+4. Run `docker compose up`
 
 If you do not have a super user / admin to administrate the API yet, you can create one with:
 
@@ -116,7 +119,11 @@ The project is now running at http://localhost:8081 and using public Keycloak te
 
 ### Development without Docker
 
-Start by installing the [requirements](#requirements).
+1. Install [requirements](#requirements)
+2. Copy `.env.example` to `.env`
+3. Set value for `SECRET_KEY` to `.env` with [Generating secret key for Django](#generating-secret-key-for-django) instructions
+4. Set value for `SOCIAL_AUTH_TUNNISTAMO_SECRET` to `.env` with [Getting secret for django admin login](#getting-secret-for-django-admin-login) instructions
+5. Modify `DATABASE_URL` and `DATABASE_HOST` in your `.env` file based on where your PostgreSQL database is set up
 
 #### Installing Python requirements
 
@@ -169,6 +176,40 @@ KUKKUU_NOTIFICATIONS_SHEET_ID=1TkdQsO50DHOg5pi1JhzudOL1GKpiK-V2DCIoAipKj-M
 - Run `python manage.py runserver localhost:8081`
 - The project is now running at http://localhost:8081
 
+### Generating secret key for Django
+
+Django needs a value for [SECRET_KEY](https://docs.djangoproject.com/en/4.2/ref/settings/#secret-key) to start.
+
+For production, you should use a strong, long, randomly generated key.
+
+For local development, if you prefer, you can alternatively use a shorter, manually generated key.
+
+Here's how you can generate a value for `SECRET_KEY` using Python (Based on Django v5.1.6's
+[get_random_secret_key](https://github.com/django/django/blob/5.1.6/django/core/management/utils.py#L79C5-L84) &
+[get_random_string](https://github.com/django/django/blob/5.1.6/django/utils/crypto.py#L51-L62)):
+```python
+import secrets, string
+allowed_chars = string.ascii_lowercase + string.digits + "!@#$%^&*(-_=+)"
+"".join(secrets.choice(allowed_chars) for i in range(50))
+```
+
+### Getting secret for django-admin login
+
+To be able to log in to django-admin with Keycloak you need to set `SOCIAL_AUTH_TUNNISTAMO_SECRET`
+in your environment—it is for Keycloak even though it's named Tunnistamo. Here's how you can get a
+value for local development i.e. `kukkuu-django-admin-dev` client:
+
+- Go to City of Helsinki's Azure DevOps [kukkuu project](https://dev.azure.com/City-of-Helsinki/kukkuu)
+- Open `Pipelines > Library > development-kv` for the development keyvault library
+- Read `Key vault name` value
+- Open [Azure Portal](https://portal.azure.com/)
+- Search Azure Portal with the key vault name you just read, and open the found key vault
+- Open `Objects > Secrets`
+- Find `SOCIAL-AUTH-TUNNISTAMO-SECRET` (may need pressing `Load more`), and open it
+- Click on the hexadecimal current version value to open the secret's current version
+- Click on the "Copy to clipboard" icon after the `Secret value` to copy it to clipboard
+- Paste the value into your `.env` file as `SOCIAL_AUTH_TUNNISTAMO_SECRET=paste-the-copied-value-here`
+
 ### Keeping Python requirements up to date
 
 If you're using Docker, spin up the container using `docker compose up`

compose.yaml

@@ -2,10 +2,8 @@ services:
     postgres:
         image: postgres:13
         restart: on-failure
-        environment:
-            POSTGRES_USER: kukkuu
-            POSTGRES_PASSWORD: kukkuu
-            POSTGRES_DB: kukkuu
+        env_file:
+            - .env  # For POSTGRES_USER, POSTGRES_PASSWORD and POSTGRES_DB
         ports:
             - 5434:5432
         volumes:
@@ -15,13 +13,9 @@ services:
     django:
         build:
             context: .
-            target: ${DOCKER_TARGET:-development}
+            target: ${DOCKER_TARGET:-development}  # stage of Dockerfile to build
         env_file:
-            - docker-compose.env
-        environment:
-            DATABASE_URL: postgres://kukkuu:kukkuu@kukkuu-db/kukkuu
-            DATABASE_HOST: kukkuu-db.helsinki
-            SKIP_DATABASE_CHECK: 1
+            - .env
         volumes:
             - .:/app
         ports:

docker-entrypoint.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 
-# -z for empty / not assigned variable or -o to check whether the value is 0 (=should be skipped)
+# -z for empty / not assigned variable or -o to check whether the value is 0 (=should not be skipped)
 if [ -z "$SKIP_DATABASE_CHECK" -o "$SKIP_DATABASE_CHECK" = "0" ]; then
   until nc -z -v -w30 "$DATABASE_HOST" 5432
   do

docs/setup-keycloak.md

@@ -25,8 +25,7 @@ HELUSERS_PASSWORD_LOGIN_DISABLED=False
 
 # Django-admin Keycloak login related variables:
 SOCIAL_AUTH_TUNNISTAMO_KEY=kukkuu-django-admin-dev
-# Get secret from development-kv library → hki-CpsLjsaY-dev-kv keyvault → SOCIAL-AUTH-TUNNISTAMO-SECRET:
-# https://portal.azure.com/#@helsinginkaupunki.onmicrosoft.com/asset/Microsoft_Azure_KeyVault/Secret/https://hki-cpsljsay-dev-kv.vault.azure.net/secrets/SOCIAL-AUTH-TUNNISTAMO-SECRET
-SOCIAL_AUTH_TUNNISTAMO_SECRET=please-get-secret-from-keyvault
+# Get secret from keyvault, see README.md for instructions:
+SOCIAL_AUTH_TUNNISTAMO_SECRET=
 SOCIAL_AUTH_TUNNISTAMO_OIDC_ENDPOINT=https://tunnistus.test.hel.ninja/auth/realms/helsinki-tunnistus
 ```

events/factories.py

@@ -1,4 +1,3 @@
-import random
 from zoneinfo import ZoneInfo
 
 import factory
@@ -42,14 +41,12 @@ class Meta:
         skip_postgeneration_save = True  # Not needed after factory v4.0.0
 
 
-def get_external_ticket_system():
-    "Return a random external ticket system from available choices"
-    return random.choice(list(zip(*Event.EXTERNAL_TICKET_SYSTEM_CHOICES))[0])
-
-
 class RandomExternalTicketSystemEventFactory(EventFactory):
     published_at = factory.LazyFunction(lambda: timezone.now())
-    ticket_system = factory.LazyFunction(get_external_ticket_system)
+    ticket_system = factory.Faker(
+        "random_element",
+        elements=[choice[0] for choice in Event.EXTERNAL_TICKET_SYSTEM_CHOICES],
+    )
     ticket_system_url = factory.Faker("url")
     capacity_per_occurrence = None
     duration = None