From eea646e4f24813d699e390536bd1e8d4b05461ef Mon Sep 17 00:00:00 2001 From: --global <--global> Date: Thu, 26 Sep 2024 12:07:37 +0000 Subject: [PATCH 01/11] include postgres in dev container --- .devcontainer/Dockerfile | 1 + .../devcontainer.json | 24 ++++++++---- .devcontainer/docker-compose.yml | 29 ++++++++++++++ .gitignore | 3 ++ codeforlife.code-workspace | 39 ++++++++++++++++++- .../create-multiple-postgresql-databases.sh | 25 ++++++++++++ 6 files changed, 113 insertions(+), 8 deletions(-) create mode 100644 .devcontainer/Dockerfile rename .devcontainer.json => .devcontainer/devcontainer.json (81%) create mode 100644 .devcontainer/docker-compose.yml create mode 100644 scripts/database/create-multiple-postgresql-databases.sh diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000..785375bc --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1 @@ +FROM mcr.microsoft.com/devcontainers/base:ubuntu-22.04 diff --git a/.devcontainer.json b/.devcontainer/devcontainer.json similarity index 81% rename from .devcontainer.json rename to .devcontainer/devcontainer.json index 092fd118..3cdbbd92 100644 --- a/.devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,23 +1,23 @@ { - "image": "mcr.microsoft.com/devcontainers/base:ubuntu-22.04", "customizations": { "vscode": { "extensions": [ // general "visualstudioexptteam.vscodeintellicode", - "github.vscode-pull-request-github", "redhat.vscode-yaml", "davidanson.vscode-markdownlint", "bierner.markdown-mermaid", "streetsidesoftware.code-spell-checker", "tamasfe.even-better-toml", - "github.vscode-github-actions", "codecov.codecov", "ritwickdey.liveserver", - "eamodio.gitlens", "rangav.vscode-thunder-client", "jock.svg", "tyriar.luna-paint", + // git + "github.vscode-pull-request-github", + "github.vscode-github-actions", + "eamodio.gitlens", // javascript "dbaeumer.vscode-eslint", "esbenp.prettier-vscode", @@ -29,11 +29,17 @@ "ms-python.vscode-pylance", "ms-python.mypy-type-checker", "ms-python.black-formatter", - "qwtel.sqlite-viewer", - "njpwerner.autodocstring" + "njpwerner.autodocstring", + // database + "mtxr.sqltools", + "mtxr.sqltools-driver-sqlite", + "mtxr.sqltools-driver-pg" ] } }, + "dockerComposeFile": [ + "./docker-compose.yml" + ], "features": { "ghcr.io/devcontainers/features/github-cli:1": { "version": "2.52.0" @@ -46,9 +52,13 @@ "version": "3.12" } }, + "forwardPorts": [ + 8000, // Django + 5432 // PostgreSQL + ], "name": "workspace", "postCreateCommand": "sudo chmod u+x .submodules/setup/run && .submodules/setup/run", "remoteUser": "root", - "workspaceMount": "source=${localWorkspaceFolder},target=/codeforlife-workspace,type=bind", + "service": "app", "workspaceFolder": "/codeforlife-workspace" } \ No newline at end of file diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 00000000..96a6ae33 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,29 @@ +version: "3" + +services: + app: + build: + context: .. + dockerfile: .devcontainer/Dockerfile + args: + VARIANT: 3.9 + USER_UID: 1000 + USER_GID: 1000 + volumes: + - ..:/codeforlife-workspace:cached + # Overrides default so things don't shut down after the process ends + command: sleep infinity + # Runs app on the same network as the database container, + # allows "forwardPorts" in devcontainer.json function + network_mode: service:db + + db: + image: postgres:16 + restart: unless-stopped + volumes: + - ../scripts/database:/docker-entrypoint-initdb.d + - ../.postgres-data:/var/lib/postgresql/data + environment: + POSTGRES_MULTIPLE_DATABASES: contributor,portal,template + POSTGRES_USER: root + POSTGRES_PASSWORD: password diff --git a/.gitignore b/.gitignore index ff08be71..598df170 100644 --- a/.gitignore +++ b/.gitignore @@ -167,3 +167,6 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ + +# Database +/.postgres-data diff --git a/codeforlife.code-workspace b/codeforlife.code-workspace index 05fd50ef..f3496875 100644 --- a/codeforlife.code-workspace +++ b/codeforlife.code-workspace @@ -45,6 +45,43 @@ "autoDocstring.customTemplatePath": "python-docstring.mustache", "workbench.colorCustomizations": { "editorRuler.foreground": "#008000" - } + }, + "sqltools.connections": [ + { + "previewLimit": 50, + "server": "localhost", + "port": 5432, + "driver": "PostgreSQL", + "name": "contributor", + "database": "contributor", + "username": "root", + "password": "password" + }, + { + "previewLimit": 50, + "server": "localhost", + "port": 5432, + "driver": "PostgreSQL", + "name": "portal", + "database": "portal", + "username": "root", + "password": "password" + }, + { + "previewLimit": 50, + "server": "localhost", + "port": 5432, + "driver": "PostgreSQL", + "name": "template", + "database": "template", + "username": "root", + "password": "password" + } + ] + }, + "extensions": { + "unwantedRecommendations": [ + "ms-azuretools.vscode-docker" + ] } } \ No newline at end of file diff --git a/scripts/database/create-multiple-postgresql-databases.sh b/scripts/database/create-multiple-postgresql-databases.sh new file mode 100644 index 00000000..340483d8 --- /dev/null +++ b/scripts/database/create-multiple-postgresql-databases.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# This file was taken from: +# https://github.com/mrts/docker-postgresql-multiple-databases/tree/dfc6df914b031c0b1de017248e50914013734b38 + +set -e +set -u + +function create_user_and_database() { + local database=$1 + echo " Creating user and database '$database'" + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL + CREATE USER $database; + CREATE DATABASE $database; + GRANT ALL PRIVILEGES ON DATABASE $database TO $database; +EOSQL +} + +if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then + echo "Multiple database creation requested: $POSTGRES_MULTIPLE_DATABASES" + for db in $(echo $POSTGRES_MULTIPLE_DATABASES | tr ',' ' '); do + create_user_and_database $db + done + echo "Multiple databases created" +fi From f004d03359edc254f64428c5664c8412bba22c5c Mon Sep 17 00:00:00 2001 From: --global <--global> Date: Thu, 26 Sep 2024 15:33:51 +0000 Subject: [PATCH 02/11] install postgres --- .github/workflows/test-python-code.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/test-python-code.yaml b/.github/workflows/test-python-code.yaml index f812327f..4e47efed 100644 --- a/.github/workflows/test-python-code.yaml +++ b/.github/workflows/test-python-code.yaml @@ -50,6 +50,16 @@ jobs: PYPROJECT_TOML: ${{ inputs.pyproject-toml-directory }}/pyproject.toml COVERAGE_REPORT: coverage.xml # NOTE: COVERAGE_FILE is reserved - do not use. OCADO_TECH_ORG_ID: 2088731 + services: + postgres: + image: postgres:16 # TODO: dynamically set version + env: + POSTGRES_PASSWORD: password + options: >- # Set health checks to wait until postgres has started + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - name: 🛫 Checkout uses: actions/checkout@v4 From b200e0e2ee19968d3657a4b142af168fbe6b91bd Mon Sep 17 00:00:00 2001 From: --global <--global> Date: Thu, 26 Sep 2024 15:42:40 +0000 Subject: [PATCH 03/11] postgres settings --- .github/workflows/test-python-code.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-python-code.yaml b/.github/workflows/test-python-code.yaml index 4e47efed..eef8624b 100644 --- a/.github/workflows/test-python-code.yaml +++ b/.github/workflows/test-python-code.yaml @@ -54,6 +54,8 @@ jobs: postgres: image: postgres:16 # TODO: dynamically set version env: + POSTGRES_DB: db + POSTGRES_USER: root POSTGRES_PASSWORD: password options: >- # Set health checks to wait until postgres has started --health-cmd pg_isready From 70c9faf61e0db25acd315ac1c5abf1dba45e1976 Mon Sep 17 00:00:00 2001 From: --global <--global> Date: Thu, 26 Sep 2024 15:53:20 +0000 Subject: [PATCH 04/11] ports --- .github/workflows/test-python-code.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-python-code.yaml b/.github/workflows/test-python-code.yaml index eef8624b..df6c65bf 100644 --- a/.github/workflows/test-python-code.yaml +++ b/.github/workflows/test-python-code.yaml @@ -62,6 +62,8 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 + ports: + - 5432:5432 steps: - name: 🛫 Checkout uses: actions/checkout@v4 From 97d91d4a980f48fcfc9be006f5e0c85d7e394a1f Mon Sep 17 00:00:00 2001 From: --global <--global> Date: Fri, 27 Sep 2024 08:36:00 +0000 Subject: [PATCH 05/11] add env vars --- .github/workflows/test-python-code.yaml | 33 ++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-python-code.yaml b/.github/workflows/test-python-code.yaml index df6c65bf..7237723c 100644 --- a/.github/workflows/test-python-code.yaml +++ b/.github/workflows/test-python-code.yaml @@ -38,6 +38,26 @@ on: type: string required: false default: "./codecov.yml" + postgres-version: + description: "The postgres version to set up." + type: number + required: false + default: 17 + postgres-db: + description: "The postgres database to test against." + type: string + required: false + default: "db" + postgres-user: + description: "The postgres user to test with." + type: string + required: false + default: "root" + postgres-password: + description: "The postgres user's password." + type: string + required: false + default: "password" secrets: CODECOV_TOKEN: description: "The token used to gain access to Codecov." @@ -50,13 +70,18 @@ jobs: PYPROJECT_TOML: ${{ inputs.pyproject-toml-directory }}/pyproject.toml COVERAGE_REPORT: coverage.xml # NOTE: COVERAGE_FILE is reserved - do not use. OCADO_TECH_ORG_ID: 2088731 + DB_NAME: ${{ inputs.postgres-db }} + DB_HOST: localhost + DB_USER: ${{ inputs.postgres-user }} + DB_PASSWORD: ${{ inputs.postgres-password }} services: postgres: - image: postgres:16 # TODO: dynamically set version + image: postgres:${{ inputs.postgres-version }} env: - POSTGRES_DB: db - POSTGRES_USER: root - POSTGRES_PASSWORD: password + POSTGRES_DB: ${{ env.DB_NAME }} + POSTGRES_HOST: ${{ env.DB_HOST }} + POSTGRES_USER: ${{ env.DB_USER }} + POSTGRES_PASSWORD: ${{ env.DB_PASSWORD }} options: >- # Set health checks to wait until postgres has started --health-cmd pg_isready --health-interval 10s From ae5a0d7c793d2c4188928aa9dc141ce56ac521a3 Mon Sep 17 00:00:00 2001 From: --global <--global> Date: Fri, 27 Sep 2024 10:53:34 +0000 Subject: [PATCH 06/11] use new test python job --- .github/workflows/backend.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/backend.yaml b/.github/workflows/backend.yaml index 4b199fc0..21e28c69 100644 --- a/.github/workflows/backend.yaml +++ b/.github/workflows/backend.yaml @@ -14,6 +14,11 @@ on: type: string required: false default: "api" + postgres-db: + description: "The postgres database to test against." + type: string + required: false + default: "db" secrets: CODECOV_TOKEN: description: "The token used to gain access to Codecov." @@ -24,11 +29,12 @@ jobs: uses: ocadotechnology/codeforlife-workspace/.github/workflows/validate-pull-request-refs.yaml@main test: - uses: ocadotechnology/codeforlife-workspace/.github/workflows/test-python-code.yaml@main + uses: ocadotechnology/codeforlife-workspace/.github/workflows/test-python-code.yaml@workspace-145 # TODO: set to @main secrets: inherit with: python-version: ${{ inputs.python-version }} source-path: ${{ inputs.source-path }} + postgres-db: ${{ inputs.postgres-db }} deploy: permissions: From ea6f9017f56ba9e9a8b2631313d8e311e1dd6fb0 Mon Sep 17 00:00:00 2001 From: --global <--global> Date: Fri, 27 Sep 2024 11:39:19 +0000 Subject: [PATCH 07/11] postgres 17 --- .devcontainer/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 96a6ae33..23fac3bf 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -18,7 +18,7 @@ services: network_mode: service:db db: - image: postgres:16 + image: postgres:17 restart: unless-stopped volumes: - ../scripts/database:/docker-entrypoint-initdb.d From 68077ddf5417d2450ff3bc725da4893820d78386 Mon Sep 17 00:00:00 2001 From: --global <--global> Date: Fri, 27 Sep 2024 11:39:30 +0000 Subject: [PATCH 08/11] install psql --- .devcontainer/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 785375bc..9563a3b1 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1 +1,6 @@ FROM mcr.microsoft.com/devcontainers/base:ubuntu-22.04 + +# Install PostgreSQL (psql) client. +RUN apt-get update && \ + export DEBIAN_FRONTEND=noninteractive && \ + apt-get -y install --no-install-recommends postgresql-client From 2406888c7e02db3f043f60e9eed6517e63c27508 Mon Sep 17 00:00:00 2001 From: --global <--global> Date: Fri, 27 Sep 2024 11:48:40 +0000 Subject: [PATCH 09/11] remove args --- .devcontainer/docker-compose.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 23fac3bf..8a27ec92 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -5,10 +5,6 @@ services: build: context: .. dockerfile: .devcontainer/Dockerfile - args: - VARIANT: 3.9 - USER_UID: 1000 - USER_GID: 1000 volumes: - ..:/codeforlife-workspace:cached # Overrides default so things don't shut down after the process ends From 51fee3159f3798d27aa9069022afeb380325ea06 Mon Sep 17 00:00:00 2001 From: --global <--global> Date: Fri, 27 Sep 2024 13:07:55 +0000 Subject: [PATCH 10/11] reference example --- .github/workflows/test-python-code.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-python-code.yaml b/.github/workflows/test-python-code.yaml index 7237723c..c8524a71 100644 --- a/.github/workflows/test-python-code.yaml +++ b/.github/workflows/test-python-code.yaml @@ -75,7 +75,7 @@ jobs: DB_USER: ${{ inputs.postgres-user }} DB_PASSWORD: ${{ inputs.postgres-password }} services: - postgres: + postgres: # https://docs.github.com/en/actions/use-cases-and-examples/using-containerized-services/creating-postgresql-service-containers image: postgres:${{ inputs.postgres-version }} env: POSTGRES_DB: ${{ env.DB_NAME }} From 344b5cedbb38a318c709b77cc225f32cd14512cd Mon Sep 17 00:00:00 2001 From: --global <--global> Date: Fri, 27 Sep 2024 14:47:40 +0000 Subject: [PATCH 11/11] set to main --- .github/workflows/backend.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backend.yaml b/.github/workflows/backend.yaml index 21e28c69..d281f751 100644 --- a/.github/workflows/backend.yaml +++ b/.github/workflows/backend.yaml @@ -29,7 +29,7 @@ jobs: uses: ocadotechnology/codeforlife-workspace/.github/workflows/validate-pull-request-refs.yaml@main test: - uses: ocadotechnology/codeforlife-workspace/.github/workflows/test-python-code.yaml@workspace-145 # TODO: set to @main + uses: ocadotechnology/codeforlife-workspace/.github/workflows/test-python-code.yaml@main secrets: inherit with: python-version: ${{ inputs.python-version }}