Enables PIXL DB to operate as a service separate from Orthanc Raw (#593)

* Adds environment variables to enable PIXL DB to be used on a db separate from Orthanc Raw * Implements 'profiles' in docker-compose.yaml to control PIXL DB location * Updates pixl dc up command to use profiles functionality --------- Co-authored-by: Paul Smith <[email protected]>
SAFEHR-data · Feb 6, 2025 · 3f3419c · 3f3419c
1 parent 28db049
commit 3f3419c
Show file tree

Hide file tree

Showing 10 changed files with 164 additions and 32 deletions.
diff --git a/.env.sample b/.env.sample
@@ -4,15 +4,27 @@ PIXL_QUERY_TIMEOUT=10
 CLI_RETRY_SECONDS=300
 
 # PIXL PostgreSQL instance
-PIXL_DB_HOST=postgres
+PIXL_DB_HOST=postgres-exposed
+PIXL_DB_PORT=5432
 PIXL_DB_NAME=pixl
 PIXL_DB_USER=pixl
 PIXL_DB_PASSWORD=
 SKIP_ALEMBIC=false
+EXTERNAL_PIXL_DB=false
+
+# PIXL DB Postgres host
+CLI_PIXL_DB_HOST=localhost
+
+# Orthanc Raw PostgreSQL instance
+ORTHANC_RAW_DB_HOST=postgres-exposed  # change to correct host if PIXL DB is external postgres instance
+ORTHANC_RAW_DB_PORT=5432
+ORTHANC_RAW_DB_NAME=pixl
+ORTHANC_RAW_DB_USER=pixl
+ORTHANC_RAW_DB_PASSWORD=
 
 # Exposed ports
 HASHER_API_PORT=
-POSTGRES_PORT=
+CLI_PIXL_DB_PORT=
 ORTHANC_ANON_DICOM_PORT=
 ORTHANC_ANON_WEB_PORT=
 ORTHANC_RAW_DICOM_PORT=

diff --git a/cli/README.md b/cli/README.md
@@ -20,20 +20,47 @@ uv sync
 
 ## Usage
 
-**Note** The `rabbitmq`, `export-api` and `imaging-api` services must be started prior to using the CLI
+**Note:** The `rabbitmq`, `export-api` and `imaging-api` services must be started prior to using the CLI
 This is done by spinning up the necessary Docker containers through `docker compose`.
+
+See general pixl commands and subcommands with:
+
+```bash
+pixl --help
+```
+
+### Starting PIXL
+
 For convenience, we provide the `pixl dc` command, which acts as a wrapper for `docker compose`,
 but takes care of some of the configuration for you.
 
-See the commands and subcommands with
+**1) Default Start-up**
 
 ```bash
-pixl --help
+pixl dc up
+```
+
+**2) Start-up with External PIXL DB**
+
+PIXL can be set up so that the PIXL DB uses a separate postgres instance to Orthanc Raw, e.g. for production environment configurations. 
+Edit the .env file to enable this:
+
+```bash
+EXTERNAL_PIXL_DB=true
+
+CLI_PIXL_DB_PORT=7001
+
+ORTHANC_RAW_DB_HOST=postgres
+```
+
+Start-up PIXL:
+```bash
+pixl dc up
 ```
 
 ### Configuration
 
-The `rabbitmq` and `postgres` services are configured by setting the following environment variables
+The `rabbitmq` and PIXL DB `postgres` services are configured by setting the following environment variables
 (default values shown):
 
 ```sh
@@ -42,8 +69,8 @@ RABBITMQ_PORT=7008
 RABBITMQ_USERNAME=rabbitmq_username
 RABBITMQ_PASSWORD=rabbitmq_password
 
-POSTGRES_HOST=localhost
-POSTGRES_PORT=7001
+CLI_PIXL_DB_HOST=localhost
+CLI_PIXL_DB_PORT=7001
 PIXL_DB_USER=pixl_db_username
 PIXL_DB_PASSWORD=pixl_db_password
 PIXL_DB_NAME=pixl

diff --git a/cli/src/pixl_cli/_config.py b/cli/src/pixl_cli/_config.py
@@ -29,8 +29,8 @@
         "password": config("RABBITMQ_PASSWORD"),
     },
     "postgres": {
-        "host": config("POSTGRES_HOST"),
-        "port": int(config("POSTGRES_PORT")),
+        "host": config("CLI_PIXL_DB_HOST"),
+        "port": int(config("CLI_PIXL_DB_PORT")),
         "username": config("PIXL_DB_USER"),
         "password": config("PIXL_DB_PASSWORD"),
         "database": config("PIXL_DB_NAME"),

diff --git a/cli/src/pixl_cli/_docker_commands.py b/cli/src/pixl_cli/_docker_commands.py
@@ -34,13 +34,27 @@ def dc(args: tuple[str]) -> None:
     docker_args = list(args)
 
     if "up" in args:
-        docker_args = [*args, "--wait", "--build", "--remove-orphans"]
+        docker_args = _parse_up_args(args)
     if "down" in args:
         docker_args = _check_down_args(args)
 
     run_docker_compose(docker_args, working_dir=PIXL_ROOT)
 
 
+def _parse_up_args(args: tuple[str, ...]) -> list:
+    """Check up args and set docker compose profile"""
+    args_list = list(args)
+
+    up_index = args.index("up")
+    external_pixl_db_env = config("EXTERNAL_PIXL_DB", cast=bool)
+    args_list[up_index:up_index] = (
+        ["--profile", "postgres"] if external_pixl_db_env else ["--profile", "postgres-exposed"]
+    )
+
+    args_list.extend(["--wait", "--build", "--remove-orphans"])
+    return args_list
+
+
 def _check_down_args(args: tuple[str, ...]) -> list:
     """Stop all the PIXL services"""
     if config("ENV") == "prod" and "--volumes" in args:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -46,6 +46,13 @@ x-pixl-db: &pixl-db
     PIXL_DB_PASSWORD: ${PIXL_DB_PASSWORD}
     PIXL_DB_NAME: ${PIXL_DB_NAME}
 
+x-orthanc-raw-db: &orthanc-raw-db
+    ORTHANC_RAW_DB_HOST: ${ORTHANC_RAW_DB_HOST}
+    ORTHANC_RAW_DB_PORT: ${ORTHANC_RAW_DB_PORT}
+    ORTHANC_RAW_DB_USER: ${ORTHANC_RAW_DB_USER}
+    ORTHANC_RAW_DB_PASSWORD: ${ORTHANC_RAW_DB_PASSWORD}
+    ORTHANC_RAW_DB_NAME: ${ORTHANC_RAW_DB_NAME}
+
 x-azure-keyvault: &azure-keyvault
     AZURE_CLIENT_ID: ${EXPORT_AZ_CLIENT_ID}
     AZURE_CLIENT_SECRET: ${EXPORT_AZ_CLIENT_PASSWORD}
@@ -158,8 +165,9 @@ services:
         extra_hosts:
             - "host.docker.internal:host-gateway"
         depends_on:
-            postgres:
+            postgres-exposed:
                 condition: service_healthy
+                required: false
         healthcheck:
             test:
                 [
@@ -186,7 +194,7 @@ services:
         platform: linux/amd64
         command: /run/secrets
         environment:
-            <<: [*pixl-db, *proxy-common, *pixl-common-env]
+            <<: [*orthanc-raw-db, *proxy-common, *pixl-common-env]
             ORTHANC_NAME: "PIXL: Raw"
             ORTHANC_USERNAME: ${ORTHANC_RAW_USERNAME}
             ORTHANC_PASSWORD: ${ORTHANC_RAW_PASSWORD}
@@ -216,6 +224,10 @@ services:
         depends_on:
             postgres:
                 condition: service_healthy
+                required: false
+            postgres-exposed:
+                condition: service_healthy
+                required: false
             orthanc-anon:
                 condition: service_started
         healthcheck:
@@ -283,6 +295,10 @@ services:
                 condition: service_healthy
             postgres:
                 condition: service_healthy
+                required: false
+            postgres-exposed:
+                condition: service_healthy
+                required: false
             hasher-api:
                 condition: service_healthy
         ports:
@@ -352,16 +368,17 @@ services:
 
     ################################################################################
     # Data Stores
-    postgres:
+    postgres-exposed:
+        profiles: [postgres-exposed]
         build:
             context: .
             dockerfile: ./docker/postgres/Dockerfile
             args:
                 <<: *build-args-common
         environment:
-            POSTGRES_USER: ${PIXL_DB_USER}
-            POSTGRES_PASSWORD: ${PIXL_DB_PASSWORD}
-            POSTGRES_DB: ${PIXL_DB_NAME}
+            POSTGRES_USER: ${ORTHANC_RAW_DB_USER}
+            POSTGRES_PASSWORD: ${ORTHANC_RAW_DB_PASSWORD}
+            POSTGRES_DB: ${ORTHANC_RAW_DB_NAME}
             PGTZ: ${TZ:-Europe/London}
         env_file:
             - ./docker/common.env
@@ -371,9 +388,36 @@ services:
               source: postgres-data
               target: /var/lib/postgresql/data
         ports:
-            - "${POSTGRES_PORT}:5432"
+            - "${CLI_PIXL_DB_PORT}:5432"
+        healthcheck:
+            test: ["CMD", "pg_isready", "-U", "${ORTHANC_RAW_DB_USER}", "--dbname", "${ORTHANC_RAW_DB_NAME}"]
+            interval: 10s
+            timeout: 30s
+            retries: 5
+        restart: always
+        networks:
+            - pixl-net
+    postgres:
+        profiles: [postgres]
+        build:
+            context: .
+            dockerfile: ./docker/postgres/Dockerfile
+            args:
+                <<: *build-args-common
+        environment:
+            POSTGRES_USER: ${ORTHANC_RAW_DB_USER}
+            POSTGRES_PASSWORD: ${ORTHANC_RAW_DB_PASSWORD}
+            POSTGRES_DB: ${ORTHANC_RAW_DB_NAME}
+            PGTZ: ${TZ:-Europe/London}
+        env_file:
+            - ./docker/common.env
+        command: postgres -c 'config_file=/etc/postgresql/postgresql.conf'
+        volumes:
+            - type: volume
+              source: postgres-data
+              target: /var/lib/postgresql/data
         healthcheck:
-            test: ["CMD", "pg_isready", "-U", "${PIXL_DB_USER}", "--dbname", "${PIXL_DB_NAME}"]
+            test: ["CMD", "pg_isready", "-U", "${ORTHANC_RAW_DB_USER}", "--dbname", "${ORTHANC_RAW_DB_NAME}"]
             interval: 10s
             timeout: 30s
             retries: 5

diff --git a/orthanc/orthanc-raw/README.md b/orthanc/orthanc-raw/README.md
@@ -41,14 +41,14 @@ secrets. Orthanc interprets all `.json` files in the `/run/secrets` mount as con
 
 ### Step 1
 
-Save credentials `.env` for the VNA (primary) and PACS (secondary) Q/R targets, postgreSQL and 'Orthanc anon'.
+Save credentials `.env` for the VNA (primary) and PACS (secondary) Q/R targets, postgreSQL and 'Orthanc Anon':
 ```
-# PIXL PostgreSQL instance
-PIXL_DB_HOST=
-PIXL_DB_PORT=
-PIXL_DB_NAME=
-PIXL_DB_USER=
-PIXL_DB_PASSWORD=
+# Orthanc Raw PostgreSQL instance
+ORTHANC_RAW_DB_HOST=
+ORTHANC_RAW_DB_PORT=
+ORTHANC_RAW_DB_NAME=
+ORTHANC_RAW_DB_USER=
+ORTHANC_RAW_DB_PASSWORD=
 
 # Exposed ports
 ORTHANC_RAW_DICOM_PORT=XXXX

diff --git a/orthanc/orthanc-raw/config/postgres.json b/orthanc/orthanc-raw/config/postgres.json
@@ -4,7 +4,7 @@
   "PostgreSQL" : {
     "EnableIndex" : true,
     "EnableStorage" : false,
-    "ConnectionUri" : "postgresql://${PIXL_DB_USER}:${PIXL_DB_PASSWORD}@${PIXL_DB_HOST}:${PIXL_DB_PORT}/${PIXL_DB_NAME}",
+    "ConnectionUri" : "postgresql://${ORTHANC_RAW_DB_USER}:${ORTHANC_RAW_DB_PASSWORD}@${ORTHANC_RAW_DB_HOST}:${ORTHANC_RAW_DB_PORT}/${ORTHANC_RAW_DB_NAME}",
     "Lock" : true
   }
 }
diff --git a/test/.env b/test/.env
@@ -8,16 +8,27 @@ PIXL_MAX_MESSAGES_IN_FLIGHT=5
 TZ=Europe/London
 
 # PIXL PostgreSQL instance
-PIXL_DB_HOST=postgres
+PIXL_DB_HOST=postgres-pixl-db
 PIXL_DB_PORT=5432
 PIXL_DB_NAME=pixl
 PIXL_DB_USER=pixl_db_username
 PIXL_DB_PASSWORD=pixl_db_password
 SKIP_ALEMBIC=false
+EXTERNAL_PIXL_DB=true
+
+# PIXL DB Postgres host
+CLI_PIXL_DB_HOST=localhost
+
+# Orthanc Raw PostgreSQL instance
+ORTHANC_RAW_DB_HOST=postgres
+ORTHANC_RAW_DB_PORT=5432
+ORTHANC_RAW_DB_NAME=pixl
+ORTHANC_RAW_DB_USER=orthanc_raw_db_username
+ORTHANC_RAW_DB_PASSWORD=orthanc_raw_db_password
 
 # Exposed ports
 HASHER_API_PORT=7010
-POSTGRES_PORT=7001
+CLI_PIXL_DB_PORT=7001
 ORTHANC_ANON_DICOM_PORT=7002
 ORTHANC_ANON_WEB_PORT=7003
 ORTHANC_RAW_DICOM_PORT=7004
@@ -29,7 +40,6 @@ RABBITMQ_ADMIN_PORT=7009
 FTP_PORT=20021
 
 # PIXL Export API
-POSTGRES_HOST=localhost
 PIXL_EXPORT_API_HOST=localhost
 
 # PIXL Imaging API

diff --git a/test/docker-compose.yml b/test/docker-compose.yml
@@ -13,6 +13,7 @@
 #  limitations under the License.
 volumes:
     vna-qr-data:
+    postgres-pixl-db-data:
 
 networks:
     pixl-net:
@@ -73,3 +74,27 @@ services:
             retries: 2
             interval: 3s
             timeout: 2s
+    postgres-pixl-db:
+        build:
+            context: ../
+            dockerfile: ./docker/postgres/Dockerfile
+        environment:
+            POSTGRES_USER: ${PIXL_DB_USER}
+            POSTGRES_PASSWORD: ${PIXL_DB_PASSWORD}
+            POSTGRES_DB: ${PIXL_DB_NAME}
+            PGTZ: ${TZ:-Europe/London}
+        command: postgres -c 'config_file=/etc/postgresql/postgresql.conf'
+        volumes:
+            - type: volume
+              source: postgres-pixl-db-data
+              target: /var/lib/postgresql/data
+        ports:
+            - "${CLI_PIXL_DB_PORT}:5432"
+        healthcheck:
+            test: ["CMD", "pg_isready", "-U", "${PIXL_DB_USER}", "--dbname", "${PIXL_DB_NAME}"]
+            interval: 10s
+            timeout: 30s
+            retries: 5
+        restart: always
+        networks:
+            - pixl-net
diff --git a/test/run-system-test.sh b/test/run-system-test.sh
@@ -28,14 +28,14 @@ setup() {
     # Warning: Requires to be run from the project root
     (
     	cd "${PACKAGE_DIR}"
-    	docker compose --env-file test/.env --env-file test/.secrets.env -p system-test up --wait -d --build
+    	docker compose --env-file test/.env --env-file test/.secrets.env --profile postgres -p system-test up --wait -d --build
     )
 }
 
 teardown() {
     (
     	cd "${PACKAGE_DIR}"
-    	docker compose -f docker-compose.yml -f test/docker-compose.yml -p system-test down --volumes
+    	docker compose -f docker-compose.yml -f test/docker-compose.yml --profile postgres -p system-test down --volumes
     )
 }