Passage de Postgresql 11 à Postgresql 17

.env.dev.defaults

@@ -52,6 +52,8 @@ GEOSERVER_HOST=localhost
 ################################################################################
 # PostgreSQL
 
+DB_VERSION=pg17-postgis3.5.1
+DB_DATA_VOLUME_NAME=db-data-17
 POSTGRES_PORT=5432
 POSTGRES_USER=postgres
 POSTGRES_DB=monitorenvdb

.env.infra.example

@@ -51,6 +51,8 @@ GEOSERVER_HOST=
 ################################################################################
 # PostgreSQL
 
+DB_VERSION=
+DB_DATA_VOLUME_NAME=
 POSTGRES_PORT=
 POSTGRES_USER=
 POSTGRES_DB=

.env.test.defaults

@@ -42,6 +42,8 @@ MONITORENV_HTTP_NON_PROXY_HOSTS=
 ################################################################################
 # PostgreSQL
 
+DB_VERSION=pg17-postgis3.5.1
+DB_DATA_VOLUME_NAME=db-data-17
 POSTGRES_PORT=5432
 POSTGRES_USER=postgres
 POSTGRES_DB=monitorenvdb

.github/workflows/cicd-database.yml

@@ -4,11 +4,11 @@ on:
   push:
     paths:
       - ".github/workflows/cicd-database.yml"
-      - "infra/docker/database/Dockerfile"
+      - "infra/docker/database/*"
 
 jobs:
-  build:
-    name: Build database image
+  build-legacy-database-image:
+    name: Build Postgresq 11 TimescaleDB 1.7.4 PostGIS 3.3.2 database image
     runs-on: ubuntu-22.04
     permissions:
       contents: read
@@ -38,7 +38,7 @@ jobs:
           context: .
           load: true
           builder: ${{ steps.buildx.outputs.name }}
-          file: infra/docker/database/Dockerfile
+          file: infra/docker/database/ts_postgis.Dockerfile
           push: false
           tags: ${{ env.IMAGE_TAG }}
           cache-from: type=local,src=/tmp/.buildx-cache-database
@@ -56,3 +56,93 @@ jobs:
         run: |
           echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${GITHUB_ACTOR} --password-stdin
           docker push ${{ env.IMAGE_TAG }}
+
+  build-database-image:
+    name: Build database image
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        include:
+          - pg_major: "11"
+            postgis_version: "3.3.2"
+          - pg_major: "15"
+            postgis_version: "3.5.1"
+          - pg_major: "17"
+            postgis_version: "3.5.1"
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set IMAGE_TAG
+        run: echo "IMAGE_TAG=ghcr.io/mtes-mct/monitorenv/monitorenv-database:pg${{ matrix.pg_major }}-postgis${{ matrix.postgis_version }}" >> $GITHUB_ENV
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@master
+
+      - name: Build image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          load: true
+          builder: ${{ steps.buildx.outputs.name }}
+          file: infra/docker/database/database.Dockerfile
+          push: false
+          tags: ${{ env.IMAGE_TAG }}
+          build-args: |
+            PG_MAJOR=${{ matrix.pg_major }}
+            POSTGIS_VERSION=${{ matrix.postgis_version }}
+
+      - name: Push docker image to registry
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${GITHUB_ACTOR} --password-stdin
+          docker push ${{ env.IMAGE_TAG }}
+
+  build-database-upgrade-image:
+    name: Build database upgrade image
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        include:
+          - from_pg_major: "11"
+            to_pg_major: "15"
+            postgis_version: "3.3.2"
+          - from_pg_major: "15"
+            to_pg_major: "17"
+            postgis_version: "3.5.1"
+
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set IMAGE_TAG
+        run: echo "IMAGE_TAG=ghcr.io/mtes-mct/monitorenv/monitorenv-database-upgrade:pg${{ matrix.from_pg_major }}_to_pg${{ matrix.to_pg_major }}-postgis${{ matrix.postgis_version }}" >> $GITHUB_ENV
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@master
+
+      - name: Build image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          load: true
+          builder: ${{ steps.buildx.outputs.name }}
+          file: infra/docker/database/upgrade_pg.Dockerfile
+          push: false
+          tags: ${{ env.IMAGE_TAG }}
+          build-args: |
+            FROM_PG_MAJOR=${{ matrix.from_pg_major }}
+            TO_PG_MAJOR=${{ matrix.to_pg_major }}
+            POSTGIS_VERSION=${{ matrix.postgis_version }}
+
+      - name: Push docker image to registry
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${GITHUB_ACTOR} --password-stdin
+          docker push ${{ env.IMAGE_TAG }}

Makefile

@@ -55,7 +55,7 @@ dev-run-keycloak:
 
 dev-run-infra:
 	@echo "Preparing database"
-	docker compose up -d db
+	docker compose -f docker-compose.yml -f docker-compose-test.yml up -d db
 	@echo "Waiting for TimescaleDB to be ready to accept connections"
 	@while [ -z "$$(docker logs monitorenv_database 2>&1 | grep -o "database system is ready to accept connections")" ]; \
 	do \
@@ -73,7 +73,7 @@ dev-restore-db:
 	sh ./infra/scripts/restore_dev_db.sh
 
 dev-erase-db:
-	docker compose down --remove-orphans -v
+	docker compose -p monitorenv down --remove-orphans -v
 
 dev-clean-build-env:
 	rm -rf $(shell pwd)/backend/build
@@ -135,7 +135,7 @@ docker-push-app:
 test-init-infra-env:
 	npm i @import-meta-env/[email protected] && npx import-meta-env-prepare -u -x ./.env.infra.example -p ./.env.test.defaults
 test-run-infra-for-frontend:
-	export MONITORENV_VERSION=$(VERSION) && docker compose --profile=test up -d
+	export MONITORENV_VERSION=$(VERSION) && docker compose --profile=test -f docker-compose.yml -f docker-compose-test.yml up -d
 test: test-back
 	cd frontend && CI=true npm run test:unit
 
@@ -166,6 +166,40 @@ endif
 restart-app:
 	docker compose --profile production up -d --build --pull always
 
+################################################################################
+# Database upgrade
+
+print_pg_conf_files:
+	docker run --rm -i \
+		-v $(DB_DATA_VOLUME_NAME):/var/lib/postgresql/data \
+		debian:buster \
+		bash < infra/database_upgrade/print_pg_conf_files.sh;
+
+check-database-extensions-versions:
+	docker exec -i monitorenv_database bash < infra/database_upgrade/check_extensions_versions.sh
+
+update-database-extensions:
+	docker exec -i monitorenv_database bash < infra/database_upgrade/update_postgis.sh
+
+upgrade-postgres-11-to-15:
+	docker run --rm \
+		-v $(PG_11_DATA_VOLUME_NAME):/var/lib/postgresql/11/data \
+		-v $(PG_15_DATA_VOLUME_NAME):/var/lib/postgresql/15/data \
+		ghcr.io/mtes-mct/monitorenv/monitorenv-database-upgrade:pg11_to_pg15-postgis3.3.2;
+
+upgrade-postgres-15-to-17:
+	docker run --rm \
+		-v $(PG_15_DATA_VOLUME_NAME):/var/lib/postgresql/15/data \
+		-v $(PG_17_DATA_VOLUME_NAME):/var/lib/postgresql/17/data \
+		ghcr.io/mtes-mct/monitorenv/monitorenv-database-upgrade:pg15_to_pg17-postgis3.5.1;
+
+fix_pg_hba:
+	docker run --rm \
+		-v $(DB_DATA_VOLUME_NAME):/var/lib/postgresql/data \
+		debian:buster \
+		bash -c 'echo "host all all all md5" >> /var/lib/postgresql/data/pg_hba.conf';
+
+
 # ALIASES
 .PHONY: dev
 

backend/src/main/kotlin/fr/gouv/cacem/monitorenv/infrastructure/database/model/StationModel.kt

@@ -63,7 +63,7 @@ data class StationModel(
 
         return FullStationDTO(
             station = toStation(),
-            controlUnitResources = controlUnitResourceModels.map { it.toControlUnitResource() },
+            controlUnitResources = controlUnitResourceModels.map { it.toControlUnitResource() }.sortedBy { it.id },
         )
     }
 

backend/src/main/resources/db/testdata/V666.10__insert_dummy_env_actions.sql

@@ -157,7 +157,7 @@ VALUES ('b8007c8a-5135-4bc3-816f-c69c7b75d807', 100),
        ('c52c6f20-e495-4b29-b3df-d7edfb67fdd7', 100),
        ('f3e90d3a-6ba4-4bb3-805e-d391508aa46d', 112),
        ('e2257638-ddef-4611-960c-7675a3254c38', 107),
-       ('4d9a3139-6c60-49a5-b443-0e6238a6a120', 12),
+       ('4d9a3139-6c60-49a5-b443-0e6238a6a120', 15),
        ('6d4b7d0a-79ce-47cf-ac26-2024d2b27f28', 101),
        ('5865b619-3280-4c67-94ca-9f15da7d5aa7', 3)
 ;
@@ -173,10 +173,10 @@ VALUES ('e2257638-ddef-4611-960c-7675a3254c38', 214),
        ('c52c6f20-e495-4b29-b3df-d7edfb67fdd7', 195),
        ('c52c6f20-e495-4b29-b3df-d7edfb67fdd7', 196),
        ('b8007c8a-5135-4bc3-816f-c69c7b75d807', 180),
-       ('4d9a3139-6c60-49a5-b443-0e6238a6a120', 42),
+       ('4d9a3139-6c60-49a5-b443-0e6238a6a120', 44),
        ('5865b619-3280-4c67-94ca-9f15da7d5aa7', 5)
 ;
 
 INSERT INTO public.env_actions_control_plan_tags(env_action_id, tag_id)
 VALUES ('c52c6f20-e495-4b29-b3df-d7edfb67fdd7', 11),
-       ('c52c6f20-e495-4b29-b3df-d7edfb67fdd7', 15);
+       ('c52c6f20-e495-4b29-b3df-d7edfb67fdd7', 18);

backend/src/main/resources/db/testdata/V666.12__insert_dummy_reportings.sql

@@ -115,14 +115,14 @@ CREATE SEQUENCE IF NOT EXISTS reportings_2023_seq;
 SELECT setval('reportings_2023_seq', (SELECT max(id) FROM reportings), true);
 
 INSERT INTO public.reportings_control_plan_sub_themes
-VALUES (1, 62);
+VALUES (1, 63);
 INSERT INTO public.reportings_control_plan_sub_themes
 VALUES (1, 27);
 INSERT INTO public.reportings_control_plan_sub_themes
-VALUES (2, 82);
+VALUES (2, 83);
 INSERT INTO public.reportings_control_plan_sub_themes
-VALUES (3, 82);
+VALUES (3, 83);
 INSERT INTO public.reportings_control_plan_sub_themes
-VALUES (4, 25);
+VALUES (4, 24);
 INSERT INTO public.reportings_control_plan_sub_themes
-VALUES (6, 82);
+VALUES (6, 83);

backend/src/test/kotlin/fr/gouv/cacem/monitorenv/infrastructure/database/repositories/AbstractDBTests.kt

@@ -25,7 +25,7 @@ abstract class AbstractDBTests {
         @JvmStatic
         val container: GenericContainer<Nothing> =
             GenericContainer<Nothing>(
-                "ghcr.io/mtes-mct/monitorenv/monitorenv-database:pg11-ts1.7.4-postgis3.3.2",
+                "ghcr.io/mtes-mct/monitorenv/monitorenv-database:pg17-postgis3.5.1",
             )
                 .apply {
                     withExposedPorts(5432)

backend/src/test/kotlin/fr/gouv/cacem/monitorenv/infrastructure/database/repositories/JpaBaseRepositoryITests.kt

@@ -19,7 +19,6 @@ class JpaBaseRepositoryITests : AbstractDBTests() {
         val foundFullStations = jpaBaseRepository.findAll().sortedBy { requireNotNull(it.station.id) }
 
         assertThat(foundFullStations).hasSize(3)
-
         assertThat(foundFullStations[0]).isEqualTo(
             FullStationDTO(
                 station =

backend/src/test/kotlin/fr/gouv/cacem/monitorenv/infrastructure/database/repositories/JpaControlPlanTagRepositoryITests.kt

@@ -14,7 +14,7 @@ class JpaControlPlanTagRepositoryITests : AbstractDBTests() {
         // Then
         assertThat(requestedControlPlanTags.size).isEqualTo(17)
         assertThat(requestedControlPlanTags[5].id).isEqualTo(6)
-        assertThat(requestedControlPlanTags[5].themeId).isEqualTo(11)
+        assertThat(requestedControlPlanTags[5].themeId).isEqualTo(14)
         assertThat(requestedControlPlanTags[5].tag)
             .isEqualTo(
                 "Mammifères marins",

backend/src/test/kotlin/fr/gouv/cacem/monitorenv/infrastructure/database/repositories/JpaEnvActionRepositoryITest.kt

@@ -61,7 +61,7 @@ class JpaEnvActionRepositoryITest : AbstractDBTests() {
                 missionId = 38,
                 controlPlans =
                     listOf(
-                        EnvActionControlPlanEntity(themeId = 9, subThemeIds = listOf(51), tagIds = listOf()),
+                        EnvActionControlPlanEntity(themeId = 11, subThemeIds = listOf(51), tagIds = listOf()),
                     ),
             )
 

backend/src/test/kotlin/fr/gouv/cacem/monitorenv/infrastructure/database/repositories/JpaMissionRepositoryITests.kt

@@ -980,8 +980,8 @@ class JpaMissionRepositoryITests : AbstractDBTests() {
                 controlPlans =
                     listOf(
                         EnvActionControlPlanEntity(
-                            subThemeIds = listOf(48),
-                            themeId = 1,
+                            subThemeIds = listOf(47),
+                            themeId = 2,
                         ),
                     ),
                 missionId = id,
@@ -1058,15 +1058,15 @@ class JpaMissionRepositoryITests : AbstractDBTests() {
         val nextControlPlans =
             listOf(
                 EnvActionControlPlanEntity(
-                    subThemeIds = listOf(53, 34),
-                    themeId = 2,
+                    subThemeIds = listOf(57, 33),
+                    themeId = 1,
                 ),
                 EnvActionControlPlanEntity(
                     tagIds = listOf(1, 2, 3),
-                    themeId = 11,
+                    themeId = 14,
                 ),
                 EnvActionControlPlanEntity(
-                    themeId = 17,
+                    themeId = 10,
                 ),
             )
         val nextMission =
@@ -1091,7 +1091,7 @@ class JpaMissionRepositoryITests : AbstractDBTests() {
                 ?.controlPlans
         assertThat(updatedControlPlan?.size).isEqualTo(3)
         assertThat(updatedControlPlan?.get(0)?.subThemeIds?.size).isEqualTo(2)
-        assertThat(updatedControlPlan?.get(0)?.subThemeIds?.get(0)).isEqualTo(53)
+        assertThat(updatedControlPlan?.get(0)?.subThemeIds?.get(0)).isEqualTo(57)
         assertThat(updatedControlPlan?.get(1)?.tagIds?.size).isEqualTo(3)
         assertThat(updatedControlPlan?.get(1)?.tagIds?.get(0)).isEqualTo(1)
     }

backend/src/test/kotlin/fr/gouv/cacem/monitorenv/infrastructure/database/repositories/JpaNatinfRepositoryITests.kt

@@ -14,6 +14,6 @@ class JpaNatinfRepositoryITests : AbstractDBTests() {
     fun `findAll Should return all natinfs`() {
         // When
         val natinfs = jpaNatinfsRepository.findAll()
-        assertThat(natinfs).hasSize(644)
+        assertThat(natinfs).hasSize(645)
     }
 }

datascience/tests/conftest.py

@@ -132,7 +132,7 @@ def start_remote_database_container(
     client = create_docker_client
     print("Starting database container")
     remote_database_container = client.containers.run(
-        "ghcr.io/mtes-mct/monitorenv/monitorenv-database:pg11-ts1.7.4-postgis3.3.2",
+        "ghcr.io/mtes-mct/monitorenv/monitorenv-database:pg17-postgis3.5.1",
         environment={
             "POSTGRES_PASSWORD": os.environ["MONITORENV_REMOTE_DB_PWD"],
             "POSTGRES_USER": os.environ["MONITORENV_REMOTE_DB_USER"],
@@ -199,7 +199,7 @@ def create_tables(set_environment_variables, start_remote_database_container):
             raise Exception(
                 f"Error running migration {m.path.name}. Error message is: {result.output}"
             )
-        
+
 
 @pytest.fixture()
 def create_cacem_tables(create_tables):
@@ -212,6 +212,7 @@ def create_cacem_tables(create_tables):
             print(f"{s.major}.{s.minor}.{s.patch}: {s.path.name}")
             connection.execute(text(s.script))
 
+
 @pytest.fixture()
 def reset_test_data(create_tables):
     e = create_engine("monitorenv_remote")

datascience/tests/test_data/remote_database/V666.3__insert_dummy_env_actions.sql

@@ -13,14 +13,14 @@ INSERT INTO public.env_actions (
 
 INSERT INTO env_actions_control_plan_themes (
     env_action_id, theme_id) VALUES 
-    ('dfb9710a-2217-4f98-94dc-283d3b7bbaae', 2),
+    ('dfb9710a-2217-4f98-94dc-283d3b7bbaae', 1),
     ('88713755-3966-4ca4-ae18-10cab6249485', 107),
-    ('88713755-3966-4ca4-ae18-10cab6249485', 11),
-    ('b05d96b8-387f-4599-bff0-cd7dab71dfb8', 2);
+    ('88713755-3966-4ca4-ae18-10cab6249485', 14),
+    ('b05d96b8-387f-4599-bff0-cd7dab71dfb8', 1);
 
 INSERT INTO env_actions_control_plan_sub_themes (
     env_action_id, subtheme_id) VALUES 
     ('88713755-3966-4ca4-ae18-10cab6249485', 143),
-    ('88713755-3966-4ca4-ae18-10cab6249485', 57),
-    ('88713755-3966-4ca4-ae18-10cab6249485', 58),
-    ('b05d96b8-387f-4599-bff0-cd7dab71dfb8', 34);
+    ('88713755-3966-4ca4-ae18-10cab6249485', 54),
+    ('88713755-3966-4ca4-ae18-10cab6249485', 56),
+    ('b05d96b8-387f-4599-bff0-cd7dab71dfb8', 33);