From b5c6166dbf433617e59aa08a735bfdfe61b002c3 Mon Sep 17 00:00:00 2001 From: Moritz Kirmse Date: Mon, 13 Jan 2025 15:39:26 +0100 Subject: [PATCH 01/18] add stub of fastapi server --- backend/#Dockerfile# | 8 ++++++++ backend/.#Dockerfile | 1 + 2 files changed, 9 insertions(+) create mode 100644 backend/#Dockerfile# create mode 120000 backend/.#Dockerfile diff --git a/backend/#Dockerfile# b/backend/#Dockerfile# new file mode 100644 index 0000000..1788ca3 --- /dev/null +++ b/backend/#Dockerfile# @@ -0,0 +1,8 @@ +FROM python:3.12-slim-bullseye + +RUN pip install "fastapi[standard]" requests + +RUN mkdir /app +COPY backend /app + +WORKDIR /app diff --git a/backend/.#Dockerfile b/backend/.#Dockerfile new file mode 120000 index 0000000..9f5b8c0 --- /dev/null +++ b/backend/.#Dockerfile @@ -0,0 +1 @@ +mkirmse@wrk32.738398:1736497701 \ No newline at end of file From c60cdd643094e697bb42098d9861ac312903863c Mon Sep 17 00:00:00 2001 From: Moritz Kirmse Date: Tue, 14 Jan 2025 10:38:01 +0100 Subject: [PATCH 02/18] clean up --- backend/#Dockerfile# | 8 -------- backend/.#Dockerfile | 1 - 2 files changed, 9 deletions(-) delete mode 100644 backend/#Dockerfile# delete mode 120000 backend/.#Dockerfile diff --git a/backend/#Dockerfile# b/backend/#Dockerfile# deleted file mode 100644 index 1788ca3..0000000 --- a/backend/#Dockerfile# +++ /dev/null @@ -1,8 +0,0 @@ -FROM python:3.12-slim-bullseye - -RUN pip install "fastapi[standard]" requests - -RUN mkdir /app -COPY backend /app - -WORKDIR /app diff --git a/backend/.#Dockerfile b/backend/.#Dockerfile deleted file mode 120000 index 9f5b8c0..0000000 --- a/backend/.#Dockerfile +++ /dev/null @@ -1 +0,0 @@ -mkirmse@wrk32.738398:1736497701 \ No newline at end of file From 758fe0ce42ab3642748c6088665b763e78fc53a3 Mon Sep 17 00:00:00 2001 From: Jean-Michel Crepel Date: Tue, 14 Jan 2025 14:39:46 +0100 Subject: [PATCH 03/18] Default docker composition for developping maelstro --- docker/.envs-common | 4 + docker/.envs-database-datafeeder | 10 + docker/.envs-database-georchestra | 23 ++ docker/.envs-hosts | 24 ++ docker/.envs-ldap | 10 + docker/.envs-rabbitmq | 5 + ...nfig-to-host-maelstro-backend-servic.patch | 44 ++++ docker/README.md | 16 ++ docker/config | 1 + .../docker-compose.yml | 0 docker/geor-compose.override.yml | 24 ++ docker/geor-compose.yml | 210 ++++++++++++++++++ docker/resources/caddy/etc/Caddyfile | 55 +++++ .../geoserver_privileged_user_passwd.txt | 1 + docker/secrets/slapd_password.txt | 1 + 15 files changed, 428 insertions(+) create mode 100644 docker/.envs-common create mode 100644 docker/.envs-database-datafeeder create mode 100644 docker/.envs-database-georchestra create mode 100644 docker/.envs-hosts create mode 100644 docker/.envs-ldap create mode 100644 docker/.envs-rabbitmq create mode 100644 docker/0001-tweat-gateway-config-to-host-maelstro-backend-servic.patch create mode 100644 docker/README.md create mode 160000 docker/config rename docker-compose.yml => docker/docker-compose.yml (100%) create mode 100644 docker/geor-compose.override.yml create mode 100644 docker/geor-compose.yml create mode 100644 docker/resources/caddy/etc/Caddyfile create mode 100644 docker/secrets/geoserver_privileged_user_passwd.txt create mode 100644 docker/secrets/slapd_password.txt diff --git a/docker/.envs-common b/docker/.envs-common new file mode 100644 index 0000000..8351395 --- /dev/null +++ b/docker/.envs-common @@ -0,0 +1,4 @@ +# envs-common +FQDN=georchestra-127-0-0-1.nip.io +SMTPHOST=smtp +SMTPPORT=25 \ No newline at end of file diff --git a/docker/.envs-database-datafeeder b/docker/.envs-database-datafeeder new file mode 100644 index 0000000..e12bec0 --- /dev/null +++ b/docker/.envs-database-datafeeder @@ -0,0 +1,10 @@ +# envs-database-datafeeder +DF_PGHOST=postgis +DF_PGDATABASE=datafeeder +DF_PGPORT=5432 +DF_PGUSER=georchestra +DF_PGPASSWORD=georchestra + +POSTGRES_DB=${DF_PGDATABASE} +POSTGRES_USER=${DF_PGUSER} +POSTGRES_PASSWORD=${DF_PGPASSWORD} \ No newline at end of file diff --git a/docker/.envs-database-georchestra b/docker/.envs-database-georchestra new file mode 100644 index 0000000..d63a06f --- /dev/null +++ b/docker/.envs-database-georchestra @@ -0,0 +1,23 @@ +# envs-database-georchestra +PGHOST=database +PGPORT=5432 +PGDATABASE=georchestra +PGUSER=georchestra +PGPASSWORD=georchestra + +POSTGRES_USER=${PGUSER} +POSTGRES_PASSWORD=${PGPASSWORD} + +# extra env var for jdbc +GEODATA_PGDATABASE=${PGDATABASE} +GEODATA_PGHOST=${PGHOST} +GEODATA_PGPORT=${PGPORT} +GEODATA_PGUSER=${PGUSER} +GEODATA_PGPASSWORD=${PGPASSWORD} + +# extra env for gs jdbc/gwc +GWC_PGDATABASE=${PGDATABASE} +GWC_PGHOST=${PGHOST} +GWC_PGPORT=${PGPORT} +GWC_PGUSERNAME=${PGUSER} +GWC_PGPASSWORD=${PGPASSWORD} diff --git a/docker/.envs-hosts b/docker/.envs-hosts new file mode 100644 index 0000000..ea11b68 --- /dev/null +++ b/docker/.envs-hosts @@ -0,0 +1,24 @@ +ANALYTICS_HOST=analytics +CAS_HOST=cas +CONSOLE_HOST=console +GEONETWORK_HOST=geonetwork +GEOSERVER_HOST=geoserver +HEADER_HOST=header +GEOWEBCACHE_HOST=geowebcache +MAPSTORE_HOST=mapstore +DATAFEEDER_HOST=datafeeder +IMPORT_HOST=import +DATAHUB_HOST=datahub +OGC_API_RECORDS_HOST=ogc-api-records +KB_HOST=kibana +KB_PORT=5601 +ES_HOST=elasticsearch +ES_PORT=9200 +RABBITMQ_HOST=rabbitmq + +# needed for geonetwork entrypoint DO NOT REMOVE +CONSOLE_URL=http://${CONSOLE_HOST}:8080 +# needed for kibana DO NOT REMOVE +ELASTICSEARCH_HOSTS=http://${ES_HOST}:${ES_PORT} + +MAELSTRO_HOST=maelstro-back diff --git a/docker/.envs-ldap b/docker/.envs-ldap new file mode 100644 index 0000000..3258f75 --- /dev/null +++ b/docker/.envs-ldap @@ -0,0 +1,10 @@ +# envs-ldap +LDAPHOST=ldap +LDAPPORT=389 +LDAPSCHEME=ldap +LDAPBASEDN=dc=georchestra,dc=org +LDAPADMINDN=cn=admin,dc=georchestra,dc=org +LDAPADMINPASSWORD=secret +LDAPUSERSRDN=ou=users +LDAPROLESRDN=ou=roles +LDAPORGSRDN=ou=orgs \ No newline at end of file diff --git a/docker/.envs-rabbitmq b/docker/.envs-rabbitmq new file mode 100644 index 0000000..8e2fac0 --- /dev/null +++ b/docker/.envs-rabbitmq @@ -0,0 +1,5 @@ +# envs-rabbitmq +RABBITMQ_USERNAME=georchestra +RABBITMQ_PASSWORD=georchestra +# Only apply to applications consuming rabbitmq, doesn't change the port rabbitmq server running in Docker. +RABBITMQ_PORT=5672 diff --git a/docker/0001-tweat-gateway-config-to-host-maelstro-backend-servic.patch b/docker/0001-tweat-gateway-config-to-host-maelstro-backend-servic.patch new file mode 100644 index 0000000..e40ba69 --- /dev/null +++ b/docker/0001-tweat-gateway-config-to-host-maelstro-backend-servic.patch @@ -0,0 +1,44 @@ +From 28f3bc9691e02c9897aa5d774bd03e3d0604ed94 Mon Sep 17 00:00:00 2001 +From: Jean-Michel Crepel +Date: Tue, 14 Jan 2025 12:20:28 +0100 +Subject: [PATCH] tweat gateway config to host maelstro backend service + +--- + gateway/routes.yaml | 10 +++++++++- + 1 file changed, 9 insertions(+), 1 deletion(-) + +diff --git a/gateway/routes.yaml b/gateway/routes.yaml +index 86c25b9..3ca3c2f 100644 +--- a/gateway/routes.yaml ++++ b/gateway/routes.yaml +@@ -10,7 +10,7 @@ spring: + predicates: + - Path=/ + filters: +- - RedirectTo=308, /datahub/ ++ - RedirectTo=308, /geonetwork/ + - id: header + uri: ${georchestra.gateway.services.header.target} + predicates: +@@ -59,6 +59,13 @@ spring: + uri: ${georchestra.gateway.services.ogc-api-records.target} + predicates: + - Path=/ogc-api-records/** ++ - id: maelstro-back ++ uri: ${georchestra.gateway.services.maelstro-back.target} ++ predicates: ++ - Path=/maelstrob/** ++ filters: ++ - RewritePath=/maelstrob/(?.*),/$\{segment} ++ + + georchestra.gateway.services: + console.target: http://${CONSOLE_HOST}:8080/console/ +@@ -71,3 +78,4 @@ georchestra.gateway.services: + import.target: http://${IMPORT_HOST}:80/ + mapstore.target: http://${MAPSTORE_HOST}:8080/mapstore/ + ogc-api-records.target: http://${OGC_API_RECORDS_HOST}:8080/ogc-api-records/ ++ maelstro-back.target: http://${MAELSTRO_HOST}:8000/ +-- +2.34.1 + diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..4f2886e --- /dev/null +++ b/docker/README.md @@ -0,0 +1,16 @@ +# Maelstro docker composition + +accessible from https://georchestra-127-0-0-1.nip.io/maelstrob/ + + + +Start composition + +``` +git submodule update --init --recursive +cd config +git apply ../0001-tweat-gateway-config-to-host-maelstro-backend-servic.patch +cd .. +docker compose up -d +``` + diff --git a/docker/config b/docker/config new file mode 160000 index 0000000..dc2d3c6 --- /dev/null +++ b/docker/config @@ -0,0 +1 @@ +Subproject commit dc2d3c6af2a15a88970e25c0f9624a51aa4435b4 diff --git a/docker-compose.yml b/docker/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to docker/docker-compose.yml diff --git a/docker/geor-compose.override.yml b/docker/geor-compose.override.yml new file mode 100644 index 0000000..49fc190 --- /dev/null +++ b/docker/geor-compose.override.yml @@ -0,0 +1,24 @@ + +services: + caddy: + image: caddy:2.8-alpine + ports: + - "80:80" + - "443:443" + - "127.0.0.1:2019:2019" + environment: + - CADDY_ADMIN=0.0.0.0:2019 + volumes: + - ./resources/ssl:/etc/certs:ro + - ./resources/caddy/etc:/etc/caddy:ro + - ./resources/caddy/data:/data/caddy + - ./resources/static:/usr/share/caddy/static:ro + restart: always + healthcheck: + test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:2019/reverse_proxy/upstreams >/dev/null || exit 1"] + interval: 30s + timeout: 10s + retries: 10 + env_file: + - .envs-common + diff --git a/docker/geor-compose.yml b/docker/geor-compose.yml new file mode 100644 index 0000000..ad5918b --- /dev/null +++ b/docker/geor-compose.yml @@ -0,0 +1,210 @@ + +volumes: + postgresql_data: + ldap_data: + ldap_config: + geoserver_geodata: + geoserver_datadir: + geoserver_tiles: + geoserver_native_libs: + geonetwork_datadir: + esdata: + georchestra_datadir: + datafeeder_postgis_data: + +secrets: + slapd_password: + file: ./secrets/slapd_password.txt + geoserver_privileged_user_passwd: + file: ./secrets/geoserver_privileged_user_passwd.txt + +services: + copy-datadir: + image: alpine + command: sh -c "rm -r /etc/georchestra/* ; cp -r -f -v /mnt/datadir/* /etc/georchestra/ ; chmod 777 -R -v /etc/georchestra/" # "sleep 6000" + volumes: + - ./config:/mnt/datadir + - georchestra_datadir:/etc/georchestra + + envsubst: + image: georchestra/k8s-initcontainer-envsubst + depends_on: + copy-datadir: + condition: service_completed_successfully + environment: + - DEBUG=yes + - SUBST_FILES=/etc/georchestra/security-proxy/targets-mapping.properties /etc/georchestra/datafeeder/frontend-config.json /etc/georchestra/datafeeder/metadata_* /etc/georchestra/geonetwork/microservices/ogc-api-records/config.yml + env_file: + - .envs-common + - .envs-hosts + volumes: + - georchestra_datadir:/etc/georchestra + + database: + image: georchestra/database:24.0.x + env_file: + - .envs-database-georchestra + depends_on: + envsubst: + condition: service_completed_successfully + volumes: + - postgresql_data:/var/lib/postgresql/data + restart: always + + ldap: + image: georchestra/ldap:24.0.x + depends_on: + envsubst: + condition: service_completed_successfully + secrets: + - slapd_password + - geoserver_privileged_user_passwd + environment: + - SLAPD_ORGANISATION=georchestra + - SLAPD_DOMAIN=georchestra.org + - SLAPD_PASSWORD_FILE=/run/secrets/slapd_password + - SLAPD_PASSWORD= + - GEOSERVER_PRIVILEGED_USER_PASSWORD_FILE=/run/secrets/geoserver_privileged_user_passwd + - SLAPD_LOG_LEVEL=32768 # See https://www.openldap.org/doc/admin24/slapdconfig.html#loglevel%20%3Clevel%3E + - RUN_AS_UID=0 + - RUN_AS_GID=0 + - LDAPHOST=localhost + env_file: + - .envs-ldap + volumes: + - ldap_data:/var/lib/ldap + - ldap_config:/etc/ldap + restart: always + + gateway: + image: georchestra/gateway:1.1.x + depends_on: + - database + volumes: + - georchestra_datadir:/etc/georchestra + environment: + - JAVA_TOOL_OPTIONS=-Dgeorchestra.datadir=/etc/georchestra + env_file: + - .envs-common + - .envs-ldap + - .envs-hosts + - .envs-database-georchestra + + geoserver: + image: georchestra/geoserver:24.0.x + healthcheck: + test: ["CMD-SHELL", "curl -s -f http://localhost:8080/geoserver/gwc/service/wmts?SERVICE=WMTS&REQUEST=GetCapabilities >/dev/null || exit 1"] + interval: 30s + timeout: 10s + retries: 10 + depends_on: + ldap: + condition: service_healthy + database: + condition: service_healthy + volumes: + - georchestra_datadir:/etc/georchestra + - geoserver_datadir:/mnt/geoserver_datadir + - geoserver_geodata:/mnt/geoserver_geodata + - geoserver_tiles:/mnt/geoserver_tiles + - geoserver_native_libs:/mnt/geoserver_native_libs + environment: + - JAVA_OPTIONS=-Dorg.eclipse.jetty.annotations.AnnotationParser.LEVEL=OFF + - XMS=256M + - XMX=8G + + env_file: + - .envs-database-georchestra + - .envs-database-datafeeder + restart: always + + console: + image: georchestra/console:24.0.x + healthcheck: + test: ["CMD-SHELL", "curl -s -f http://localhost:8080/console/account/new >/dev/null || exit 1"] + interval: 30s + timeout: 10s + retries: 10 + depends_on: + ldap: + condition: service_healthy + database: + condition: service_healthy + volumes: + - georchestra_datadir:/etc/georchestra + environment: + - JAVA_OPTIONS=-Dorg.eclipse.jetty.annotations.AnnotationParser.LEVEL=OFF + - XMS=256M + - XMX=1G + env_file: + - .envs-common + - .envs-ldap + - .envs-database-georchestra + - .envs-hosts + restart: always + + geonetwork: + image: georchestra/geonetwork:24.0.x + healthcheck: + test: ["CMD-SHELL", "curl -s -f http://localhost:8080/geonetwork/srv/eng/catalog.search >/dev/null || exit 1"] + interval: 30s + timeout: 10s + retries: 10 + depends_on: + console: + condition: service_healthy + database: + condition: service_healthy + elasticsearch: + condition: service_healthy + volumes: + - georchestra_datadir:/etc/georchestra + - geonetwork_datadir:/mnt/geonetwork_datadir + environment: + - JAVA_OPTIONS=-Duser.home=/tmp/jetty -Dgeorchestra.datadir=/etc/georchestra -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005 -Dorg.eclipse.jetty.annotations.AnnotationParser.LEVEL=OFF + - XMS=256M + - XMX=6G + env_file: + - .envs-hosts + - .envs-database-georchestra + restart: always + + postgis: + # used by datafeeder to ingest uploaded user datasets into + image: postgis/postgis:13-3.1-alpine + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 10s + timeout: 3s + retries: 3 + depends_on: + envsubst: + condition: service_completed_successfully + env_file: + - .envs-database-datafeeder + volumes: + - datafeeder_postgis_data:/var/lib/postgresql/data + restart: always + + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.21 + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata:/usr/share/elasticsearch/data + healthcheck: + test: ["CMD-SHELL", "curl -s -f http://localhost:9200/_cat/health >/dev/null || exit 1"] + interval: 30s + timeout: 10s + retries: 10 + depends_on: + envsubst: + condition: service_completed_successfully + environment: + discovery.type: single-node + ES_JAVA_OPTS: -Xms512m -Xmx512m + restart: always + + diff --git a/docker/resources/caddy/etc/Caddyfile b/docker/resources/caddy/etc/Caddyfile new file mode 100644 index 0000000..41404ed --- /dev/null +++ b/docker/resources/caddy/etc/Caddyfile @@ -0,0 +1,55 @@ +(static_fileserver) { + root * /usr/share/caddy/static + file_server +} + +{$FQDN} { + tls internal + # For using a custom certificate: + # tls /etc/certs/ca.pem /etc/certs/key.pem + + @static-resources { + path /favicon.ico + path /crossdomain.xml + path /robots.txt + } + + handle_errors { + @5xx `{err.status_code} >= 500 && {err.status_code} < 600` + handle @5xx { + import static_fileserver + rewrite * /errors/50x.html + } + } + + handle @static-resources { + import static_fileserver + } + + handle_path /public/* { + import static_fileserver + } + + handle /cas/* { + reverse_proxy cas:8080 + } + + handle /webmail/* { + reverse_proxy webmail:80 + } + + # To be removed once import container support automatic redirection. + handle /import { + redir /import /import/ + } + + handle { + reverse_proxy gateway:8080 + header { + Access-Control-Allow-Origin * + Access-Control-Allow-Methods "GET, POST, PUT, PATCH, DELETE, OPTIONS" + Access-Control-Max-Age 1800 + Access-Control-Allow-Credentials: true + } + } +} diff --git a/docker/secrets/geoserver_privileged_user_passwd.txt b/docker/secrets/geoserver_privileged_user_passwd.txt new file mode 100644 index 0000000..76b451f --- /dev/null +++ b/docker/secrets/geoserver_privileged_user_passwd.txt @@ -0,0 +1 @@ +gerlsSnFd6SmM diff --git a/docker/secrets/slapd_password.txt b/docker/secrets/slapd_password.txt new file mode 100644 index 0000000..d97c5ea --- /dev/null +++ b/docker/secrets/slapd_password.txt @@ -0,0 +1 @@ +secret From bb2738130aa0df962ee06ca35cfe0f4adc6dc32f Mon Sep 17 00:00:00 2001 From: Moritz Kirmse <61978301+mki-c2c@users.noreply.github.com> Date: Mon, 20 Jan 2025 10:06:25 +0100 Subject: [PATCH 04/18] rename docker folder (#1) move docker-compose to root --- .../docker-compose.yml => docker-compose.yml | 0 docker/.envs-common | 4 - docker/.envs-database-datafeeder | 10 - docker/.envs-database-georchestra | 23 -- docker/.envs-hosts | 24 -- docker/.envs-ldap | 10 - docker/.envs-rabbitmq | 5 - docker/README.md | 16 -- docker/config | 1 - docker/geor-compose.override.yml | 24 -- docker/geor-compose.yml | 210 ------------------ docker/resources/caddy/etc/Caddyfile | 55 ----- .../geoserver_privileged_user_passwd.txt | 1 - docker/secrets/slapd_password.txt | 1 - ...nfig-to-host-maelstro-backend-servic.patch | 0 15 files changed, 384 deletions(-) rename docker/docker-compose.yml => docker-compose.yml (100%) delete mode 100644 docker/.envs-common delete mode 100644 docker/.envs-database-datafeeder delete mode 100644 docker/.envs-database-georchestra delete mode 100644 docker/.envs-hosts delete mode 100644 docker/.envs-ldap delete mode 100644 docker/.envs-rabbitmq delete mode 100644 docker/README.md delete mode 160000 docker/config delete mode 100644 docker/geor-compose.override.yml delete mode 100644 docker/geor-compose.yml delete mode 100644 docker/resources/caddy/etc/Caddyfile delete mode 100644 docker/secrets/geoserver_privileged_user_passwd.txt delete mode 100644 docker/secrets/slapd_password.txt rename {docker => georchestra}/0001-tweat-gateway-config-to-host-maelstro-backend-servic.patch (100%) diff --git a/docker/docker-compose.yml b/docker-compose.yml similarity index 100% rename from docker/docker-compose.yml rename to docker-compose.yml diff --git a/docker/.envs-common b/docker/.envs-common deleted file mode 100644 index 8351395..0000000 --- a/docker/.envs-common +++ /dev/null @@ -1,4 +0,0 @@ -# envs-common -FQDN=georchestra-127-0-0-1.nip.io -SMTPHOST=smtp -SMTPPORT=25 \ No newline at end of file diff --git a/docker/.envs-database-datafeeder b/docker/.envs-database-datafeeder deleted file mode 100644 index e12bec0..0000000 --- a/docker/.envs-database-datafeeder +++ /dev/null @@ -1,10 +0,0 @@ -# envs-database-datafeeder -DF_PGHOST=postgis -DF_PGDATABASE=datafeeder -DF_PGPORT=5432 -DF_PGUSER=georchestra -DF_PGPASSWORD=georchestra - -POSTGRES_DB=${DF_PGDATABASE} -POSTGRES_USER=${DF_PGUSER} -POSTGRES_PASSWORD=${DF_PGPASSWORD} \ No newline at end of file diff --git a/docker/.envs-database-georchestra b/docker/.envs-database-georchestra deleted file mode 100644 index d63a06f..0000000 --- a/docker/.envs-database-georchestra +++ /dev/null @@ -1,23 +0,0 @@ -# envs-database-georchestra -PGHOST=database -PGPORT=5432 -PGDATABASE=georchestra -PGUSER=georchestra -PGPASSWORD=georchestra - -POSTGRES_USER=${PGUSER} -POSTGRES_PASSWORD=${PGPASSWORD} - -# extra env var for jdbc -GEODATA_PGDATABASE=${PGDATABASE} -GEODATA_PGHOST=${PGHOST} -GEODATA_PGPORT=${PGPORT} -GEODATA_PGUSER=${PGUSER} -GEODATA_PGPASSWORD=${PGPASSWORD} - -# extra env for gs jdbc/gwc -GWC_PGDATABASE=${PGDATABASE} -GWC_PGHOST=${PGHOST} -GWC_PGPORT=${PGPORT} -GWC_PGUSERNAME=${PGUSER} -GWC_PGPASSWORD=${PGPASSWORD} diff --git a/docker/.envs-hosts b/docker/.envs-hosts deleted file mode 100644 index ea11b68..0000000 --- a/docker/.envs-hosts +++ /dev/null @@ -1,24 +0,0 @@ -ANALYTICS_HOST=analytics -CAS_HOST=cas -CONSOLE_HOST=console -GEONETWORK_HOST=geonetwork -GEOSERVER_HOST=geoserver -HEADER_HOST=header -GEOWEBCACHE_HOST=geowebcache -MAPSTORE_HOST=mapstore -DATAFEEDER_HOST=datafeeder -IMPORT_HOST=import -DATAHUB_HOST=datahub -OGC_API_RECORDS_HOST=ogc-api-records -KB_HOST=kibana -KB_PORT=5601 -ES_HOST=elasticsearch -ES_PORT=9200 -RABBITMQ_HOST=rabbitmq - -# needed for geonetwork entrypoint DO NOT REMOVE -CONSOLE_URL=http://${CONSOLE_HOST}:8080 -# needed for kibana DO NOT REMOVE -ELASTICSEARCH_HOSTS=http://${ES_HOST}:${ES_PORT} - -MAELSTRO_HOST=maelstro-back diff --git a/docker/.envs-ldap b/docker/.envs-ldap deleted file mode 100644 index 3258f75..0000000 --- a/docker/.envs-ldap +++ /dev/null @@ -1,10 +0,0 @@ -# envs-ldap -LDAPHOST=ldap -LDAPPORT=389 -LDAPSCHEME=ldap -LDAPBASEDN=dc=georchestra,dc=org -LDAPADMINDN=cn=admin,dc=georchestra,dc=org -LDAPADMINPASSWORD=secret -LDAPUSERSRDN=ou=users -LDAPROLESRDN=ou=roles -LDAPORGSRDN=ou=orgs \ No newline at end of file diff --git a/docker/.envs-rabbitmq b/docker/.envs-rabbitmq deleted file mode 100644 index 8e2fac0..0000000 --- a/docker/.envs-rabbitmq +++ /dev/null @@ -1,5 +0,0 @@ -# envs-rabbitmq -RABBITMQ_USERNAME=georchestra -RABBITMQ_PASSWORD=georchestra -# Only apply to applications consuming rabbitmq, doesn't change the port rabbitmq server running in Docker. -RABBITMQ_PORT=5672 diff --git a/docker/README.md b/docker/README.md deleted file mode 100644 index 4f2886e..0000000 --- a/docker/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Maelstro docker composition - -accessible from https://georchestra-127-0-0-1.nip.io/maelstrob/ - - - -Start composition - -``` -git submodule update --init --recursive -cd config -git apply ../0001-tweat-gateway-config-to-host-maelstro-backend-servic.patch -cd .. -docker compose up -d -``` - diff --git a/docker/config b/docker/config deleted file mode 160000 index dc2d3c6..0000000 --- a/docker/config +++ /dev/null @@ -1 +0,0 @@ -Subproject commit dc2d3c6af2a15a88970e25c0f9624a51aa4435b4 diff --git a/docker/geor-compose.override.yml b/docker/geor-compose.override.yml deleted file mode 100644 index 49fc190..0000000 --- a/docker/geor-compose.override.yml +++ /dev/null @@ -1,24 +0,0 @@ - -services: - caddy: - image: caddy:2.8-alpine - ports: - - "80:80" - - "443:443" - - "127.0.0.1:2019:2019" - environment: - - CADDY_ADMIN=0.0.0.0:2019 - volumes: - - ./resources/ssl:/etc/certs:ro - - ./resources/caddy/etc:/etc/caddy:ro - - ./resources/caddy/data:/data/caddy - - ./resources/static:/usr/share/caddy/static:ro - restart: always - healthcheck: - test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:2019/reverse_proxy/upstreams >/dev/null || exit 1"] - interval: 30s - timeout: 10s - retries: 10 - env_file: - - .envs-common - diff --git a/docker/geor-compose.yml b/docker/geor-compose.yml deleted file mode 100644 index ad5918b..0000000 --- a/docker/geor-compose.yml +++ /dev/null @@ -1,210 +0,0 @@ - -volumes: - postgresql_data: - ldap_data: - ldap_config: - geoserver_geodata: - geoserver_datadir: - geoserver_tiles: - geoserver_native_libs: - geonetwork_datadir: - esdata: - georchestra_datadir: - datafeeder_postgis_data: - -secrets: - slapd_password: - file: ./secrets/slapd_password.txt - geoserver_privileged_user_passwd: - file: ./secrets/geoserver_privileged_user_passwd.txt - -services: - copy-datadir: - image: alpine - command: sh -c "rm -r /etc/georchestra/* ; cp -r -f -v /mnt/datadir/* /etc/georchestra/ ; chmod 777 -R -v /etc/georchestra/" # "sleep 6000" - volumes: - - ./config:/mnt/datadir - - georchestra_datadir:/etc/georchestra - - envsubst: - image: georchestra/k8s-initcontainer-envsubst - depends_on: - copy-datadir: - condition: service_completed_successfully - environment: - - DEBUG=yes - - SUBST_FILES=/etc/georchestra/security-proxy/targets-mapping.properties /etc/georchestra/datafeeder/frontend-config.json /etc/georchestra/datafeeder/metadata_* /etc/georchestra/geonetwork/microservices/ogc-api-records/config.yml - env_file: - - .envs-common - - .envs-hosts - volumes: - - georchestra_datadir:/etc/georchestra - - database: - image: georchestra/database:24.0.x - env_file: - - .envs-database-georchestra - depends_on: - envsubst: - condition: service_completed_successfully - volumes: - - postgresql_data:/var/lib/postgresql/data - restart: always - - ldap: - image: georchestra/ldap:24.0.x - depends_on: - envsubst: - condition: service_completed_successfully - secrets: - - slapd_password - - geoserver_privileged_user_passwd - environment: - - SLAPD_ORGANISATION=georchestra - - SLAPD_DOMAIN=georchestra.org - - SLAPD_PASSWORD_FILE=/run/secrets/slapd_password - - SLAPD_PASSWORD= - - GEOSERVER_PRIVILEGED_USER_PASSWORD_FILE=/run/secrets/geoserver_privileged_user_passwd - - SLAPD_LOG_LEVEL=32768 # See https://www.openldap.org/doc/admin24/slapdconfig.html#loglevel%20%3Clevel%3E - - RUN_AS_UID=0 - - RUN_AS_GID=0 - - LDAPHOST=localhost - env_file: - - .envs-ldap - volumes: - - ldap_data:/var/lib/ldap - - ldap_config:/etc/ldap - restart: always - - gateway: - image: georchestra/gateway:1.1.x - depends_on: - - database - volumes: - - georchestra_datadir:/etc/georchestra - environment: - - JAVA_TOOL_OPTIONS=-Dgeorchestra.datadir=/etc/georchestra - env_file: - - .envs-common - - .envs-ldap - - .envs-hosts - - .envs-database-georchestra - - geoserver: - image: georchestra/geoserver:24.0.x - healthcheck: - test: ["CMD-SHELL", "curl -s -f http://localhost:8080/geoserver/gwc/service/wmts?SERVICE=WMTS&REQUEST=GetCapabilities >/dev/null || exit 1"] - interval: 30s - timeout: 10s - retries: 10 - depends_on: - ldap: - condition: service_healthy - database: - condition: service_healthy - volumes: - - georchestra_datadir:/etc/georchestra - - geoserver_datadir:/mnt/geoserver_datadir - - geoserver_geodata:/mnt/geoserver_geodata - - geoserver_tiles:/mnt/geoserver_tiles - - geoserver_native_libs:/mnt/geoserver_native_libs - environment: - - JAVA_OPTIONS=-Dorg.eclipse.jetty.annotations.AnnotationParser.LEVEL=OFF - - XMS=256M - - XMX=8G - - env_file: - - .envs-database-georchestra - - .envs-database-datafeeder - restart: always - - console: - image: georchestra/console:24.0.x - healthcheck: - test: ["CMD-SHELL", "curl -s -f http://localhost:8080/console/account/new >/dev/null || exit 1"] - interval: 30s - timeout: 10s - retries: 10 - depends_on: - ldap: - condition: service_healthy - database: - condition: service_healthy - volumes: - - georchestra_datadir:/etc/georchestra - environment: - - JAVA_OPTIONS=-Dorg.eclipse.jetty.annotations.AnnotationParser.LEVEL=OFF - - XMS=256M - - XMX=1G - env_file: - - .envs-common - - .envs-ldap - - .envs-database-georchestra - - .envs-hosts - restart: always - - geonetwork: - image: georchestra/geonetwork:24.0.x - healthcheck: - test: ["CMD-SHELL", "curl -s -f http://localhost:8080/geonetwork/srv/eng/catalog.search >/dev/null || exit 1"] - interval: 30s - timeout: 10s - retries: 10 - depends_on: - console: - condition: service_healthy - database: - condition: service_healthy - elasticsearch: - condition: service_healthy - volumes: - - georchestra_datadir:/etc/georchestra - - geonetwork_datadir:/mnt/geonetwork_datadir - environment: - - JAVA_OPTIONS=-Duser.home=/tmp/jetty -Dgeorchestra.datadir=/etc/georchestra -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005 -Dorg.eclipse.jetty.annotations.AnnotationParser.LEVEL=OFF - - XMS=256M - - XMX=6G - env_file: - - .envs-hosts - - .envs-database-georchestra - restart: always - - postgis: - # used by datafeeder to ingest uploaded user datasets into - image: postgis/postgis:13-3.1-alpine - healthcheck: - test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] - interval: 10s - timeout: 3s - retries: 3 - depends_on: - envsubst: - condition: service_completed_successfully - env_file: - - .envs-database-datafeeder - volumes: - - datafeeder_postgis_data:/var/lib/postgresql/data - restart: always - - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.21 - ulimits: - memlock: - soft: -1 - hard: -1 - volumes: - - esdata:/usr/share/elasticsearch/data - healthcheck: - test: ["CMD-SHELL", "curl -s -f http://localhost:9200/_cat/health >/dev/null || exit 1"] - interval: 30s - timeout: 10s - retries: 10 - depends_on: - envsubst: - condition: service_completed_successfully - environment: - discovery.type: single-node - ES_JAVA_OPTS: -Xms512m -Xmx512m - restart: always - - diff --git a/docker/resources/caddy/etc/Caddyfile b/docker/resources/caddy/etc/Caddyfile deleted file mode 100644 index 41404ed..0000000 --- a/docker/resources/caddy/etc/Caddyfile +++ /dev/null @@ -1,55 +0,0 @@ -(static_fileserver) { - root * /usr/share/caddy/static - file_server -} - -{$FQDN} { - tls internal - # For using a custom certificate: - # tls /etc/certs/ca.pem /etc/certs/key.pem - - @static-resources { - path /favicon.ico - path /crossdomain.xml - path /robots.txt - } - - handle_errors { - @5xx `{err.status_code} >= 500 && {err.status_code} < 600` - handle @5xx { - import static_fileserver - rewrite * /errors/50x.html - } - } - - handle @static-resources { - import static_fileserver - } - - handle_path /public/* { - import static_fileserver - } - - handle /cas/* { - reverse_proxy cas:8080 - } - - handle /webmail/* { - reverse_proxy webmail:80 - } - - # To be removed once import container support automatic redirection. - handle /import { - redir /import /import/ - } - - handle { - reverse_proxy gateway:8080 - header { - Access-Control-Allow-Origin * - Access-Control-Allow-Methods "GET, POST, PUT, PATCH, DELETE, OPTIONS" - Access-Control-Max-Age 1800 - Access-Control-Allow-Credentials: true - } - } -} diff --git a/docker/secrets/geoserver_privileged_user_passwd.txt b/docker/secrets/geoserver_privileged_user_passwd.txt deleted file mode 100644 index 76b451f..0000000 --- a/docker/secrets/geoserver_privileged_user_passwd.txt +++ /dev/null @@ -1 +0,0 @@ -gerlsSnFd6SmM diff --git a/docker/secrets/slapd_password.txt b/docker/secrets/slapd_password.txt deleted file mode 100644 index d97c5ea..0000000 --- a/docker/secrets/slapd_password.txt +++ /dev/null @@ -1 +0,0 @@ -secret diff --git a/docker/0001-tweat-gateway-config-to-host-maelstro-backend-servic.patch b/georchestra/0001-tweat-gateway-config-to-host-maelstro-backend-servic.patch similarity index 100% rename from docker/0001-tweat-gateway-config-to-host-maelstro-backend-servic.patch rename to georchestra/0001-tweat-gateway-config-to-host-maelstro-backend-servic.patch From 2fa515ef40d14a5926cd89b450f6e44f0d3bc4fc Mon Sep 17 00:00:00 2001 From: Jean-Michel Crepel Date: Tue, 21 Jan 2025 10:20:44 +0100 Subject: [PATCH 05/18] switch to security proxy and config to make maelstro accessible --- ...nfig-to-host-maelstro-backend-servic.patch | 44 ------------------- 1 file changed, 44 deletions(-) delete mode 100644 georchestra/0001-tweat-gateway-config-to-host-maelstro-backend-servic.patch diff --git a/georchestra/0001-tweat-gateway-config-to-host-maelstro-backend-servic.patch b/georchestra/0001-tweat-gateway-config-to-host-maelstro-backend-servic.patch deleted file mode 100644 index e40ba69..0000000 --- a/georchestra/0001-tweat-gateway-config-to-host-maelstro-backend-servic.patch +++ /dev/null @@ -1,44 +0,0 @@ -From 28f3bc9691e02c9897aa5d774bd03e3d0604ed94 Mon Sep 17 00:00:00 2001 -From: Jean-Michel Crepel -Date: Tue, 14 Jan 2025 12:20:28 +0100 -Subject: [PATCH] tweat gateway config to host maelstro backend service - ---- - gateway/routes.yaml | 10 +++++++++- - 1 file changed, 9 insertions(+), 1 deletion(-) - -diff --git a/gateway/routes.yaml b/gateway/routes.yaml -index 86c25b9..3ca3c2f 100644 ---- a/gateway/routes.yaml -+++ b/gateway/routes.yaml -@@ -10,7 +10,7 @@ spring: - predicates: - - Path=/ - filters: -- - RedirectTo=308, /datahub/ -+ - RedirectTo=308, /geonetwork/ - - id: header - uri: ${georchestra.gateway.services.header.target} - predicates: -@@ -59,6 +59,13 @@ spring: - uri: ${georchestra.gateway.services.ogc-api-records.target} - predicates: - - Path=/ogc-api-records/** -+ - id: maelstro-back -+ uri: ${georchestra.gateway.services.maelstro-back.target} -+ predicates: -+ - Path=/maelstrob/** -+ filters: -+ - RewritePath=/maelstrob/(?.*),/$\{segment} -+ - - georchestra.gateway.services: - console.target: http://${CONSOLE_HOST}:8080/console/ -@@ -71,3 +78,4 @@ georchestra.gateway.services: - import.target: http://${IMPORT_HOST}:80/ - mapstore.target: http://${MAPSTORE_HOST}:8080/mapstore/ - ogc-api-records.target: http://${OGC_API_RECORDS_HOST}:8080/ogc-api-records/ -+ maelstro-back.target: http://${MAELSTRO_HOST}:8000/ --- -2.34.1 - From ac2d7c538baa78a46c40795d0ca7f9557603c781 Mon Sep 17 00:00:00 2001 From: Moritz Kirmse Date: Tue, 14 Jan 2025 15:42:04 +0100 Subject: [PATCH 06/18] setup project with poetry requirement management, CI, and code checks --- .github/workflows/ci.yml | 18 ++++++++ backend/.dockerignore | 1 + backend/Dockerfile | 12 ++++- backend/README.md | 1 + backend/health_check.py | 5 -- backend/maelstro/__init__.py | 58 ++++++++++++++++++++++++ backend/{ => maelstro}/main.py | 13 ++++++ backend/maelstro/scripts/code_check.sh | 10 ++++ backend/maelstro/scripts/code_fix.sh | 5 ++ backend/maelstro/scripts/health_check.py | 7 +++ backend/pyproject.toml | 31 +++++++++++++ backend/tests/__init__.py | 0 docker-compose.yml | 10 ++-- 13 files changed, 158 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 backend/.dockerignore create mode 100644 backend/README.md delete mode 100755 backend/health_check.py create mode 100644 backend/maelstro/__init__.py rename backend/{ => maelstro}/main.py (63%) create mode 100755 backend/maelstro/scripts/code_check.sh create mode 100755 backend/maelstro/scripts/code_fix.sh create mode 100755 backend/maelstro/scripts/health_check.py create mode 100644 backend/pyproject.toml create mode 100644 backend/tests/__init__.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..76d3740 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,18 @@ +name: CI + +on: + push: + +jobs: + check: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v1 + + - name: build + run: docker compose build + + - name: check + run: docker compose run --rm fastapi /app/maelstro/scripts/code_check.sh diff --git a/backend/.dockerignore b/backend/.dockerignore new file mode 100644 index 0000000..9414382 --- /dev/null +++ b/backend/.dockerignore @@ -0,0 +1 @@ +Dockerfile diff --git a/backend/Dockerfile b/backend/Dockerfile index 3ac3ab2..9b5821a 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,8 +1,16 @@ FROM python:3.12-slim-bullseye -RUN pip install "fastapi[standard]" requests +RUN pip install poetry RUN mkdir /app +WORKDIR /app + +COPY pyproject.toml /app + +RUN poetry install --no-root + COPY . /app -WORKDIR /app +RUN poetry install + +CMD ["poetry", "run", "fastapi", "dev", "--host", "0.0.0.0", "main.py"] diff --git a/backend/README.md b/backend/README.md new file mode 100644 index 0000000..86febc7 --- /dev/null +++ b/backend/README.md @@ -0,0 +1 @@ +Maelstro sync app for Georchestra diff --git a/backend/health_check.py b/backend/health_check.py deleted file mode 100755 index ebc0aeb..0000000 --- a/backend/health_check.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/local/bin/python - -import requests - -assert requests.get('http://localhost:8000/health').status_code == 200 diff --git a/backend/maelstro/__init__.py b/backend/maelstro/__init__.py new file mode 100644 index 0000000..b5c3870 --- /dev/null +++ b/backend/maelstro/__init__.py @@ -0,0 +1,58 @@ +""" +Entry point scripts for maelstro backend server +""" + +import uvicorn +from fastapi_cli.utils.cli import get_uvicorn_log_config + + +def dev(): + """ + Dev server entrypoint: + special configuration: + - listens only on localhost + - restarts on code change + """ + uvicorn.run( + app="maelstro.main:app", + host="127.0.0.1", + port=8000, + reload=True, + workers=None, + root_path="", + # proxy_headers=proxy_headers, + log_config=get_uvicorn_log_config(), + ) + + +def docker_dev(): + """ + Dev server entrypoint for running the server inside a docker container: + special configuration: + - restarts on code change + """ + uvicorn.run( + app="maelstro.main:app", + host="0.0.0.0", + port=8000, + reload=True, + workers=None, + root_path="", + # proxy_headers=proxy_headers, + log_config=get_uvicorn_log_config(), + ) + + +def prod(): + """ + Server entrypoint for running the server inside a docker container: + """ + uvicorn.run( + app="maelstro.main:app", + host="0.0.0.0", + port=8000, + workers=None, # to be configured + root_path="", + # proxy_headers=proxy_headers, + log_config=get_uvicorn_log_config(), + ) diff --git a/backend/main.py b/backend/maelstro/main.py similarity index 63% rename from backend/main.py rename to backend/maelstro/main.py index 5ddff2b..8ed9a2e 100644 --- a/backend/main.py +++ b/backend/maelstro/main.py @@ -1,16 +1,29 @@ +""" +Main backend app setup +""" + from fastapi import FastAPI, Response + app = FastAPI() app.state.health_countdown = 5 + @app.get("/") def root_page(): + """ + Hello world dummy route + """ return {"Hello": "World"} @app.get("/health") def health_check(response: Response): + """ + Health check to make sure the server is up and running + For test purposes, the server is reported healthy only from the 5th request onwards + """ status: str = "healthy" if app.state.health_countdown > 0: app.state.health_countdown -= 1 diff --git a/backend/maelstro/scripts/code_check.sh b/backend/maelstro/scripts/code_check.sh new file mode 100755 index 0000000..5bc7ca8 --- /dev/null +++ b/backend/maelstro/scripts/code_check.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +cd "$(dirname "$0")"/.. + +poetry run black --check . && \ +poetry run mypy . && \ +poetry run pyflakes . && \ +poetry run pylint . + +! (( $? & 7 )) # mask exit code for minor findings (refactor, convention, usage) diff --git a/backend/maelstro/scripts/code_fix.sh b/backend/maelstro/scripts/code_fix.sh new file mode 100755 index 0000000..c223dc4 --- /dev/null +++ b/backend/maelstro/scripts/code_fix.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +cd "$(dirname "$0")"/.. + +poetry run black . diff --git a/backend/maelstro/scripts/health_check.py b/backend/maelstro/scripts/health_check.py new file mode 100755 index 0000000..8c1e30c --- /dev/null +++ b/backend/maelstro/scripts/health_check.py @@ -0,0 +1,7 @@ +#!/usr/local/bin/python + +import requests + + +def check(): + assert requests.get("http://localhost:8000/health").status_code == 200 diff --git a/backend/pyproject.toml b/backend/pyproject.toml new file mode 100644 index 0000000..a64e8d0 --- /dev/null +++ b/backend/pyproject.toml @@ -0,0 +1,31 @@ +[project] +name = "maelstro" +version = "0.1.0" +description = "" +authors = [ + {name = "Your Name",email = "you@example.com"} +] +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "fastapi[standard] (>=0.115.6,<0.116.0)", + "requests (>=2.32.3,<3.0.0)", + "mypy (>=1.14.1,<2.0.0)", + "types-requests (>=2.32.0.20241016,<3.0.0.0)", + "pylint (>=3.3.3,<4.0.0)", + "pyflakes (>=3.2.0,<4.0.0)", + "black (>=24.10.0,<25.0.0)" +] + +[tool.poetry.scripts] +health_check = "maelstro.scripts.health_check:check" +dev = "maelstro:dev" +docker_dev = "maelstro:docker_dev" +prod = "maelstro:prod" + +[build-system] +requires = ["poetry-core>=2.0.0,<3.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.pylint.format] +max-line-length = "88" diff --git a/backend/tests/__init__.py b/backend/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/docker-compose.yml b/docker-compose.yml index 5a2d623..91ecdff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,7 @@ services: volumes: - ./backend:/app healthcheck: - test: "/app/health_check.py" + test: "poetry run health_check" interval: 10s retries: 5 start_period: 10s @@ -19,8 +19,6 @@ services: - 8000:8000 command: - - fastapi - - dev - - --host - - 0.0.0.0 - - main.py + - poetry + - run + - docker_dev From d113353f10b1e1b92620b15176fd98e79c83a9c1 Mon Sep 17 00:00:00 2001 From: Moritz Kirmse Date: Tue, 14 Jan 2025 16:07:35 +0100 Subject: [PATCH 07/18] 2 stage build Dockerfile --- .github/workflows/ci.yml | 4 ++-- backend/Dockerfile | 9 ++++++++- backend/pyproject.toml | 15 ++++++++++----- docker-compose.yml | 13 +++++++++++-- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76d3740..0b83ede 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: - uses: actions/checkout@v1 - name: build - run: docker compose build + run: docker compose build check - name: check - run: docker compose run --rm fastapi /app/maelstro/scripts/code_check.sh + run: docker compose run --rm check diff --git a/backend/Dockerfile b/backend/Dockerfile index 9b5821a..1e3c21c 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim-bullseye +FROM python:3.12-slim-bullseye AS poetry RUN pip install poetry @@ -7,6 +7,7 @@ WORKDIR /app COPY pyproject.toml /app +FROM poetry as server RUN poetry install --no-root COPY . /app @@ -14,3 +15,9 @@ COPY . /app RUN poetry install CMD ["poetry", "run", "fastapi", "dev", "--host", "0.0.0.0", "main.py"] + +FROM server as check + +RUN poetry install --no-root --with check + +CMD ["/app/maelstro/scripts/code_check.sh"] diff --git a/backend/pyproject.toml b/backend/pyproject.toml index a64e8d0..fc8abdd 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -10,13 +10,18 @@ requires-python = ">=3.12" dependencies = [ "fastapi[standard] (>=0.115.6,<0.116.0)", "requests (>=2.32.3,<3.0.0)", - "mypy (>=1.14.1,<2.0.0)", - "types-requests (>=2.32.0.20241016,<3.0.0.0)", - "pylint (>=3.3.3,<4.0.0)", - "pyflakes (>=3.2.0,<4.0.0)", - "black (>=24.10.0,<25.0.0)" ] +[tool.poetry.group.check] +optional = true + +[tool.poetry.group.check.dependencies] +mypy=">=1.14.1,<2.0.0" +types-requests=">=2.32.0.20241016,<3.0.0.0" +pylint=">=3.3.3,<4.0.0" +pyflakes=">=3.2.0,<4.0.0" +black=">=24.10.0,<25.0.0" + [tool.poetry.scripts] health_check = "maelstro.scripts.health_check:check" dev = "maelstro:dev" diff --git a/docker-compose.yml b/docker-compose.yml index 91ecdff..70fb767 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,8 +4,10 @@ include: - georchestra/geor-compose.override.yml services: - maelstro-back: - build: ./backend + fastapi: + build: + context: ./backend + target: server volumes: - ./backend:/app healthcheck: @@ -22,3 +24,10 @@ services: - poetry - run - docker_dev + + check: + build: + context: ./backend + target: check + volumes: + - ./backend:/app From 99e1aeab28b603f0a1f3dea219491b726a3fa7ca Mon Sep 17 00:00:00 2001 From: Moritz Kirmse Date: Tue, 14 Jan 2025 16:14:01 +0100 Subject: [PATCH 08/18] do not run check container in socker-compose up --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 70fb767..13ccc30 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,6 +26,8 @@ services: - docker_dev check: + profiles: + - check build: context: ./backend target: check From 60e6ffaf196c7c24279fc6690ef1d295ecf2508e Mon Sep 17 00:00:00 2001 From: Moritz Kirmse Date: Mon, 20 Jan 2025 16:36:30 +0100 Subject: [PATCH 09/18] fix basic backend in docker compo --- backend/maelstro/__init__.py | 2 +- backend/maelstro/main.py | 2 +- docker-compose.yml | 5 +---- georchestra/geor-compose.override.yml | 1 - georchestra/geor-compose.yml | 2 -- georchestra/resources/caddy/data/.gitignore | 2 ++ 6 files changed, 5 insertions(+), 9 deletions(-) create mode 100644 georchestra/resources/caddy/data/.gitignore diff --git a/backend/maelstro/__init__.py b/backend/maelstro/__init__.py index b5c3870..77d8869 100644 --- a/backend/maelstro/__init__.py +++ b/backend/maelstro/__init__.py @@ -19,7 +19,7 @@ def dev(): port=8000, reload=True, workers=None, - root_path="", + root_path="/maelstrob/", # proxy_headers=proxy_headers, log_config=get_uvicorn_log_config(), ) diff --git a/backend/maelstro/main.py b/backend/maelstro/main.py index 8ed9a2e..1c3b5b4 100644 --- a/backend/maelstro/main.py +++ b/backend/maelstro/main.py @@ -5,7 +5,7 @@ from fastapi import FastAPI, Response -app = FastAPI() +app = FastAPI(root_path='/maelstrob') app.state.health_countdown = 5 diff --git a/docker-compose.yml b/docker-compose.yml index 13ccc30..d94ad2d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ include: - georchestra/geor-compose.override.yml services: - fastapi: + maelstro-back: build: context: ./backend target: server @@ -17,9 +17,6 @@ services: start_period: 10s timeout: 10s - ports: - - 8000:8000 - command: - poetry - run diff --git a/georchestra/geor-compose.override.yml b/georchestra/geor-compose.override.yml index 49fc190..7042013 100644 --- a/georchestra/geor-compose.override.yml +++ b/georchestra/geor-compose.override.yml @@ -21,4 +21,3 @@ services: retries: 10 env_file: - .envs-common - diff --git a/georchestra/geor-compose.yml b/georchestra/geor-compose.yml index 58f013a..fcbf274 100644 --- a/georchestra/geor-compose.yml +++ b/georchestra/geor-compose.yml @@ -251,5 +251,3 @@ services: discovery.type: single-node ES_JAVA_OPTS: -Xms512m -Xmx512m restart: no - - diff --git a/georchestra/resources/caddy/data/.gitignore b/georchestra/resources/caddy/data/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/georchestra/resources/caddy/data/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore From 0811ed36d5f96ad730e560baa194084ad6dc4948 Mon Sep 17 00:00:00 2001 From: Moritz Kirmse Date: Tue, 21 Jan 2025 08:33:28 +0100 Subject: [PATCH 10/18] fix app behaviour behind https gatewway --- backend/maelstro/__init__.py | 6 +++- backend/maelstro/main.py | 59 ++++++++++++++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/backend/maelstro/__init__.py b/backend/maelstro/__init__.py index 77d8869..39938b1 100644 --- a/backend/maelstro/__init__.py +++ b/backend/maelstro/__init__.py @@ -19,7 +19,7 @@ def dev(): port=8000, reload=True, workers=None, - root_path="/maelstrob/", + root_path="", # proxy_headers=proxy_headers, log_config=get_uvicorn_log_config(), ) @@ -56,3 +56,7 @@ def prod(): # proxy_headers=proxy_headers, log_config=get_uvicorn_log_config(), ) + + +if __name__ == "__main__": + docker_dev() diff --git a/backend/maelstro/main.py b/backend/maelstro/main.py index 1c3b5b4..1ef1695 100644 --- a/backend/maelstro/main.py +++ b/backend/maelstro/main.py @@ -2,14 +2,16 @@ Main backend app setup """ -from fastapi import FastAPI, Response +from typing import Annotated +from fastapi import FastAPI, Request, Response, Header -app = FastAPI(root_path='/maelstrob') +app = FastAPI(root_path="/maelstrob") app.state.health_countdown = 5 +@app.head("/") @app.get("/") def root_page(): """ @@ -18,6 +20,59 @@ def root_page(): return {"Hello": "World"} +@app.get("/user") +def user_page( + sec_username: Annotated[str | None, Header()] = None, + sec_org: Annotated[str | None, Header()] = None, + sec_roles: Annotated[str | None, Header()] = None, + sec_external_authentication: Annotated[str | None, Header()] = None, + sec_proxy: Annotated[str | None, Header()] = None, + sec_orgname: Annotated[str | None, Header()] = None, +): + """ + Display user information provided by gateway + """ + return { + "username": sec_username, + "org": sec_org, + "roles": sec_roles, + "external-authentication": sec_external_authentication, + "proxy": sec_proxy, + "orgname": sec_orgname, + } + + +@app.head("/debug") +@app.get("/debug") +@app.put("/debug") +@app.post("/debug") +@app.delete("/debug") +@app.options("/debug") +@app.patch("/debug") +def debug_page(request: Request): + """ + Display details of query including headers + """ + return { + **{ + k: request[k] + for k in [ + "type", + "asgi", + "http_version", + "server", + "client", + "scheme", + "root_path", + ] + }, + "method": request.method, + "url": request.url, + "headers": dict(request.headers), + "query_params": request.query_params.multi_items(), + } + + @app.get("/health") def health_check(response: Response): """ From f72c720a3ab0d8d0ab70c88fcc7add17519dee74 Mon Sep 17 00:00:00 2001 From: Moritz Kirmse Date: Tue, 21 Jan 2025 08:41:24 +0100 Subject: [PATCH 11/18] add some doc --- README.md | 29 +++++++++++++++++++++++++++++ backend/maelstro/main.py | 4 +++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 96334e1..05f6a65 100644 --- a/README.md +++ b/README.md @@ -6,4 +6,33 @@ geOrchestra Maelstro is an application which helps synchronise geonetwork and ge Refer to documentation from https://github.com/georchestra/docker/tree/master?tab=readme-ov-file#on-linux to trust caddy certificate Also you need to run few commands before to start documented here : [georchestra/README.md](georchestra/README.md) + +## Backend + +The folder [backend](backend) contains the API written with FastAPI. + +### Access +In the global dev composition, the backend is accessible via the https gateway: +https://georchestra-127-0-0-1.nip.io/maelstrob/ + +### SwaggerUI + +FastAPI automatically builds a swagger API web interface which can be found at +https://georchestra-127-0-0-1.nip.io/maelstrob/docs + +Here the various API entrypoints can be tested + +## CI + +Automatic code quality checks are implemented in the CI. + +The code test can be launched manually via the docker command below. +``` +docker compose run --rm check +``` + +In case formatting issues are found, the code can be auto-fixed with: +``` +docker compose run --rm check /app/maelstro/scripts/code_fix.sh +``` diff --git a/backend/maelstro/main.py b/backend/maelstro/main.py index 1ef1695..1385080 100644 --- a/backend/maelstro/main.py +++ b/backend/maelstro/main.py @@ -51,7 +51,9 @@ def user_page( @app.patch("/debug") def debug_page(request: Request): """ - Display details of query including headers + Display details of query including headers. + This may be useful in development to check all the headers procided by the gateway. + This entrypoint should be deactivaed in prod. """ return { **{ From 8772f108db43955bc4299e9e956a173a5bba3fe0 Mon Sep 17 00:00:00 2001 From: Moritz Kirmse Date: Tue, 21 Jan 2025 08:58:59 +0100 Subject: [PATCH 12/18] hide header params in open_api --- backend/maelstro/main.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/backend/maelstro/main.py b/backend/maelstro/main.py index 1385080..24c236e 100644 --- a/backend/maelstro/main.py +++ b/backend/maelstro/main.py @@ -22,12 +22,14 @@ def root_page(): @app.get("/user") def user_page( - sec_username: Annotated[str | None, Header()] = None, - sec_org: Annotated[str | None, Header()] = None, - sec_roles: Annotated[str | None, Header()] = None, - sec_external_authentication: Annotated[str | None, Header()] = None, - sec_proxy: Annotated[str | None, Header()] = None, - sec_orgname: Annotated[str | None, Header()] = None, + sec_username: Annotated[str | None, Header(include_in_schema=False)] = None, + sec_org: Annotated[str | None, Header(include_in_schema=False)] = None, + sec_roles: Annotated[str | None, Header(include_in_schema=False)] = None, + sec_external_authentication: Annotated[ + str | None, Header(include_in_schema=False) + ] = None, + sec_proxy: Annotated[str | None, Header(include_in_schema=False)] = None, + sec_orgname: Annotated[str | None, Header(include_in_schema=False)] = None, ): """ Display user information provided by gateway From 288bfe20d805b06bab0098a7c88c543346e74ce9 Mon Sep 17 00:00:00 2001 From: Moritz Kirmse Date: Tue, 21 Jan 2025 09:25:18 +0100 Subject: [PATCH 13/18] add test --- .github/workflows/ci.yml | 3 +++ backend/pyproject.toml | 1 + backend/tests/test_API.py | 12 ++++++++++++ 3 files changed, 16 insertions(+) create mode 100644 backend/tests/test_API.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b83ede..5bfb9bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,3 +16,6 @@ jobs: - name: check run: docker compose run --rm check + + - name: test + run: docker compose run --rm check poetry run pytest tests/ diff --git a/backend/pyproject.toml b/backend/pyproject.toml index fc8abdd..66e0de9 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -21,6 +21,7 @@ types-requests=">=2.32.0.20241016,<3.0.0.0" pylint=">=3.3.3,<4.0.0" pyflakes=">=3.2.0,<4.0.0" black=">=24.10.0,<25.0.0" +pytest = "^8.3.4" [tool.poetry.scripts] health_check = "maelstro.scripts.health_check:check" diff --git a/backend/tests/test_API.py b/backend/tests/test_API.py new file mode 100644 index 0000000..d3322e0 --- /dev/null +++ b/backend/tests/test_API.py @@ -0,0 +1,12 @@ +from fastapi.testclient import TestClient + +from maelstro.main import app + + +client = TestClient(app) + + +def test_read_main(): + response = client.get("/") + assert response.status_code == 200 + assert response.json() == {'Hello': 'World'} From e8236efc9e9938e2e0590f59e27334f761cefaca Mon Sep 17 00:00:00 2001 From: Moritz Kirmse <61978301+mki-c2c@users.noreply.github.com> Date: Tue, 21 Jan 2025 12:22:51 +0100 Subject: [PATCH 14/18] Apply suggestions from code review Co-authored-by: Arnaud Morvan --- backend/maelstro/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/maelstro/main.py b/backend/maelstro/main.py index 24c236e..0db3dca 100644 --- a/backend/maelstro/main.py +++ b/backend/maelstro/main.py @@ -54,8 +54,8 @@ def user_page( def debug_page(request: Request): """ Display details of query including headers. - This may be useful in development to check all the headers procided by the gateway. - This entrypoint should be deactivaed in prod. + This may be useful in development to check all the headers provided by the gateway. + This entrypoint should be deactivated in prod. """ return { **{ From 9e14ea065904cfef9aa7196b4cc89fd35589fb81 Mon Sep 17 00:00:00 2001 From: Moritz Kirmse Date: Tue, 21 Jan 2025 13:20:44 +0100 Subject: [PATCH 15/18] remove venv in poetry and strealmine docker image (review comments) --- backend/Dockerfile | 15 ++++++++++----- backend/maelstro/main.py | 12 ++++++++++++ backend/pyproject.toml | 6 +++--- docker-compose.yml | 4 +--- 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 1e3c21c..f4579f4 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,6 +1,8 @@ FROM python:3.12-slim-bullseye AS poetry -RUN pip install poetry +RUN --mount=type=cache,target=/root/.cache \ + pip install poetry +RUN poetry config virtualenvs.create false RUN mkdir /app WORKDIR /app @@ -8,16 +10,19 @@ WORKDIR /app COPY pyproject.toml /app FROM poetry as server -RUN poetry install --no-root +RUN --mount=type=cache,target=/root/.cache \ + poetry install --no-root COPY . /app -RUN poetry install +RUN --mount=type=cache,target=/root/.cache \ + poetry install -CMD ["poetry", "run", "fastapi", "dev", "--host", "0.0.0.0", "main.py"] +CMD ["serve_prod"] FROM server as check -RUN poetry install --no-root --with check +RUN --mount=type=cache,target=/root/.cache \ + poetry install --no-root --with check CMD ["/app/maelstro/scripts/code_check.sh"] diff --git a/backend/maelstro/main.py b/backend/maelstro/main.py index 0db3dca..4b18ae0 100644 --- a/backend/maelstro/main.py +++ b/backend/maelstro/main.py @@ -20,6 +20,16 @@ def root_page(): return {"Hello": "World"} +@app.get("/auth") +def auth_page( + sec_roles: Annotated[str, Header(include_in_schema=False)] = "", +): + """ + Check if the user is authorized for the maestro sync platform via the MAELSTRO role + """ + return {"is_authorized": "ROLE_MAELSTRO" in sec_roles.split(";")} + + @app.get("/user") def user_page( sec_username: Annotated[str | None, Header(include_in_schema=False)] = None, @@ -44,6 +54,8 @@ def user_page( } +# pylint: disable=fixme +# TODO: deactivate for prod @app.head("/debug") @app.get("/debug") @app.put("/debug") diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 66e0de9..fa86fa0 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -25,9 +25,9 @@ pytest = "^8.3.4" [tool.poetry.scripts] health_check = "maelstro.scripts.health_check:check" -dev = "maelstro:dev" -docker_dev = "maelstro:docker_dev" -prod = "maelstro:prod" +serve_dev = "maelstro:dev" +serve_docker_dev = "maelstro:docker_dev" +serve_prod = "maelstro:prod" [build-system] requires = ["poetry-core>=2.0.0,<3.0.0"] diff --git a/docker-compose.yml b/docker-compose.yml index d94ad2d..ac18f2d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,9 +18,7 @@ services: timeout: 10s command: - - poetry - - run - - docker_dev + - serve_docker_dev check: profiles: From ad26a607f7221257b71e51040fd25cd25b321ba3 Mon Sep 17 00:00:00 2001 From: Moritz Kirmse Date: Tue, 21 Jan 2025 14:26:23 +0100 Subject: [PATCH 16/18] move maelstrob to maelstro-bakckend --- README.md | 2 +- backend/maelstro/main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 05f6a65..0ca5df2 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ The folder [backend](backend) contains the API written with FastAPI. ### Access In the global dev composition, the backend is accessible via the https gateway: -https://georchestra-127-0-0-1.nip.io/maelstrob/ +https://georchestra-127-0-0-1.nip.io/maelstro-backend/ ### SwaggerUI diff --git a/backend/maelstro/main.py b/backend/maelstro/main.py index 4b18ae0..cd227b7 100644 --- a/backend/maelstro/main.py +++ b/backend/maelstro/main.py @@ -6,7 +6,7 @@ from fastapi import FastAPI, Request, Response, Header -app = FastAPI(root_path="/maelstrob") +app = FastAPI(root_path="/maelstro-backend") app.state.health_countdown = 5 From c53abc22421105edd6c54f4fd2889c6a2a7b4a27 Mon Sep 17 00:00:00 2001 From: Moritz Kirmse Date: Tue, 21 Jan 2025 14:37:26 +0100 Subject: [PATCH 17/18] remove auth check --- backend/maelstro/main.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/backend/maelstro/main.py b/backend/maelstro/main.py index cd227b7..468c35f 100644 --- a/backend/maelstro/main.py +++ b/backend/maelstro/main.py @@ -20,16 +20,6 @@ def root_page(): return {"Hello": "World"} -@app.get("/auth") -def auth_page( - sec_roles: Annotated[str, Header(include_in_schema=False)] = "", -): - """ - Check if the user is authorized for the maestro sync platform via the MAELSTRO role - """ - return {"is_authorized": "ROLE_MAELSTRO" in sec_roles.split(";")} - - @app.get("/user") def user_page( sec_username: Annotated[str | None, Header(include_in_schema=False)] = None, From ff7b4a606cde7f49835bc9a0b40667bf47f8bb7e Mon Sep 17 00:00:00 2001 From: Moritz Kirmse Date: Tue, 21 Jan 2025 14:42:37 +0100 Subject: [PATCH 18/18] add paths for mounted volumes to preserve access rights --- georchestra/resources/ssl/.gitignore | 2 ++ georchestra/resources/static/.gitignore | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 georchestra/resources/ssl/.gitignore create mode 100644 georchestra/resources/static/.gitignore diff --git a/georchestra/resources/ssl/.gitignore b/georchestra/resources/ssl/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/georchestra/resources/ssl/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/georchestra/resources/static/.gitignore b/georchestra/resources/static/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/georchestra/resources/static/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore