From 4446f9b9acb75de7eed198c275911a3740a854b7 Mon Sep 17 00:00:00 2001 From: Pierre Delpy Date: Tue, 15 Oct 2024 10:29:13 +0200 Subject: [PATCH 1/4] feat: use project overarching vars --- bbmri/docker-compose.yml | 2 +- bridgehead | 2 +- cce/docker-compose.yml | 2 +- ccp/docker-compose.yml | 2 +- ccp/modules/blaze-secondary-compose.yml | 5 +++-- ccp/vars | 2 -- dhki/docker-compose.yml | 4 ++-- dhki/vars | 2 -- itcc/docker-compose.yml | 2 +- kr/docker-compose.yml | 4 ++-- kr/modules/exporter.md | 15 --------------- kr/modules/teiler.md | 19 ------------------- lib/functions.sh | 6 +++--- lib/prepare-system.sh | 6 +++--- vars | 2 ++ 15 files changed, 20 insertions(+), 55 deletions(-) delete mode 100644 kr/modules/exporter.md delete mode 100644 kr/modules/teiler.md create mode 100644 vars diff --git a/bbmri/docker-compose.yml b/bbmri/docker-compose.yml index 000df01a..334b0487 100644 --- a/bbmri/docker-compose.yml +++ b/bbmri/docker-compose.yml @@ -4,7 +4,7 @@ version: "3.7" services: blaze: - image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + image: docker.verbis.dkfz.de/cache/samply/blaze:${BLAZE_TAG} container_name: bridgehead-bbmri-blaze environment: BASE_URL: "http://bridgehead-bbmri-blaze:8080" diff --git a/bridgehead b/bridgehead index d5d3a20d..b7cbd669 100755 --- a/bridgehead +++ b/bridgehead @@ -64,7 +64,7 @@ loadVars() { fetchVarsFromVaultByFile /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "Unable to fetchVarsFromVaultByFile" setHostname optimizeBlazeMemoryUsage - [ -e ./$PROJECT/vars ] && source ./$PROJECT/vars + [ -e ./$PROJECT/vars ] && source ./$PROJECT/vars ./vars set +a OVERRIDE=${OVERRIDE:=""} diff --git a/cce/docker-compose.yml b/cce/docker-compose.yml index 87b6b1c1..0ea5e56b 100644 --- a/cce/docker-compose.yml +++ b/cce/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.7" services: blaze: - image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + image: docker.verbis.dkfz.de/cache/samply/blaze:${BLAZE_TAG} container_name: bridgehead-cce-blaze environment: BASE_URL: "http://bridgehead-cce-blaze:8080" diff --git a/ccp/docker-compose.yml b/ccp/docker-compose.yml index fa1dc419..c2267d31 100644 --- a/ccp/docker-compose.yml +++ b/ccp/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.7" services: blaze: - image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + image: docker.verbis.dkfz.de/cache/samply/blaze:${BLAZE_TAG} container_name: bridgehead-ccp-blaze environment: BASE_URL: "http://bridgehead-ccp-blaze:8080" diff --git a/ccp/modules/blaze-secondary-compose.yml b/ccp/modules/blaze-secondary-compose.yml index b57bfbe2..ecd80ee6 100644 --- a/ccp/modules/blaze-secondary-compose.yml +++ b/ccp/modules/blaze-secondary-compose.yml @@ -2,13 +2,14 @@ version: "3.7" services: blaze-secondary: - image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + image: docker.verbis.dkfz.de/cache/samply/blaze:${BLAZE_TAG} container_name: bridgehead-ccp-blaze-secondary environment: BASE_URL: "http://bridgehead-ccp-blaze-secondary:8080" JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m" DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000} - DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP + DB_BLOCK_CACHE_SIZE: ${BLAZE_MEMORY_CAP} + CQL_EXPR_CACHE_SIZE: ${BLAZE_CQL_CACHE_CAP:-32} ENFORCE_REFERENTIAL_INTEGRITY: "false" volumes: - "blaze-secondary-data:/app/data" diff --git a/ccp/vars b/ccp/vars index 0900914b..c57f8c82 100644 --- a/ccp/vars +++ b/ccp/vars @@ -18,8 +18,6 @@ OIDC_URL="https://login.verbis.dkfz.de" OIDC_ISSUER_URL="${OIDC_URL}/realms/${OIDC_REALM}" OIDC_GROUP_CLAIM="groups" -POSTGRES_TAG=15.6-alpine - for module in $PROJECT/modules/*.sh do log DEBUG "sourcing $module" diff --git a/dhki/docker-compose.yml b/dhki/docker-compose.yml index ee8cd17d..677b9918 100644 --- a/dhki/docker-compose.yml +++ b/dhki/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.7" services: blaze: - image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + image: docker.verbis.dkfz.de/cache/samply/blaze:${BLAZE_TAG} container_name: bridgehead-dhki-blaze environment: BASE_URL: "http://bridgehead-dhki-blaze:8080" @@ -39,7 +39,7 @@ services: - "blaze" beam-proxy: - image: docker.verbis.dkfz.de/cache/samply/beam-proxy:develop + image: docker.verbis.dkfz.de/cache/samply/beam-proxy:${BEAM_TAG} container_name: bridgehead-beam-proxy environment: BROKER_URL: ${BROKER_URL} diff --git a/dhki/vars b/dhki/vars index b728925f..6662468c 100644 --- a/dhki/vars +++ b/dhki/vars @@ -8,8 +8,6 @@ PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem BROKER_URL_FOR_PREREQ=$BROKER_URL -POSTGRES_TAG=15.6-alpine - for module in ccp/modules/*.sh do log DEBUG "sourcing $module" diff --git a/itcc/docker-compose.yml b/itcc/docker-compose.yml index 7aab26d5..41831ea2 100644 --- a/itcc/docker-compose.yml +++ b/itcc/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.7" services: blaze: - image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + image: docker.verbis.dkfz.de/cache/samply/blaze:${BLAZE_TAG} container_name: bridgehead-itcc-blaze environment: BASE_URL: "http://bridgehead-itcc-blaze:8080" diff --git a/kr/docker-compose.yml b/kr/docker-compose.yml index 47a9db64..98632bde 100644 --- a/kr/docker-compose.yml +++ b/kr/docker-compose.yml @@ -6,7 +6,7 @@ services: replicas: 0 #deactivate landing page blaze: - image: docker.verbis.dkfz.de/cache/samply/blaze:0.28 + image: docker.verbis.dkfz.de/cache/samply/blaze:${BLAZE_TAG} container_name: bridgehead-kr-blaze environment: BASE_URL: "http://bridgehead-kr-blaze:8080" @@ -40,7 +40,7 @@ services: - "blaze" beam-proxy: - image: docker.verbis.dkfz.de/cache/samply/beam-proxy:develop + image: docker.verbis.dkfz.de/cache/samply/beam-proxy:${BEAM_TAG} container_name: bridgehead-beam-proxy environment: BROKER_URL: ${BROKER_URL} diff --git a/kr/modules/exporter.md b/kr/modules/exporter.md deleted file mode 100644 index 24e81b05..00000000 --- a/kr/modules/exporter.md +++ /dev/null @@ -1,15 +0,0 @@ -# Exporter and Reporter - - -## Exporter -The exporter is a REST API that exports the data of the different databases of the bridgehead in a set of tables. -It can accept different output formats as CSV, Excel, JSON or XML. It can also export data into Opal. - -## Exporter-DB -It is a database to save queries for its execution in the exporter. -The exporter manages also the different executions of the same query in through the database. - -## Reporter -This component is a plugin of the exporter that allows to create more complex Excel reports described in templates. -It is compatible with different template engines as Groovy, Thymeleaf,... -It is perfect to generate a document as our traditional CCP quality report. diff --git a/kr/modules/teiler.md b/kr/modules/teiler.md deleted file mode 100644 index 51e94e46..00000000 --- a/kr/modules/teiler.md +++ /dev/null @@ -1,19 +0,0 @@ -# Teiler -This module orchestrates the different microfrontends of the bridgehead as a single page application. - -## Teiler Orchestrator -Single SPA component that consists on the root HTML site of the single page application and a javascript code that -gets the information about the microfrontend calling the teiler backend and is responsible for registering them. With the -resulting mapping, it can initialize, mount and unmount the required microfrontends on the fly. - -The microfrontends run independently in different containers and can be based on different frameworks (Angular, Vue, React,...) -This microfrontends can run as single alone but need an extension with Single-SPA (https://single-spa.js.org/docs/ecosystem). -There are also available three templates (Angular, Vue, React) to be directly extended to be used directly in the teiler. - -## Teiler Dashboard -It consists on the main dashboard and a set of embedded services. -### Login -user and password in ccp.local.conf - -## Teiler Backend -In this component, the microfrontends are configured. diff --git a/lib/functions.sh b/lib/functions.sh index 3fcae384..a3ff8b5a 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -170,11 +170,11 @@ optimizeBlazeMemoryUsage() { available_system_memory_chunks=$((BLAZE_MEMORY_CAP / 1000)) if [ $available_system_memory_chunks -eq 0 ]; then log WARN "Only ${BLAZE_MEMORY_CAP} system memory available for Blaze. If your Blaze stores more than 128000 fhir ressources it will run significally slower." - export BLAZE_RESOURCE_CACHE_CAP=128000; - export BLAZE_CQL_CACHE_CAP=32; + export BLAZE_RESOURCE_CACHE_CAP=128000 + export BLAZE_CQL_CACHE_CAP=32 else export BLAZE_RESOURCE_CACHE_CAP=$((available_system_memory_chunks * 312500)) - export BLAZE_CQL_CACHE_CAP=$((($system_memory_in_mb/4)/16)); + export BLAZE_CQL_CACHE_CAP=$((($system_memory_in_mb/4)/16)) fi fi } diff --git a/lib/prepare-system.sh b/lib/prepare-system.sh index b6aba52b..111f49f9 100755 --- a/lib/prepare-system.sh +++ b/lib/prepare-system.sh @@ -57,9 +57,9 @@ case "$PROJECT" in ;; itcc) site_configuration_repository_middle="git.verbis.dkfz.de/itcc-sites/" - ;; - dhki) - site_configuration_repository_middle="git.verbis.dkfz.de/dhki/" + ;; + dhki) + site_configuration_repository_middle="git.verbis.dkfz.de/dhki/" ;; kr) site_configuration_repository_middle="git.verbis.dkfz.de/krebsregister-sites/" diff --git a/vars b/vars new file mode 100644 index 00000000..38fd2c0b --- /dev/null +++ b/vars @@ -0,0 +1,2 @@ +BLAZE_TAG=0.28 +POSTGRES_TAG=15.6-alpine \ No newline at end of file From 022d6e9a72ebbc2d5008b6bd98601ddda92e2d67 Mon Sep 17 00:00:00 2001 From: Pierre Delpy Date: Tue, 15 Oct 2024 10:44:14 +0200 Subject: [PATCH 2/4] to squash --- bbmri/vars | 1 + cce/vars | 1 + ccp/vars | 1 + dhki/vars | 1 + itcc/vars | 1 + kr/vars | 1 + lib/prerequisites.sh | 2 +- minimal/vars | 2 ++ 8 files changed, 9 insertions(+), 1 deletion(-) diff --git a/bbmri/vars b/bbmri/vars index 248fbee4..1c08f8e5 100644 --- a/bbmri/vars +++ b/bbmri/vars @@ -7,6 +7,7 @@ FOCUS_RETRY_COUNT=${FOCUS_RETRY_COUNT:-64} PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem +source vars for module in $PROJECT/modules/*.sh do log DEBUG "sourcing $module" diff --git a/cce/vars b/cce/vars index 7d0c1a3c..a5c850a6 100644 --- a/cce/vars +++ b/cce/vars @@ -7,6 +7,7 @@ SUPPORT_EMAIL=manoj.waikar@dkfz-heidelberg.de PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem BROKER_URL_FOR_PREREQ=$BROKER_URL +source vars for module in $PROJECT/modules/*.sh do log DEBUG "sourcing $module" diff --git a/ccp/vars b/ccp/vars index c57f8c82..66e886d1 100644 --- a/ccp/vars +++ b/ccp/vars @@ -18,6 +18,7 @@ OIDC_URL="https://login.verbis.dkfz.de" OIDC_ISSUER_URL="${OIDC_URL}/realms/${OIDC_REALM}" OIDC_GROUP_CLAIM="groups" +source vars for module in $PROJECT/modules/*.sh do log DEBUG "sourcing $module" diff --git a/dhki/vars b/dhki/vars index 6662468c..21c62fc9 100644 --- a/dhki/vars +++ b/dhki/vars @@ -8,6 +8,7 @@ PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem BROKER_URL_FOR_PREREQ=$BROKER_URL +source vars for module in ccp/modules/*.sh do log DEBUG "sourcing $module" diff --git a/itcc/vars b/itcc/vars index b03403b8..cec493fc 100644 --- a/itcc/vars +++ b/itcc/vars @@ -7,6 +7,7 @@ SUPPORT_EMAIL=arturo.macias@dkfz-heidelberg.de PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem BROKER_URL_FOR_PREREQ=$BROKER_URL +source vars for module in $PROJECT/modules/*.sh do log DEBUG "sourcing $module" diff --git a/kr/vars b/kr/vars index d4e5a27a..bbba5156 100644 --- a/kr/vars +++ b/kr/vars @@ -7,6 +7,7 @@ SUPPORT_EMAIL=arturo.macias@dkfz-heidelberg.de PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem BROKER_URL_FOR_PREREQ=$BROKER_URL +source vars for module in $PROJECT/modules/*.sh do log DEBUG "sourcing $module" diff --git a/lib/prerequisites.sh b/lib/prerequisites.sh index 2c1e186e..b37aac40 100755 --- a/lib/prerequisites.sh +++ b/lib/prerequisites.sh @@ -67,7 +67,7 @@ fi log INFO "Checking network access ($BROKER_URL_FOR_PREREQ) ..." source "${CONFIG_DIR}${PROJECT}".conf -source ${PROJECT}/vars +source ${PROJECT}/vars vars if [ "${PROJECT}" != "minimal" ]; then set +e diff --git a/minimal/vars b/minimal/vars index 11d07ff5..ee4701fd 100644 --- a/minimal/vars +++ b/minimal/vars @@ -1,3 +1,5 @@ + +source vars for module in $PROJECT/modules/*.sh do log DEBUG "sourcing $module" From 2b391a515614f82007e6dcdbf1132b1a58fca6e0 Mon Sep 17 00:00:00 2001 From: Pierre Delpy Date: Tue, 15 Oct 2024 11:54:26 +0200 Subject: [PATCH 3/4] minor fix --- bridgehead | 2 +- vars => versions | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename vars => versions (100%) diff --git a/bridgehead b/bridgehead index b7cbd669..465682d9 100755 --- a/bridgehead +++ b/bridgehead @@ -64,7 +64,7 @@ loadVars() { fetchVarsFromVaultByFile /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "Unable to fetchVarsFromVaultByFile" setHostname optimizeBlazeMemoryUsage - [ -e ./$PROJECT/vars ] && source ./$PROJECT/vars ./vars + [ -e ./$PROJECT/vars ] && source ./vars ./$PROJECT/vars set +a OVERRIDE=${OVERRIDE:=""} diff --git a/vars b/versions similarity index 100% rename from vars rename to versions From 5f5e62650123561c12b6dbe8b835e4380b2954be Mon Sep 17 00:00:00 2001 From: Pierre Delpy Date: Tue, 15 Oct 2024 11:54:26 +0200 Subject: [PATCH 4/4] minor fix --- bbmri/vars | 2 +- bridgehead | 2 +- cce/vars | 2 +- ccp/vars | 2 +- dhki/vars | 2 +- itcc/vars | 2 +- kr/vars | 2 +- minimal/vars | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bbmri/vars b/bbmri/vars index 1c08f8e5..69c6f2c9 100644 --- a/bbmri/vars +++ b/bbmri/vars @@ -7,7 +7,7 @@ FOCUS_RETRY_COUNT=${FOCUS_RETRY_COUNT:-64} PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem -source vars +source versions for module in $PROJECT/modules/*.sh do log DEBUG "sourcing $module" diff --git a/bridgehead b/bridgehead index 465682d9..dc2da220 100755 --- a/bridgehead +++ b/bridgehead @@ -64,7 +64,7 @@ loadVars() { fetchVarsFromVaultByFile /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "Unable to fetchVarsFromVaultByFile" setHostname optimizeBlazeMemoryUsage - [ -e ./$PROJECT/vars ] && source ./vars ./$PROJECT/vars + [ -e ./$PROJECT/vars ] && source ./versions ./$PROJECT/vars set +a OVERRIDE=${OVERRIDE:=""} diff --git a/cce/vars b/cce/vars index a5c850a6..53cd4224 100644 --- a/cce/vars +++ b/cce/vars @@ -7,7 +7,7 @@ SUPPORT_EMAIL=manoj.waikar@dkfz-heidelberg.de PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem BROKER_URL_FOR_PREREQ=$BROKER_URL -source vars +source versions for module in $PROJECT/modules/*.sh do log DEBUG "sourcing $module" diff --git a/ccp/vars b/ccp/vars index 66e886d1..1e5599df 100644 --- a/ccp/vars +++ b/ccp/vars @@ -18,7 +18,7 @@ OIDC_URL="https://login.verbis.dkfz.de" OIDC_ISSUER_URL="${OIDC_URL}/realms/${OIDC_REALM}" OIDC_GROUP_CLAIM="groups" -source vars +source versions for module in $PROJECT/modules/*.sh do log DEBUG "sourcing $module" diff --git a/dhki/vars b/dhki/vars index 21c62fc9..e9047ce7 100644 --- a/dhki/vars +++ b/dhki/vars @@ -8,7 +8,7 @@ PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem BROKER_URL_FOR_PREREQ=$BROKER_URL -source vars +source versions for module in ccp/modules/*.sh do log DEBUG "sourcing $module" diff --git a/itcc/vars b/itcc/vars index cec493fc..a7964ecb 100644 --- a/itcc/vars +++ b/itcc/vars @@ -7,7 +7,7 @@ SUPPORT_EMAIL=arturo.macias@dkfz-heidelberg.de PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem BROKER_URL_FOR_PREREQ=$BROKER_URL -source vars +source versions for module in $PROJECT/modules/*.sh do log DEBUG "sourcing $module" diff --git a/kr/vars b/kr/vars index bbba5156..687d0ce2 100644 --- a/kr/vars +++ b/kr/vars @@ -7,7 +7,7 @@ SUPPORT_EMAIL=arturo.macias@dkfz-heidelberg.de PRIVATEKEYFILENAME=/etc/bridgehead/pki/${SITE_ID}.priv.pem BROKER_URL_FOR_PREREQ=$BROKER_URL -source vars +source versions for module in $PROJECT/modules/*.sh do log DEBUG "sourcing $module" diff --git a/minimal/vars b/minimal/vars index ee4701fd..1b4f7c54 100644 --- a/minimal/vars +++ b/minimal/vars @@ -1,5 +1,5 @@ -source vars +source versions for module in $PROJECT/modules/*.sh do log DEBUG "sourcing $module"