Generalize scripts to upload a model parameter repo to a new DB.

database_scripts/setup_local_db.sh

@@ -55,3 +55,16 @@ if ! $CMD exec $CONTAINER_NAME mongosh admin -u root -p example --eval "db.runCo
   $CMD logs $CONTAINER_NAME
   exit 1
 fi
+
+echo "Create admin user"
+$CMD exec -it $CONTAINER_NAME mongosh admin -u root -p example --eval "
+db.createUser({
+  user: 'api',
+  pwd: 'password',
+  roles: [
+    { role: 'readWriteAnyDatabase', db: 'admin' },
+    { role: 'dbAdminAnyDatabase', db: 'admin' },
+    { role: 'userAdminAnyDatabase', db: 'admin' },
+  ]
+});
+"

database_scripts/upload_from_model_repository_to_db.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Upload model parameter from repository to a local or remote mongoDB.
+#
+
+SIMTOOLS_DB_SIMULATION_MODEL_URL="https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/model_parameters.git"
+SIMTOOLS_DB_SIMULATION_MODEL_BRANCH="main"
+
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 <DB simulation model name>"
+    echo ""
+    echo "Uses the .env file in the simtools directory for database configuration."
+    exit 1
+fi
+
+SIMTOOLS_DB_SIMULATION_MODEL="$1"
+
+echo "Cloning model parameters from $SIMTOOLS_DB_SIMULATION_MODEL_URL"
+rm -rf ./tmp_model_parameters
+git clone --depth=1 -b $SIMTOOLS_DB_SIMULATION_MODEL_BRANCH $SIMTOOLS_DB_SIMULATION_MODEL_URL ./tmp_model_parameters
+
+CURRENTDIR=$(pwd)
+cd ./tmp_model_parameters/ || exit
+cp -f "$CURRENTDIR"/../.env .env
+
+# upload files to DB
+model_directory="./model_versions/"
+for dir in "${model_directory}"*/; do
+  model_version=$(basename "${dir}")
+  if [ "$model_version" = "metadata" ]; then
+    simtools-db-add-model-parameters-from-repository-to-db \
+    --input_path "${dir}"/ \
+    --db_name "$SIMTOOLS_DB_SIMULATION_MODEL" \
+    --type "metadata"
+  else
+    simtools-db-add-model-parameters-from-repository-to-db \
+    --model_version "${model-version}" \
+    --input_path "${dir}" \
+    --db_name "$SIMTOOLS_DB_SIMULATION_MODEL" \
+    --type "model_parameters"
+  fi
+done
+
+cd "$CURRENTDIR" || exit
+
+rm -rf ./tmp_model_parameters

docs/source/user-guide/databases.md

@@ -73,8 +73,8 @@ Best to define a new database when adding new values or files.
 
 The following applications are important:
 
-* update or define a single model parameter from a json file (as defined in the model parameter repository): [add_value_from_json_to_db.py](add_value_from_json_to_db)
-* upload a model parameter file: [add_file_to_db.py](add_file_to_db)
+* update or define a single model parameter from a json file (as defined in the model parameter repository): [db_add_value_from_json_to_db.py](db_add_value_from_json_to_db)
+* upload a model parameter file: [db_add_file_to_db.py](db_add_file_to_db)
 * upload all model parameters and files from the model parameter repository: [db_add_model_parameters_from_repository_to_db.py](db_add_model_parameters_from_repository_to_db)
 
 ## Configure and use a local copy of the model parameter database
@@ -109,10 +109,10 @@ Note that database names are hardcoded in the scripts and need to be adjusted ac
 
 #### Option 2: Fill local database from model parameter repository
 
-The script `upload_from_model_repository_to_local_db.sh` uses the [model parameter repository](https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/model_parameters) from the CTAO gitlab and
+The script `upload_from_model_repository_to_db.sh` uses the [model parameter repository](https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/model_parameters) from the CTAO gitlab and
 uploads its contents to the local database instance.
 
-Note that database names and repository branches are hardcoded in the scripts and need to be adjusted accordingly.
+Note that repository branches are hardcoded in the scripts and need to be adjusted accordingly.
 
 ### Purge the local database instance and all networks, images, containers
 
@@ -146,3 +146,16 @@ For using simtools inside a container:
 ```bash
 podman run --rm -it -v "$(pwd)/:/workdir/external" --network simtools-mongo-network ghcr.io/gammasim/simtools-dev:latest bash
 ```
+
+For completeness, here the full `.env` file to be used with a container:
+
+```console
+# Environmental variables
+SIMTOOLS_DB_API_PORT=27017 #Port on the MongoDB server
+SIMTOOLS_DB_SERVER='simtools-mongodb'
+SIMTOOLS_DB_API_USER='api' # username for MongoDB
+SIMTOOLS_DB_API_PW='password' # Password for MongoDB
+SIMTOOLS_DB_API_AUTHENTICATION_DATABASE='admin'
+SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-Model-LATEST'
+SIMTOOLS_SIMTEL_PATH='/workdir/sim_telarray'
+```

simtools/applications/db_add_model_parameters_from_repository_to_db.py

@@ -72,22 +72,24 @@ def _parse(label=None, description=None):
     )
     config.parser.add_argument(
         "--db_name",
-        help="Name of the new DB to be created.",
+        help="Name of the new model parameter database to be created.",
         type=str,
         required=True,
     )
     config.parser.add_argument(
         "--type",
-        help="Type of data to be uploaded to the DB.",
+        help="Type of data to be uploaded to the database.",
         type=str,
         required=False,
         default="model_parameters",
         choices=["model_parameters", "metadata"],
     )
 
-    return config.initialize(
+    args_dict, db_config = config.initialize(
         output=True, require_command_line=True, db_config=True, simulation_model="version"
     )
+    db_config["db_simulation_model"] = args_dict["db_name"]  # overwrite explicitly DB configuration
+    return args_dict, db_config
 
 
 def add_values_from_json_to_db(file, collection, db, db_name, file_prefix, logger):