From 5b147cead5fc304d103b61023a320f85c7bff545 Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Thu, 18 Jan 2024 17:57:15 +0200 Subject: [PATCH 1/9] tests: Add a note about sharing empty_deployment --- conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conftest.py b/conftest.py index 961e7906..d77ad1f9 100644 --- a/conftest.py +++ b/conftest.py @@ -50,6 +50,8 @@ params=["empty_deployment"] if os.environ.get("KCIDB_DEPLOYMENT", "") == "This deployment is empty" else [], + # Only clean up after all tests have run, to reduce testing time + # Tests are supposed to avoid interfering with each other scope="session" ) def empty_deployment(): From e3f89297534bc5bb0a5ff1f4e4560557070df976 Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Fri, 19 Jan 2024 15:41:14 +0200 Subject: [PATCH 2/9] cloud: Add a missing declare in psql_databases_deploy() --- kcidb/cloud/psql.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/kcidb/cloud/psql.sh b/kcidb/cloud/psql.sh index 1896dc4f..b2f5d363 100644 --- a/kcidb/cloud/psql.sh +++ b/kcidb/cloud/psql.sh @@ -291,6 +291,7 @@ function psql_databases_deploy() { declare database declare editor declare init + declare exists while (($#)); do database="$1"; shift From bb0f4aa772e5ffb35349e3e62ab9d8d83d90261c Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Fri, 19 Jan 2024 13:58:38 +0200 Subject: [PATCH 3/9] cloud: Switch to password_secret_ namespace --- cloud | 8 ++++---- kcidb/cloud/password.sh | 12 ++++++------ kcidb/cloud/secrets.sh | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/cloud b/cloud index a7bf4f6e..fde72e1e 100755 --- a/cloud +++ b/cloud @@ -169,9 +169,9 @@ function execute_command() { declare -a -r psql_args # Enable fetching PostgreSQL passwords from their secrets - password_set_secret "psql_superuser" "$project" "kcidb_psql_superuser" - password_set_secret "psql_viewer" "$project" "kcidb_psql_viewer" - password_set_secret "psql_editor" "$project" "${prefix}psql_editor" + password_secret_set "psql_superuser" "$project" "kcidb_psql_superuser" + password_secret_set "psql_viewer" "$project" "kcidb_psql_viewer" + password_secret_set "psql_editor" "$project" "${prefix}psql_editor" declare -r bigquery_dataset="${prefix}${version}" declare -r bigquery_clean_test_dataset="${prefix}${version}_clean_test" @@ -237,7 +237,7 @@ function execute_command() { fi # Register SMTP password secret - password_set_secret "smtp" "$project" "$SECRETS_SMTP_PASSWORD" + password_secret_set "smtp" "$project" "$SECRETS_SMTP_PASSWORD" declare -r smtp_topic=$("$smtp_mocked" && echo "${prefix}smtp" || true) declare -r smtp_subscription=$( diff --git a/kcidb/cloud/password.sh b/kcidb/cloud/password.sh index 563014c6..e479a674 100644 --- a/kcidb/cloud/password.sh +++ b/kcidb/cloud/password.sh @@ -148,7 +148,7 @@ function password_set_file() { # specified name. The password will be retrieved from the secret, if it wasn't # cached, and if its source file wasn't specified. # Args: name project secret -function password_set_secret() { +function password_secret_set() { declare -r name="$1"; shift declare -r project="$1"; shift declare -r secret="$1"; shift @@ -195,10 +195,10 @@ function password_is_specified() { } # Deploy passwords to their secrets (assuming they're set with -# "password_set_secret"). For every password deploy only if the password is +# "password_secret_set"). For every password deploy only if the password is # specified, or the secret doesn't exist. # Args: name... -function password_deploy_secret() { +function password_secret_deploy() { declare name declare project declare secret @@ -226,9 +226,9 @@ function password_deploy_secret() { } # Withdraw passwords from their secrets (assuming they're set with -# "password_set_secret"). +# "password_secret_set"). # Args: name... -function password_withdraw_secret() { +function password_secret_withdraw() { declare name declare project declare secret @@ -252,7 +252,7 @@ function password_withdraw_secret() { # Deploy only if one of the passwords is specified, or if the pgpass secret # doesn't exist. # Args: project pgpass_secret [password_name user_name]... -function password_deploy_pgpass_secret() { +function password_secret_deploy_pgpass() { declare -r project="$1"; shift declare -r pgpass_secret="$1"; shift declare -a -r password_and_user_names=("$@") diff --git a/kcidb/cloud/secrets.sh b/kcidb/cloud/secrets.sh index ab9173eb..a1799db2 100644 --- a/kcidb/cloud/secrets.sh +++ b/kcidb/cloud/secrets.sh @@ -19,7 +19,7 @@ function secrets_deploy() { declare exists # Make sure the shared SMTP password secret is deployed - password_deploy_secret smtp + password_secret_deploy smtp # Give Cloud Functions access to the shared SMTP password secret mute gcloud secrets add-iam-policy-binding \ --quiet --project="$project" "$SECRETS_SMTP_PASSWORD" \ @@ -27,11 +27,11 @@ function secrets_deploy() { --member "serviceAccount:$project@appspot.gserviceaccount.com" # Make sure all PostgreSQL's password secrets are deployed - password_deploy_secret psql_superuser psql_editor psql_viewer + password_secret_deploy psql_superuser psql_editor psql_viewer # DO NOT give Cloud Functions access to *any* PostgreSQL password secrets # Make sure PostgreSQL's .pgpass secret is deployed - password_deploy_pgpass_secret "$project" "$psql_pgpass_secret" \ + password_secret_deploy_pgpass "$project" "$psql_pgpass_secret" \ psql_editor "$psql_editor_username" # Give Cloud Functions access to the .pgpass secret @@ -47,7 +47,7 @@ function secrets_deploy() { function secrets_withdraw() { declare -r project="$1"; shift declare -r psql_pgpass_secret="$1"; shift - password_withdraw_secret psql_editor + password_secret_withdraw psql_editor secret_withdraw "$project" "$psql_pgpass_secret" # NOTE: Not withdrawing the shared secrets } From 61d9d9b9cf4b78b1da8a7a6c70f1ed907aee192e Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Fri, 19 Jan 2024 14:32:52 +0200 Subject: [PATCH 4/9] cloud: Add assert() --- kcidb/cloud/misc.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/kcidb/cloud/misc.sh b/kcidb/cloud/misc.sh index 4ed70824..8a52815e 100644 --- a/kcidb/cloud/misc.sh +++ b/kcidb/cloud/misc.sh @@ -14,6 +14,32 @@ export TMPDIR=$(mktemp -d -t "kcidb_cloud.XXXXXXXXXX") # Remove the directory with all the temporary files on exit atexit_push "rm -Rf ${TMPDIR@Q}" +# Evaluate and execute a command string, +# exit shell with error message and status 1 if unsuccessfull. +# Args: [eval_arg]... +function assert() +{ + # Use private global-style variable names + # to avoid clashes with "evaled" names + declare _ASSERT_ATTRS + declare _ASSERT_STATUS + + # Prevent shell from exiting due to `set -e` if the command fails + read -rd '' _ASSERT_ATTRS < <(set +o) || [ $? == 1 ] + set +o errexit + ( + eval "$_ASSERT_ATTRS" + eval "$@" + ) + _ASSERT_STATUS=$? + eval "$_ASSERT_ATTRS" + + if [ "$_ASSERT_STATUS" != 0 ]; then + echo "Assertion failed: $*" >&1 + exit 1 + fi +} + # Generate code declaring parameter variables with names and values passed # through long-option command-line argument list, and assigning the positional # arguments. From e19d32093f1def9218394cf2c688ed270889a263 Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Fri, 19 Jan 2024 14:51:35 +0200 Subject: [PATCH 5/9] cloud: Add and use password_exists() --- kcidb/cloud/password.sh | 59 ++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 36 deletions(-) diff --git a/kcidb/cloud/password.sh b/kcidb/cloud/password.sh index e479a674..54fc62bf 100644 --- a/kcidb/cloud/password.sh +++ b/kcidb/cloud/password.sh @@ -4,6 +4,7 @@ if [ -z "${_PASSWORD_SH+set}" ]; then declare _PASSWORD_SH= . secret.sh +. misc.sh # A map of password names and their descriptions declare -r -A PASSWORD_DESCS=( @@ -31,15 +32,25 @@ declare -A PASSWORD_FILES=() # A map of password names and their strings declare -A PASSWORD_STRINGS=() +# Check that every specified password exists. +# Args: name... +function password_exists() { + declare name + while (($#)); do + name="$1"; shift + if ! [[ -v PASSWORD_DESCS[$name] ]]; then + return 1 + fi + done + return 0 +} + # Ask the user to input a password with specified name. # Args: name # Output: The retrieved password function password_input() { declare -r name="$1"; shift - if ! [[ -v PASSWORD_DESCS[$name] ]]; then - echo "Unknown password name ${name@Q}" >&2 - exit 1 - fi + assert password_exists "$name" declare password read -p "Enter ${PASSWORD_DESCS[$name]:-a} password: " -r -s password echo "" >&2 @@ -53,10 +64,7 @@ function password_input() { # Output: The retrieved password function password_get() { declare -r name="$1"; shift - if ! [[ -v PASSWORD_DESCS[$name] ]]; then - echo "Unknown password name ${name@Q}" >&2 - exit 1 - fi + assert password_exists "$name" declare password declare -r password_file="${PASSWORD_FILES[$name]:-}" @@ -112,10 +120,7 @@ function password_get_pgpass() { while (($#)); do name="$1"; shift - if ! [[ -v PASSWORD_DESCS[$name] ]]; then - echo "Unknown password name ${name@Q}" >&2 - exit 1 - fi + assert password_exists "$name" username="$1"; shift # Cache the password in the current shell @@ -136,10 +141,7 @@ function password_get_pgpass() { # Args: name file function password_set_file() { declare -r name="$1"; shift - if ! [[ -v PASSWORD_DESCS[$name] ]]; then - echo "Unknown password name ${name@Q}" >&2 - exit 1 - fi + assert password_exists "$name" declare -r file="$1"; shift PASSWORD_FILES[$name]="$file" } @@ -152,10 +154,7 @@ function password_secret_set() { declare -r name="$1"; shift declare -r project="$1"; shift declare -r secret="$1"; shift - if ! [[ -v PASSWORD_DESCS[$name] ]]; then - echo "Unknown password name ${name@Q}" >&2 - exit 1 - fi + assert password_exists "$name" if [[ "$project" = *:* ]]; then echo "Invalid project name ${project@Q}" >&2 exit 1 @@ -169,10 +168,7 @@ function password_secret_set() { function password_set_generate() { declare -r name="$1"; shift declare -r generate="$1"; shift - if ! [[ -v PASSWORD_DESCS[$name] ]]; then - echo "Unknown password name ${name@Q}" >&2 - exit 1 - fi + assert password_exists "$name" PASSWORD_GENERATE[$name]="$generate" } @@ -181,12 +177,9 @@ function password_set_generate() { # Args: name... function password_is_specified() { declare name + assert password_exists "$@" while (($#)); do name="$1"; shift - if ! [[ -v PASSWORD_DESCS[$name] ]]; then - echo "Unknown password name ${name@Q}" >&2 - exit 1 - fi if ! [[ -v PASSWORD_FILES[$name] ]]; then return 1 fi @@ -203,12 +196,9 @@ function password_secret_deploy() { declare project declare secret declare exists + assert password_exists "$@" while (($#)); do name="$1"; shift - if ! [[ -v PASSWORD_DESCS[$name] ]]; then - echo "Unknown password name ${name@Q}" >&2 - exit 1 - fi if ! [[ -v PASSWORD_SECRETS[$name] ]]; then echo "Password ${name@Q} has no secret specified" >&2 exit 1 @@ -232,12 +222,9 @@ function password_secret_withdraw() { declare name declare project declare secret + assert password_exists "$@" while (($#)); do name="$1"; shift - if ! [[ -v PASSWORD_DESCS[$name] ]]; then - echo "Unknown password name ${name@Q}" >&2 - exit 1 - fi if ! [[ -v PASSWORD_SECRETS[$name] ]]; then echo "Password ${name@Q} has no secret specified" >&2 exit 1 From a0859f1a26183bc950ab26c03e54a4d7e6f24f72 Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Fri, 19 Jan 2024 15:24:20 +0200 Subject: [PATCH 6/9] cloud: Add and use password_secret_is_specified() --- kcidb/cloud/password.sh | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/kcidb/cloud/password.sh b/kcidb/cloud/password.sh index 54fc62bf..d69d24de 100644 --- a/kcidb/cloud/password.sh +++ b/kcidb/cloud/password.sh @@ -162,6 +162,21 @@ function password_secret_set() { PASSWORD_SECRETS[$name]="$project:$secret" } +# Check if every specified password has its secret set +# Assumes every specified password is known/exists. +# Args: name... +function password_secret_is_specified() { + declare name + assert password_exists "$@" + while (($#)); do + name="$1"; shift + if ! [[ -v PASSWORD_SECRETS[$name] ]]; then + return 1 + fi + done + return 0 +} + # Specify the single-word command returning exit status specifying if the # password with specified name could be auto-generated or not. # Args: name generate @@ -197,12 +212,9 @@ function password_secret_deploy() { declare secret declare exists assert password_exists "$@" + assert password_secret_is_specified "$@" while (($#)); do name="$1"; shift - if ! [[ -v PASSWORD_SECRETS[$name] ]]; then - echo "Password ${name@Q} has no secret specified" >&2 - exit 1 - fi project="${PASSWORD_SECRETS[$name]%%:*}" secret="${PASSWORD_SECRETS[$name]#*:}" exists=$(secret_exists "$project" "$secret") From 73016502b4bd654a1a3fe8944c2993f29b27186c Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Fri, 19 Jan 2024 15:25:32 +0200 Subject: [PATCH 7/9] cloud: Add password_secret_exists() --- kcidb/cloud/password.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/kcidb/cloud/password.sh b/kcidb/cloud/password.sh index d69d24de..0daa2eb6 100644 --- a/kcidb/cloud/password.sh +++ b/kcidb/cloud/password.sh @@ -177,6 +177,30 @@ function password_secret_is_specified() { return 0 } +# Check if every specified password's secret exists. +# Assumes every specified password is known/exists and has its secret set. +# Args: name... +# Output: "true" if all secrets exists, "false" otherwise. +function password_secret_exists() { + declare name + declare project + declare secret + declare exists + assert password_exists "$@" + assert password_secret_is_specified "$@" + while (($#)); do + name="$1"; shift + project="${PASSWORD_SECRETS[$name]%%:*}" + secret="${PASSWORD_SECRETS[$name]#*:}" + exists=$(secret_exists "$project" "$secret") + if ! "$exists"; then + echo false + return + fi + done + echo true +} + # Specify the single-word command returning exit status specifying if the # password with specified name could be auto-generated or not. # Args: name generate From b3591febae2ffe3a3aae9807badbd1e839de0e78 Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Fri, 19 Jan 2024 15:26:38 +0200 Subject: [PATCH 8/9] cloud: Add password_is_updated() --- kcidb/cloud/password.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/kcidb/cloud/password.sh b/kcidb/cloud/password.sh index 0daa2eb6..2f66b10e 100644 --- a/kcidb/cloud/password.sh +++ b/kcidb/cloud/password.sh @@ -226,6 +226,32 @@ function password_is_specified() { return 0 } +# Check if any of the passwords with specified names are (going to be) updated +# (specified on the command line, or missing). +# Args: name... +# Output: "true" if all secrets exists, "false" otherwise. +function password_is_updated() { + declare name + declare secret_exists + assert password_exists "$@" + while (($#)); do + name="$1"; shift + if password_is_specified "$name"; then + echo true + return + fi + if password_secret_is_specified "$name"; then + secret_exists="$(password_secret_exists "$name")" + if "$secret_exists"; then + continue + fi + fi + echo true + return + done + echo false +} + # Deploy passwords to their secrets (assuming they're set with # "password_secret_set"). For every password deploy only if the password is # specified, or the secret doesn't exist. From 41d96dd9c4c394dd8554aff527551cf19a81d4ef Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Fri, 19 Jan 2024 15:43:28 +0200 Subject: [PATCH 9/9] cloud: Switch to password_is_updated() Use password_is_updated() instead of password_is_specified() to update PostgreSQL user passwords whenever they change. --- kcidb/cloud/password.sh | 12 ++++++------ kcidb/cloud/psql.sh | 8 ++++++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/kcidb/cloud/password.sh b/kcidb/cloud/password.sh index 2f66b10e..d2791ee8 100644 --- a/kcidb/cloud/password.sh +++ b/kcidb/cloud/password.sh @@ -254,24 +254,24 @@ function password_is_updated() { # Deploy passwords to their secrets (assuming they're set with # "password_secret_set"). For every password deploy only if the password is -# specified, or the secret doesn't exist. +# updated, or the secret doesn't exist. # Args: name... function password_secret_deploy() { declare name + declare updated declare project declare secret - declare exists assert password_exists "$@" assert password_secret_is_specified "$@" while (($#)); do name="$1"; shift - project="${PASSWORD_SECRETS[$name]%%:*}" - secret="${PASSWORD_SECRETS[$name]#*:}" - exists=$(secret_exists "$project" "$secret") - if ! "$exists" || password_is_specified "$name"; then + updated=$(password_is_updated "$name") + if "$updated"; then # Get and cache the password in the current shell first password_get "$name" > /dev/null # Deploy the cached password + project="${PASSWORD_SECRETS[$name]%%:*}" + secret="${PASSWORD_SECRETS[$name]#*:}" password_get "$name" | secret_deploy "$project" "$secret" fi done diff --git a/kcidb/cloud/psql.sh b/kcidb/cloud/psql.sh index b2f5d363..3d911782 100644 --- a/kcidb/cloud/psql.sh +++ b/kcidb/cloud/psql.sh @@ -75,6 +75,7 @@ function psql_instance_deploy() { declare -r name="$1"; shift declare -r viewer="$1"; shift declare exists + declare updated exists=$(psql_instance_exists "$project" "$name") if ! "$exists"; then @@ -97,7 +98,8 @@ function psql_instance_deploy() { # Deploy the shared viewer user exists=$(psql_user_exists "$project" "$name" "$viewer") - if ! "$exists" || password_is_specified psql_viewer; then + updated=$(password_is_updated psql_viewer) + if ! "$exists" || "$updated"; then # Get and cache the password in the current shell first password_get psql_viewer >/dev/null # Create the user with the cached password @@ -292,6 +294,7 @@ function psql_databases_deploy() { declare editor declare init declare exists + declare updated while (($#)); do database="$1"; shift @@ -317,7 +320,8 @@ function psql_databases_deploy() { # Deploy the per-database editor user exists=$(psql_user_exists "$project" "$instance" "$editor") - if ! "$exists" || password_is_specified psql_editor; then + updated=$(password_is_updated psql_editor) + if ! "$exists" || "$updated"; then # Get and cache the password in the current shell first password_get psql_editor >/dev/null # Create the user with the cached password