feat: create scripts and documentation for updating the SSL certs (#200)

* chore: production deployment (#193) * feat: create github action to trigger swithcover countdown (#179) * feat: create github action to trigger swithcover countdown * chore: remove todo comment * chore: fix action bug on preemptive failover (#180) * feat: create github action to trigger swithcover countdown * chore: remove todo comment * chore: remove null step from action * chore: add instructions to the preemptive failover scheduler * chore: create check for switchover internet access (#181) * chore: add a external internet check to the switchover agent * chore: add message to successful check * feat: refactor values ands prep quarkus deployment (#182) * feat: refactor values ands prep quarkus deployment * chore: fix TLS and image tag * chore: remove the edge change * chore: modify pod counts in sand dev * chore: create the test sandbox config * chore: update the prod sandbox config * chore: override logging add override for sandbox logging * chore: deps add dep for docker build * fix: string change int val to string * feat: update production config on feature branch (#185) * chore: update c6-dev helm for load test * chore: set the default maintenance page in the values file * chore: update values for dev production * chore: update values for test and prod production * chore: fix min availability for dev * chore: remove redundance config from sand prod helm * chore: reduce the dev production PVC size * chore: add config to prod helm values file * chore: bump c6 deb pvc to 8Gi * chore: reduce the cpu request on all sandbox deployments * chore: bump the cpu request in prod (#186) * chore: bump the cpu request in prod * chore: revert c6 config * chore: modify the liveness and readyness probes * Ssoteam 1711 4 (#187) * chore: bump the cpu request in prod * chore: update production configs * chore: reduce limit in dev and prod * chore: fix typos and revert image pull policy (#188) * chore: bump the cpu request in prod * chore: update production configs * chore: reduce limit in dev and prod * chore: implement final fixes and image version * chore: remove keycloak binary --------- Co-authored-by: jonathan langlois <[email protected]> Co-authored-by: Jonathan Langlois <[email protected]> * chore: fix typo and update PVC size in dev (#189) * feat: update sso image to 24.0.6-build.3 (#190) * chore: reduce prod log retention (#191) * chore: reduce old sand dev resources (#192) --------- Co-authored-by: jonathan langlois <[email protected]> Co-authored-by: Jonathan Langlois <[email protected]> Co-authored-by: Nithin Shekar Kuruba <[email protected]> * chore: working script fro ssl cert change * feat: create scripts to generate a new route from renewed ssl certs * chore: remove dockertest file library againg * chore: update the readme * chore: remove check ocp cluster from check * chore: merge secret and update scripts update readme * chore: remove comment --------- Co-authored-by: jonathan langlois <[email protected]> Co-authored-by: Jonathan Langlois <[email protected]> Co-authored-by: Nithin Shekar Kuruba <[email protected]>
bcgov · Oct 10, 2024 · 5a42121 · 5a42121
1 parent 08909fd
commit 5a42121
Show file tree

Hide file tree

Showing 6 changed files with 262 additions and 0 deletions.
diff --git a/ssl-renewal/.gitignore b/ssl-renewal/.gitignore
@@ -0,0 +1,2 @@
+ssltemplate.yaml
+.env/*
diff --git a/ssl-renewal/Readme.md b/ssl-renewal/Readme.md
@@ -0,0 +1,55 @@
+# The SSL Cert renewal process.
+
+## Steps
+ - Create the myservice request for a new cert
+ - Generate the cert singing request in the `./env/<<namepace>>/` folder. Submit to myservice and wait for certificates to be issued.
+ - Run the bash script `update_route_credentials.sh` to upgrade the certs.
+
+## Create the iStore Request for a new cert
+
+You will need to complete the request form: https://ssbc-client.gov.bc.ca/services/isr_forms/hosting_ssl_site_reverse.docx
+
+The form is submitted through the iStore by attaching it to a new ticket. https://imbsd.gov.bc.ca/.  You will need to be connected to the VPN to access this site.
+
+## Generate the cert singing request in the `./env/<<namepace>>/` folder.
+
+<!--  TODO CONVERT THIS CREATION TO A BASH SCRIPT -->
+There will be five secrets uploaded to the namespace:
+
+```
+L1K-for-certs.txt
+L1K-root-for-certs-G2.txt
+loginproxy.csr
+loginproxy.key
+loginproxy.txt
+```
+
+They can be stored locally in the untracked folders `./env/<<namepace>>/`.  Once the certs are generated, they can can be used by the `update_route_credentials.sh` script to update the Route objects.
+
+In certain cases a single cert is used for multiple environments.  In that case a copy of the secrets must be saved in each environment's namespace folder.
+
+
+## Run the script to update the cert values
+
+### GoldDR
+In GoldDR, run:
+
+`./update_route_credentials.sh <<namespace>> golddr year`
+
+This script will create a secret with name `loginproxy-ssl-cert-secret.<<year>>` in the GoldDR namespace.  It will error out if a secret with that name already exists.
+
+The certificate may not update imediately.  To check that the cert expiry date has updated, run the folowing script:
+
+`./check_endpoint_health.sh <<namespace>> golddr`
+
+The last line of output will be the date the certificate expires on.  It should be approximately 1 year in the future.
+
+### Gold
+
+Once the GoldDR route is confimed up and healthy, log into the Gold cluster and run:
+
+`./update_route_credentials.sh <<namespace>> gold year`
+
+Check that the Gold cert change worked by navigating to the browser. Or by running:
+
+`./check_endpoint_health.sh <<namespace>> gold`
diff --git a/ssl-renewal/check_endpoint_health.sh b/ssl-renewal/check_endpoint_health.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+set -e
+
+usage() {
+    cat <<EOF
+This checks the status of the keycloak app for a given namespace and cluster.
+It is the same check the GSLB uses to establish the applications health.
+It also provides the expiration date for the SSL cert on the route.
+
+Usages:
+    $0 <namespace> <cluster>
+
+Available namespaces:
+    - e4ca1d-dev
+    - e4ca1d-test
+    - e4ca1d-prod
+    - eb75ad-dev
+    - eb75ad-test
+    - eb75ad-prod
+Available Clusters
+    - gold
+    - golddr
+
+
+
+Examples:
+    $ $0 e4ca1d-dev gold
+EOF
+}
+
+if [ "$#" -lt 2 ]; then
+    usage
+    exit 1
+fi
+
+namespace=$1
+cluster=$2
+
+pwd="$(dirname "$0")"
+source "$pwd/../transition-scripts/helpers/_all.sh"
+
+
+KEYCLOAK_URL=$(get_vanity_url "$namespace")
+
+# Get the cluster ip address.
+if [ "$cluster" = "golddr" ]; then
+    IP="142.34.64.4"
+elif [ "$cluster" = "gold" ]; then
+    IP="142.34.229.4"
+else
+    echo "Cluster must be gold or golddr"
+    exit 1
+fi
+
+curl -Iv --resolve "$KEYCLOAK_URL":443:"$IP" -H '"Host:'"$KEYCLOAK_URL"'"' \
+https://"$KEYCLOAK_URL"/auth/realms/master/.well-known/openid-configuration
+
+
+curl -Iv --resolve "$KEYCLOAK_URL":443:"$IP" -H '"Host:'"$KEYCLOAK_URL"'"' \
+https://"$KEYCLOAK_URL"/auth/realms/master/.well-known/openid-configuration  \
+--stderr - | grep "expire date"
diff --git a/ssl-renewal/csr_details_oidc.txt b/ssl-renewal/csr_details_oidc.txt
@@ -0,0 +1,20 @@
+# Run the following command to (re)generate a CSR
+# openssl req -new -sha256 -nodes \
+# -out ./.env/<<NAMESPACE>>/loginproxy.csr -newkey rsa:2048 \
+# -keyout ./.env/<<NAMESPACE>>/loginproxy.key \
+# -config <(cat csr_details_oidc.txt)
+
+[req]
+default_bits=2048
+prompt=no
+default_md=sha256
+distinguished_name=dn
+
+[dn]
+C=CA
+ST=British Columbia
+L=Victoria
+O=Government of the Province of British Columbia
+OU=Citizens' Services
+CN=<<ENV>>.loginproxy.gov.bc.ca
+emailAddress=<<EMAIL_ADDRESS>>@gov.bc.ca
diff --git a/ssl-renewal/update_route_credentials.sh b/ssl-renewal/update_route_credentials.sh
@@ -0,0 +1,96 @@
+#!/bin/bash
+set -e
+
+usage() {
+    cat <<EOF
+This script loads the ssl certificates from openshift and patches the keycloak vanity
+route. Updating the credentials without impacting the endusers.
+
+Usages:
+    $0 <namespace> <cluster> <year>
+
+Available namespaces:
+    - e4ca1d-dev
+    - e4ca1d-test
+    - e4ca1d-prod
+    - eb75ad-dev
+    - eb75ad-test
+    - eb75ad-prod
+Available Clusters
+    - gold
+    - golddr
+Available years
+    - YYYY
+
+
+Examples:
+    $ $0 e4ca1d-dev gold 2004
+EOF
+}
+
+if [ "$#" -lt 3 ]; then
+    usage
+    exit 1
+fi
+
+namespace=$1
+cluster=$2
+year=$3
+
+pwd="$(dirname "$0")"
+source "$pwd/../transition-scripts/helpers/_all.sh"
+
+check_ocp_cluster "$cluster"
+
+
+KEYCLOAK_ROUTE=$(get_vanity_route_name "$namespace")
+KEYCLOAK_URL=$(get_vanity_url "$namespace")
+BACKUP_FILE='.env/'"$namespace"'/route-backup-'"$namespace"'-'"$cluster"'-'"$year"'.yaml'
+
+# Get the cluster ip address.
+if [ "$cluster" = "golddr" ]; then
+    IP="142.34.64.4"
+elif [ "$cluster" = "gold" ]; then
+    IP="142.34.229.4"
+else
+    echo "Cluster must be gold or golddr"
+    exit 1
+fi
+
+check_ssl_cert_expiration() {
+    curl -Iv --resolve "$KEYCLOAK_URL":443:"$IP" -H '"Host:'"$KEYCLOAK_URL"'"' \
+    https://"$KEYCLOAK_URL"/auth/realms/master/.well-known/openid-configuration  \
+    --stderr - | grep "expire date"
+}
+
+# Create a secret in openshift for the current year (will error out if the secret already exists)
+oc -n "$namespace" create secret generic loginproxy-ssl-cert-secret."$year" \
+ --from-file=private-key=.env/"$namespace"/loginproxy.key \
+ --from-file=certificate=.env/"$namespace"/loginproxy.txt \
+ --from-file=csr=.env/"$namespace"/loginproxy.csr \
+ --from-file=ca-chain-certificate=.env/"$namespace"/L1K-for-certs.txt \
+ --from-file=ca-root-certifcate=.env/"$namespace"/L1K-root-for-certs-G2.txt
+
+
+# Get the current certificate expiration date
+check_ssl_cert_expiration
+
+# Retrieve and decode the secrets from openshift cluster.
+
+keyEncoded=$(kubectl -n "$namespace" get secret loginproxy-ssl-cert-secret."$year" -o jsonpath='{.data.private-key}' )
+key=$(echo -e "$keyEncoded" | base64 --decode | sed ':a;N;$!ba;s/\n/\\n/g')
+
+certificateEncoded=$(kubectl -n "$namespace" get secret loginproxy-ssl-cert-secret."$year" -o jsonpath='{.data.certificate}')
+certificate=$(echo -e "$certificateEncoded" | base64 --decode | sed ':a;N;$!ba;s/\n/\\n/g')
+
+caCertificateEncoded=$(kubectl -n "$namespace" get secret loginproxy-ssl-cert-secret."$year" -o jsonpath='{.data.ca-chain-certificate}')
+caCertificate=$(echo -e "$caCertificateEncoded" | base64 --decode | sed ':a;N;$!ba;s/\n/\\n/g')
+
+# Creatre a backup of the old route in case something goes wrong, append to file on multiple runs
+kubectl -n "$namespace" get route "$KEYCLOAK_ROUTE" -o yaml >> "$BACKUP_FILE"
+
+kubectl -n "$namespace" patch route "$KEYCLOAK_ROUTE" -p '{"spec":{"tls":{"certificate":"'"$certificate"'", "key":"'"$key"'", "caCertificate":"'"$caCertificate"'" }}}'
+
+
+# Get the new certificate expiration date
+check_ssl_cert_expiration
diff --git a/transition-scripts/helpers/helm.sh b/transition-scripts/helpers/helm.sh
@@ -155,6 +155,34 @@ get_vanity_route_name() {
 
 }
 
+get_vanity_url() {
+  if [ "$#" -lt 1 ]; then exit 1; fi
+
+  namespace="$1"
+  if [ "$namespace" = "e4ca1d-dev" ]
+  then
+    KEYCLOAK_URL="dev.sandbox.loginproxy.gov.bc.ca"
+  elif [ "$namespace" = "e4ca1d-test" ]
+  then
+    KEYCLOAK_URL="test.sandbox.loginproxy.gov.bc.ca"
+  elif [ "$namespace" = "e4ca1d-prod" ]
+  then
+    KEYCLOAK_URL="sandbox.loginproxy.gov.bc.ca"
+  elif [ "$namespace" = "eb75ad-dev" ]
+  then
+    KEYCLOAK_URL="dev.loginproxy.gov.bc.ca"
+  elif [ "$namespace" = "eb75ad-test" ]
+  then
+    KEYCLOAK_URL="test.loginproxy.gov.bc.ca"
+  elif [ "$namespace" = "eb75ad-prod" ]
+  then
+    KEYCLOAK_URL="loginproxy.gov.bc.ca"
+  fi
+
+  echo $KEYCLOAK_URL
+
+}
+
 connect_route_to_correct_service() {
   if [ "$#" -lt 2 ]; then exit 1; fi