chore: Remove secure repo and fetch secrets from AWS for analytics je…

…nkins jobs
edx · Oct 23, 2023 · d8b8ef8 · d8b8ef8
1 parent 9d897c6
commit d8b8ef8
Show file tree

Hide file tree

Showing 6 changed files with 127 additions and 23 deletions.
diff --git a/dataeng/jobs/analytics/RetirementJobs.groovy b/dataeng/jobs/analytics/RetirementJobs.groovy
@@ -69,12 +69,11 @@ class RetirementJobs{
             multiscm {
                 git {
                     remote {
-                        url('[email protected]:edx-ops/user-retirement-secure.git')
-                        // using credentials from credentialsBinding.GITHUB_USER_PASS_COMBO
+                        url('https://github.com/edx/configuration.git')
                     }
                     branch('master')
                     extensions {
-                        relativeTargetDirectory('user-retirement-secure')
+                        relativeTargetDirectory('configuration')
                         cloneOptions {
                             shallow()
                             timeout(10)
@@ -174,12 +173,11 @@ class RetirementJobs{
             multiscm {
                 git {
                     remote {
-                        url('[email protected]:edx-ops/user-retirement-secure.git')
-                        // using credentials from credentialsBinding.GITHUB_USER_PASS_COMBO
+                        url('https://github.com/edx/configuration.git')
                     }
                     branch('master')
                     extensions {
-                        relativeTargetDirectory('user-retirement-secure')
+                        relativeTargetDirectory('configuration')
                         cloneOptions {
                             shallow()
                             timeout(10)
@@ -324,12 +322,11 @@ class RetirementJobs{
             multiscm {
                 git {
                     remote {
-                        url('[email protected]:edx-ops/user-retirement-secure.git')
-                        // using credentials from credentialsBinding.GITHUB_USER_PASS_COMBO
+                        url('https://github.com/edx/configuration.git')
                     }
                     branch('master')
                     extensions {
-                        relativeTargetDirectory('user-retirement-secure')
+                        relativeTargetDirectory('configuration')
                         cloneOptions {
                             shallow()
                             timeout(10)
@@ -444,12 +441,11 @@ class RetirementJobs{
             multiscm {
                 git {
                     remote {
-                        url('[email protected]:edx-ops/user-retirement-secure.git')
-                        // using credentials from credentialsBinding.GITHUB_USER_PASS_COMBO
+                        url('https://github.com/edx/configuration.git')
                     }
                     branch('master')
                     extensions {
-                        relativeTargetDirectory('user-retirement-secure')
+                        relativeTargetDirectory('configuration')
                         cloneOptions {
                             shallow()
                             timeout(10)
@@ -545,12 +541,11 @@ class RetirementJobs{
             multiscm {
                 git {
                     remote {
-                        url('[email protected]:edx-ops/user-retirement-secure.git')
-                        // using credentials from credentialsBinding.GITHUB_USER_PASS_COMBO
+                        url('https://github.com/edx/configuration.git')
                     }
                     branch('master')
                     extensions {
-                        relativeTargetDirectory('user-retirement-secure')
+                        relativeTargetDirectory('configuration')
                         cloneOptions {
                             shallow()
                             timeout(10)

diff --git a/dataeng/resources/retirement-partner-report-cleanup.sh b/dataeng/resources/retirement-partner-report-cleanup.sh
@@ -9,6 +9,29 @@ VENV="venv-${BUILD_NUMBER}"
 virtualenv --python=python3.8 --clear "${VENV}"
 source "${VENV}/bin/activate"
 
+#Fetch secrets from AWS
+cd $WORKSPACE/configuration
+pip install -r util/jenkins/requirements.txt
+# hide the sensitive information in the logs
+set +x
+
+CONFIG_YAML=$(aws secretsmanager get-secret-value --secret-id "user-retirement-secure/$ENVIRONMENT" --region "us-east-1" --output json | jq -r '.SecretString' | yq -y .)
+
+# Create a temporary file to store the YAML
+TEMP_CONFIG_YAML=$(mktemp $WORKSPACE/tempfile.XXXXXXXXXX.yml)
+
+# Write the YAML data to the temporary file
+echo "$CONFIG_YAML" > "$TEMP_CONFIG_YAML"
+
+# Fetch google-service-account secrets
+GOOGLE_SERVICE_ACCOUNT_JSON=$(aws secretsmanager get-secret-value --secret-id "user-retirement-secure/google-service-accounts/service-account-$ENVIRONMENT.json" --region "us-east-1" --output json )
+# Create a temporary file to store the YAML
+TEMP_GOOGLE_SECRETS=$(mktemp $WORKSPACE/tempfile.XXXXXXXXXX.json)
+
+# Write the YAML data to the temporary file
+echo "$GOOGLE_SERVICE_ACCOUNT_JSON" > "$TEMP_GOOGLE_SECRETS"
+
+set -x
 
 # prepare tubular
 cd $WORKSPACE/tubular
@@ -18,6 +41,10 @@ pip install -r requirements.txt
 
 # Call the script to cleanup the reports
 python scripts/delete_expired_partner_gdpr_reports.py \
-    --config_file=$WORKSPACE/user-retirement-secure/$ENVIRONMENT.yml \
-    --google_secrets_file=$WORKSPACE/user-retirement-secure/google-service-accounts/service-account-$ENVIRONMENT.json \
+    --config_file=$TEMP_CONFIG_YAML \
+    --google_secrets_file=$TEMP_GOOGLE_SECRETS \
     --age_in_days=$AGE_IN_DAYS
+
+# Remove the temporary files after processing
+rm -f "$TEMP_CONFIG_YAML"
+rm -f "$TEMP_GOOGLE_SECRETS"
diff --git a/dataeng/resources/retirement-partner-reporter.sh b/dataeng/resources/retirement-partner-reporter.sh
@@ -21,6 +21,29 @@ source "${VENV}/bin/activate"
 export PYTHONIOENCODING=UTF-8
 export LC_CTYPE=en_US.UTF-8
 
+#Fetch secrets from AWS
+cd $WORKSPACE/configuration
+pip install -r util/jenkins/requirements.txt
+# hide the sensitive information in the logs
+set +x
+
+CONFIG_YAML=$(aws secretsmanager get-secret-value --secret-id "user-retirement-secure/$ENVIRONMENT" --region "us-east-1" --output json | jq -r '.SecretString' | yq -y .)
+
+# Create a temporary file to store the YAML
+TEMP_CONFIG_YAML=$(mktemp $WORKSPACE/tempfile.XXXXXXXXXX.yml)
+
+# Write the YAML data to the temporary file
+echo "$CONFIG_YAML" > "$TEMP_CONFIG_YAML"
+
+# Fetch google-service-account secrets
+GOOGLE_SERVICE_ACCOUNT_JSON=$(aws secretsmanager get-secret-value --secret-id "user-retirement-secure/google-service-accounts/service-account-$ENVIRONMENT.json" --region "us-east-1" --output json )
+# Create a temporary file to store the YAML
+TEMP_GOOGLE_SECRETS=$(mktemp $WORKSPACE/tempfile.XXXXXXXXXX.json)
+
+# Write the YAML data to the temporary file
+echo "$GOOGLE_SERVICE_ACCOUNT_JSON" > "$TEMP_GOOGLE_SECRETS"
+
+set -x
 
 # prepare tubular
 cd $WORKSPACE/tubular
@@ -34,6 +57,10 @@ mkdir $PARTNER_REPORTS_DIR
 
 # Call the script to generate the reports and upload them to Google Drive
 python scripts/retirement_partner_report.py \
-    --config_file=$WORKSPACE/user-retirement-secure/$ENVIRONMENT.yml \
-    --google_secrets_file=$WORKSPACE/user-retirement-secure/google-service-accounts/service-account-$ENVIRONMENT.json \
+    --config_file=$TEMP_CONFIG_YAML \
+    --google_secrets_file=$TEMP_GOOGLE_SECRETS \
     --output_dir=$PARTNER_REPORTS_DIR
+
+# Remove the temporary files after processing
+rm -f "$TEMP_CONFIG_YAML"
+rm -f "$TEMP_GOOGLE_SECRETS"
diff --git a/dataeng/resources/user-retirement-bulk-status.sh b/dataeng/resources/user-retirement-bulk-status.sh
@@ -21,6 +21,21 @@ source "${VENV}/bin/activate"
 export PYTHONIOENCODING=UTF-8
 export LC_CTYPE=en_US.UTF-8
 
+#Fetch secrets from AWS
+cd $WORKSPACE/configuration
+pip install -r util/jenkins/requirements.txt
+# hide the sensitive information in the logs
+set +x
+
+CONFIG_YAML=$(aws secretsmanager get-secret-value --secret-id "user-retirement-secure/$ENVIRONMENT" --region "us-east-1" --output json | jq -r '.SecretString' | yq -y .)
+
+# Create a temporary file to store the YAML
+TEMP_CONFIG_YAML=$(mktemp $WORKSPACE/tempfile.XXXXXXXXXX.yml)
+
+# Write the YAML data to the temporary file
+echo "$CONFIG_YAML" > "$TEMP_CONFIG_YAML"
+
+set -x
 
 # prepare tubular
 cd $WORKSPACE/tubular
@@ -30,9 +45,12 @@ pip install -r requirements.txt
 
 # Call the script to collect the list of learners that are to be retired.
 python scripts/retirement_bulk_status_update.py \
-    --config_file=$WORKSPACE/user-retirement-secure/$ENVIRONMENT.yml \
+    --config_file=$TEMP_CONFIG_YAML \
     --start_date=$START_DATE \
     --end_date=$END_DATE \
     --initial_state=$INITIAL_STATE_NAME \
     ${NEW_STATE_NAME:+ "--new_state=$NEW_STATE_NAME"} \
     $(if [[ $REWIND_STATE == "true" ]]; then echo --rewind-state; fi)
+
+# Remove the temporary file after processing
+rm -f "$TEMP_CONFIG_YAML"
diff --git a/dataeng/resources/user-retirement-collector.sh b/dataeng/resources/user-retirement-collector.sh
@@ -21,6 +21,21 @@ source "${VENV}/bin/activate"
 export PYTHONIOENCODING=UTF-8
 export LC_CTYPE=en_US.UTF-8
 
+#Fetch secrets from AWS
+cd $WORKSPACE/configuration
+pip install -r util/jenkins/requirements.txt
+# hide the sensitive information in the logs
+set +x
+
+CONFIG_YAML=$(aws secretsmanager get-secret-value --secret-id "user-retirement-secure/$ENVIRONMENT" --region "us-east-1" --output json | jq -r '.SecretString' | yq -y .)
+
+# Create a temporary file to store the YAML
+TEMP_CONFIG_YAML=$(mktemp $WORKSPACE/tempfile.XXXXXXXXXX.yml)
+
+# Write the YAML data to the temporary file
+echo "$CONFIG_YAML" > "$TEMP_CONFIG_YAML"
+
+set -x
 
 # prepare tubular
 cd $WORKSPACE/tubular
@@ -35,8 +50,11 @@ mkdir $LEARNERS_TO_RETIRE_PROPERTIES_DIR
 
 # Call the script to collect the list of learners that are to be retired.
 python scripts/get_learners_to_retire.py \
-    --config_file=$WORKSPACE/user-retirement-secure/$ENVIRONMENT.yml \
+    --config_file=$TEMP_CONFIG_YAML \
     --output_dir=$LEARNERS_TO_RETIRE_PROPERTIES_DIR \
     --cool_off_days=$COOL_OFF_DAYS \
     --user_count_error_threshold=$USER_COUNT_ERROR_THRESHOLD \
-    --max_user_batch_size=$MAX_USER_BATCH_SIZE
+    --max_user_batch_size=$MAX_USER_BATCH_SIZE
+
+# Remove the temporary file after processing
+rm -f "$TEMP_CONFIG_YAML"
diff --git a/dataeng/resources/user-retirement-driver.sh b/dataeng/resources/user-retirement-driver.sh
@@ -21,6 +21,22 @@ source "${VENV}/bin/activate"
 export PYTHONIOENCODING=UTF-8
 export LC_CTYPE=en_US.UTF-8
 
+#Fetch secrets from AWS
+cd $WORKSPACE/configuration
+pip install -r util/jenkins/requirements.txt
+# hide the sensitive information in the logs
+set +x
+
+CONFIG_YAML=$(aws secretsmanager get-secret-value --secret-id "user-retirement-secure/$ENVIRONMENT" --region "us-east-1" --output json | jq -r '.SecretString' | yq -y .)
+
+# Create a temporary file to store the YAML
+TEMP_CONFIG_YAML=$(mktemp $WORKSPACE/tempfile.XXXXXXXXXX.yml)
+
+# Write the YAML data to the temporary file
+echo "$CONFIG_YAML" > "$TEMP_CONFIG_YAML"
+
+set -x
+
 # prepare tubular
 cd $WORKSPACE/tubular
 # snapshot the current latest versions of pip and setuptools.
@@ -30,4 +46,7 @@ pip install -r requirements.txt
 # Call the script to retire one learner.  This assumes the following build
 # parameters / environment variable is set: RETIREMENT_USERNAME.
 python scripts/retire_one_learner.py \
-    --config_file=$WORKSPACE/user-retirement-secure/$ENVIRONMENT.yml
+    --config_file=$TEMP_CONFIG_YAML
+
+# Remove the temporary file after processing
+rm -f "$TEMP_CONFIG_YAML"