Making QMR Dev Environments Not Publicly Accessible (#2036)

Co-authored-by: dwhite_stratiform <[email protected]> Co-authored-by: dwhite_stratiform <[email protected]> Co-authored-by: Berry Davenport <[email protected]>
Enterprise-CMCS · Jan 22, 2024 · 8d87bb0 · 8d87bb0
1 parent 2910023
commit 8d87bb0
Show file tree

Hide file tree

Showing 7 changed files with 310 additions and 37 deletions.
diff --git a/.github/build_vars.sh b/.github/build_vars.sh
@@ -13,12 +13,14 @@ set_value() {
   if [ ! -z "${!varname}" ]; then
     echo "Setting $varname"
     echo "${varname}=${!varname}" >> $GITHUB_ENV
+    echo "${varname}=${!varname}" >> $GITHUB_OUTPUT
   fi
 }
 
 set_name() {
   varname=${1}
   echo "BRANCH_SPECIFIC_VARNAME_$varname=${branch_name//-/_}_$varname" >> $GITHUB_ENV
+  echo "BRANCH_SPECIFIC_VARNAME_$varname=${branch_name//-/_}_$varname" >> $GITHUB_OUTPUT
 }
 
 action=${1}

diff --git a/.github/waf-controller.sh b/.github/waf-controller.sh
@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+
+CIRCUIT_BREAKER=10
+AWS_RETRY_ERROR=254
+AWS_THROTTLING_EXCEPTION=252
+#0, 1, 2 are the levels of debug, with 0 being off
+DEBUG=1
+
+set -o pipefail -o nounset -u
+
+case ${1} in
+  append)
+    OP=append
+    ;;
+  set)
+    OP=set
+    ;;
+  *)
+    echo "Error:  unkown operation"
+    echo "Usage: ${0} [append|set]" && exit 1
+    ;;
+esac
+
+shift
+NAME="${1}"
+ID="${2}"
+shift; shift
+RUNNER_CIDRS="${@}"
+
+[[ $DEBUG -ge 1 ]] && echo "Inputs:  NAME ${NAME}, ID ${ID}, RUNNER_CIDRS ${RUNNER_CIDRS}"
+
+#Exponential backoff with jitter
+jitter() {
+  #.5 seconds
+  SHORTEST=50
+  #10 seconds
+  LONGEST=1000
+  DIV=100
+  EXP=$(perl -e "use bigint; print $SHORTEST**$1")
+  MIN=$(($EXP>$LONGEST ? $LONGEST : $EXP))
+  RND=$(shuf -i$SHORTEST-$MIN -n1)
+  perl -e "print $RND/$DIV"
+}
+
+#Attempt to avoid resource contention from the start
+sleep $(jitter $(shuf -i1-10 -n1))
+
+for ((i=1; i <= $CIRCUIT_BREAKER; i++)); do
+  #This loop is ONLY for retrying if the retries exceeded exception is thrown
+  for ((j=1; j <= $CIRCUIT_BREAKER; j++)); do
+    #Read WAF configuration from AWS
+    WAF_CONFIG=$(aws wafv2 get-ip-set --scope CLOUDFRONT --id ${ID} --name ${NAME} 2>&1)
+    CMD_CD=$?
+    [[ $DEBUG -ge 1 ]] && echo "AWS CLI Read Response Code:  ${CMD_CD}"
+    [[ $DEBUG -ge 2 ]] && echo "AWS CLI Read Response:  ${WAF_CONFIG}"
+
+    #If the retries exceeded error code is returned, try again, otherwise exit the loop
+    [[ $CMD_CD -eq $AWS_RETRY_ERROR ]] || break
+
+    SLEEP_FOR=$(jitter ${j})
+    echo "CLI retries exceed.  Waiting for ${SLEEP_FOR} seconds to execute read again...$({j})"
+    sleep ${SLEEP_FOR}
+  done
+
+  #Unable to get the lock tocken and IP set so there isn't any point in attempting the write op
+  [[ $j -ge $CIRCUIT_BREAKER ]] && echo “Attempts to read WAF IPSet exceeded” && sleep $(jitter ${i}) && continue
+
+  #The loop was short circuited with an error code other than 0, so something is wrong
+  [[ $CMD_CD -eq 0 ]] || ( echo "An unexpected read error occurred:  ${CMD_CD}" && exit 2 )
+
+  echo "Read was successful."
+
+  if [ ${OP} == "append" ]; then
+    ##If this is used to whitelist individual ips or cidrs, using an additive approach is what is required
+    #Parse out IP set addresses to array
+    IP_ADDRESSES=($(jq -r '.IPSet.Addresses | .[]' <<< ${WAF_CONFIG}))
+
+    #If CIDR is already present in IP set, eject
+    grep -q $RUNNER_CIDRS <<< ${IP_ADDRESSES}
+    [[ $? -ne 0 ]] || ( echo "CIDR is present in IP Set." && exit 0 )
+
+    #Add runner CIDR to array
+    IP_ADDRESSES+=("$RUNNER_CIDRS")
+  else 
+    ##If this is used to hard set the IP set, just clobber it
+    IP_ADDRESSES=("$RUNNER_CIDRS")
+  fi
+
+  #Stringify IPs
+  STRINGIFIED=$(echo $(IFS=" " ; echo "${IP_ADDRESSES[*]}"))
+  [[ $DEBUG -ge 2 ]] && echo "Ip Addresses:  ${STRINGIFIED}"
+
+  #Parse out optimistic concurrency control token
+  OCC_TOKEN=$(jq -r '.LockToken' <<< ${WAF_CONFIG})
+  [[ $DEBUG -ge 2 ]] && echo "LockToken:  ${OCC_TOKEN}"
+
+  #This loop is ONLY for retrying if the retries exceeded exception is thrown
+  for ((k=1; k <= $CIRCUIT_BREAKER; k++)); do
+    #Write updated WAF configuration to AWS
+    OUTPUT=$(aws wafv2 update-ip-set --scope CLOUDFRONT --id ${ID} --name ${NAME} --lock-token ${OCC_TOKEN} --addresses ${STRINGIFIED} 2>&1)
+    CMD_CD=$?
+    [[ $DEBUG -ge 1 ]] && echo "AWS CLI Write Response Code:  ${CMD_CD}"
+    [[ $DEBUG -ge 2 ]] && echo "AWS CLI Write Response:  ${OUTPUT}"
+
+    #If the retries exceeded error code is returned, try again, otherwise exit the loop
+    [[ $CMD_CD -eq $AWS_RETRY_ERROR ]] || break
+    #If WAFOptimisticLockException error code is returned, exit the loop
+    [[ "$OUTPUT" =~ "WAFOptimisticLockException" ]] && break
+
+    SLEEP_FOR=$(jitter ${k})
+    echo "CLI retries exceed.  Waiting for ${SLEEP_FOR} seconds to execute write again...(${k})"
+    sleep ${SLEEP_FOR}
+  done
+
+  [[ $CMD_CD -ne 0 ]] || break
+  #Still not having success, so try again
+
+  echo "Exit Code:  ${CMD_CD}"
+
+  SLEEP_FOR=$(jitter ${i})
+  echo "Waiting for ${SLEEP_FOR} seconds to execute main loop again...(${i})"
+  sleep ${SLEEP_FOR}
+done
+
+[[ $DEBUG -ge 1 ]] && echo "Attempts to update ip set:  $i"
+
+[[ $i -gt $CIRCUIT_BREAKER ]] && echo “Attempts to update WAF IPSet exceeded, exiting.” && exit 2
+
+echo "Applied the IP Set successfully."
+
+#Things should not have made it this far without being able to successfully write the IP Set
+exit $CMD_CD
diff --git a/.github/workflows/cypress-workflow.yml b/.github/workflows/cypress-workflow.yml
@@ -46,7 +46,7 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: cypress/browsers:node16.16.0-chrome107-ff107
-      options: --user 1001
+      options: --user root
     needs: setup
     strategy:
       fail-fast: false

diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -38,6 +38,16 @@ jobs:
             echo "branch_name=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV
           fi
       - uses: actions/checkout@v3
+      - name: 'Setup jq'
+        uses: dcarbone/[email protected]
+        with:
+          version: '${{ inputs.version }}'
+          force: '${{ inputs.force }}'
+      - name: 'Check jq'
+        # language=sh
+        run: |
+          which jq
+          jq --version
       - name: Validate branch name
         run: ./.github/branchNameValidation.sh $STAGE_PREFIX$branch_name
       - name: set branch specific variable names
@@ -80,6 +90,8 @@ jobs:
         working-directory: services
     outputs:
       application_endpoint: ${{ steps.endpoint.outputs.application_endpoint}}
+      BRANCH_SPECIFIC_VARNAME_AWS_DEFAULT_REGION: ${{ steps.set_names.outputs.BRANCH_SPECIFIC_VARNAME_AWS_DEFAULT_REGION }}
+      BRANCH_SPECIFIC_VARNAME_AWS_OIDC_ROLE_TO_ASSUME: ${{ steps.set_names.outputs.BRANCH_SPECIFIC_VARNAME_AWS_OIDC_ROLE_TO_ASSUME }}
   # run e2e tests after deploy completes
   e2e-tests-init:
     name: Initialize End To End Tests
@@ -96,14 +108,19 @@ jobs:
       - uses: actions/checkout@v3
       - name: set branch_name
         run: |
-          if [[ "$GITHUB_REF" =~ ^refs/heads/dependabot/.* ]]; then # Dependabot builds very long branch names.  This is a switch to make it shorter.
-            echo "branch_name=`echo ${GITHUB_REF#refs/heads/} | md5sum | head -c 10 | sed 's/^/x/'`" >> $GITHUB_ENV
+          echo "GITHUB_REF=${GITHUB_REF}"        
+          if [[ "$GITHUB_REF" =~ ^refs/heads/dependabot/.* ]]; then
+            echo "branch_name=`echo ${GITHUB_REF##*/*-} | md5sum | head -c 10 | sed 's/^/x/'`" >> $GITHUB_ENV
+          elif [[ "$GITHUB_REF" =~ ^refs/.*/snyk-* ]]; then
+            echo "branch_name=`echo ${GITHUB_REF##*/*-} | head -c 10 | sed 's/^/s/'`" >> $GITHUB_ENV
           else
             echo "branch_name=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV
           fi
       - name: set branch specific variable names
+        id: set_names
         run: ./.github/build_vars.sh set_names
       - name: set variable values
+        id: set_values
         run: ./.github/build_vars.sh set_values
         env:
           AWS_DEFAULT_REGION: ${{ secrets[env.BRANCH_SPECIFIC_VARNAME_AWS_DEFAULT_REGION] || secrets.AWS_DEFAULT_REGION }}
@@ -139,13 +156,63 @@ jobs:
       - name: set path
         run: |
           echo "PATH=$(pwd)/node_modules/.bin/:$PATH" >> $GITHUB_ENV
+      - name: Get Runner IP
+        id: get-ip
+        run: |
+          #!/bin/bash
+          # Get the IP address of the runner
+          IP_ADDRESS=$(curl https://api.ipify.org)
+          echo "Runner IP address: $IP_ADDRESS"
+          # Store the IP address as an output variable
+          echo "RUNNER_IP=$IP_ADDRESS/32" >> $GITHUB_OUTPUT
+      - name: Get Github Actions CIDR Blocks
+        id: get-gha-cidrs
+        shell: bash
+        run: |
+          #!/bin/bash
+          GHA_RESP=$(curl --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' https://api.github.com/meta)
+          echo "Response for GHA runner CIDR blocks:  $GHA_RESP"
+          IPV4_CIDR_ARR=($(echo $GHA_RESP | jq -r '.actions | .[]' | grep -v ':'))
+          GHA_CIDRS_IPV4=$(echo $(IFS=" "; echo ${IPV4_CIDR_ARR[*]}))
+          echo "GHA_CIDRS_IPV4=$GHA_CIDRS_IPV4" >> $GITHUB_OUTPUT
+      - name: Generate IP Set Name
+        id: gen-ip-set-name
+        run: |
+          STAGE_GH_IPSET_NAME=$STAGE_PREFIX$branch_name-gh-ipset
+          echo "Github IP Set name:  $STAGE_GH_IPSET_NAME"
+          echo "STAGE_GH_IPSET_NAME=$STAGE_GH_IPSET_NAME" >> $GITHUB_OUTPUT  
+      - name: Fetch AWS IP set Metadata
+        id: fetch-ip-set-info
+        run: |
+          #!/bin/bash
+          # Fetch AWS IP set ARNs using AWS CLI and store them in a variable
+          AWS_IP_SET_INFO=$(aws wafv2 list-ip-sets --scope=CLOUDFRONT)
+          # Store the IP set ARNs in an output variable using GITHUB_OUTPUT
+          IPSET_NAME=${{ steps.gen-ip-set-name.outputs.STAGE_GH_IPSET_NAME }}
+          IPSET=$(jq '.IPSets | map(select(.Name == "'${IPSET_NAME}'")) | .[]' <<< ${AWS_IP_SET_INFO})
+          [ -z "$IPSET" ] && echo "IP Set with name ${IPSET_NAME} was not located.  Exiting..." && exit 1
+          echo "IP Set metadata:  ${IPSET}"
+          #Get Values from the IP SET
+          IPSET_ID=$(jq -r '.Id' <<< ${IPSET})
+          echo "IPSET_ARN=$IPSET_ARN" >> $GITHUB_OUTPUT
+          echo "IPSET_NAME=$IPSET_NAME" >> $GITHUB_OUTPUT
+          echo "IPSET_ID=$IPSET_ID" >> $GITHUB_OUTPUT
+      - name: Update IP Set
+        id: update-ip-set
+        run: ./.github/waf-controller.sh set ${{ steps.fetch-ip-set-info.outputs.IPSET_NAME }} ${{ steps.fetch-ip-set-info.outputs.IPSET_ID }} ${{ steps.get-gha-cidrs.outputs.GHA_CIDRS_IPV4 }}
+        env:
+          AWS_RETRY_MODE: adaptive
+          AWS_MAX_ATTEMPTS: 10
     outputs:
       application_endpoint: ${{ needs.deploy.outputs.application_endpoint }}
+      ipset_name: ${{ steps.fetch-ip-set-info.outputs.IPSET_NAME }}
+      ipset_id: ${{ steps.fetch-ip-set-info.outputs.IPSET_ID }}      
 
   setup-tests:
     name: "Setup End To End Tests"
     uses: ./.github/workflows/cypress-workflow.yml
-    needs: e2e-tests-init
+    needs: 
+      - e2e-tests-init
     with:
       test-path: "init"
       test-endpoint: "${{ needs.e2e-tests-init.outputs.application_endpoint }}"
@@ -244,3 +311,31 @@ jobs:
       cypress-user3: ${{ secrets.CYPRESS_TEST_USER_3 }}
       cypress-user4: ${{ secrets.CYPRESS_TEST_USER_4 }}
       cypress-password: ${{ secrets.CYPRESS_TEST_PASSWORD_1 }}
+
+  cleanup:
+    name: Deslist GHA Runner CIDR Blocks
+    if: ${{ github.ref != 'refs/heads/master' && github.ref != 'refs/heads/val' && github.ref != 'refs/heads/prod' }}  
+    runs-on: ubuntu-latest
+    needs:
+      - e2e-tests-init
+      - e2e-feature-tests
+      - child-e2e-measure-tests
+      - adult-e2e-measure-tests
+      - health-home-e2e-measure-tests
+      - deploy
+      - a11y-tests
+    env:
+      SLS_DEPRECATION_DISABLE: "*" # Turn off deprecation warnings in the pipeline
+    steps:
+      - uses: actions/checkout@v3
+      - name: Configure AWS credentials for GitHub Actions
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets[env.BRANCH_SPECIFIC_VARNAME_AWS_OIDC_ROLE_TO_ASSUME] || secrets.AWS_OIDC_ROLE_TO_ASSUME }}
+          aws-region: ${{ secrets[env.BRANCH_SPECIFIC_VARNAME_AWS_DEFAULT_REGION] || secrets.AWS_DEFAULT_REGION }}
+      - name: clean-up-iplist
+        id: reset-ip-set
+        run: ./.github/waf-controller.sh set ${{ needs.e2e-tests-init.outputs.ipset_name }} ${{ needs.e2e-tests-init.outputs.ipset_id }} '[]'
+        env:
+          AWS_RETRY_MODE: adaptive
+          AWS_MAX_ATTEMPTS: 10
diff --git a/services/ui/package.json b/services/ui/package.json
@@ -10,5 +10,8 @@
   "license": "CC0-1.0",
   "devDependencies": {
     "serverless-s3-bucket-helper": "Enterprise-CMCS/serverless-s3-bucket-helper#0.1.1"
+  },
+  "dependencies": {
+    "@enterprise-cmcs/serverless-waf-plugin": "^1.3.1"
   }
 }