diff --git a/.github/build_vars.sh b/.github/build_vars.sh index 1b5c603cba..380179a9cb 100755 --- a/.github/build_vars.sh +++ b/.github/build_vars.sh @@ -13,12 +13,14 @@ set_value() { if [ ! -z "${!varname}" ]; then echo "Setting $varname" echo "${varname}=${!varname}" >> $GITHUB_ENV + echo "${varname}=${!varname}" >> $GITHUB_OUTPUT fi } set_name() { varname=${1} echo "BRANCH_SPECIFIC_VARNAME_$varname=${branch_name//-/_}_$varname" >> $GITHUB_ENV + echo "BRANCH_SPECIFIC_VARNAME_$varname=${branch_name//-/_}_$varname" >> $GITHUB_OUTPUT } action=${1} diff --git a/.github/waf-controller.sh b/.github/waf-controller.sh new file mode 100755 index 0000000000..6f57206b0b --- /dev/null +++ b/.github/waf-controller.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash + +CIRCUIT_BREAKER=10 +AWS_RETRY_ERROR=254 +AWS_THROTTLING_EXCEPTION=252 +#0, 1, 2 are the levels of debug, with 0 being off +DEBUG=1 + +set -o pipefail -o nounset -u + +case ${1} in + append) + OP=append + ;; + set) + OP=set + ;; + *) + echo "Error: unkown operation" + echo "Usage: ${0} [append|set]" && exit 1 + ;; +esac + +shift +NAME="${1}" +ID="${2}" +shift; shift +RUNNER_CIDRS="${@}" + +[[ $DEBUG -ge 1 ]] && echo "Inputs: NAME ${NAME}, ID ${ID}, RUNNER_CIDRS ${RUNNER_CIDRS}" + +#Exponential backoff with jitter +jitter() { + #.5 seconds + SHORTEST=50 + #10 seconds + LONGEST=1000 + DIV=100 + EXP=$(perl -e "use bigint; print $SHORTEST**$1") + MIN=$(($EXP>$LONGEST ? $LONGEST : $EXP)) + RND=$(shuf -i$SHORTEST-$MIN -n1) + perl -e "print $RND/$DIV" +} + +#Attempt to avoid resource contention from the start +sleep $(jitter $(shuf -i1-10 -n1)) + +for ((i=1; i <= $CIRCUIT_BREAKER; i++)); do + #This loop is ONLY for retrying if the retries exceeded exception is thrown + for ((j=1; j <= $CIRCUIT_BREAKER; j++)); do + #Read WAF configuration from AWS + WAF_CONFIG=$(aws wafv2 get-ip-set --scope CLOUDFRONT --id ${ID} --name ${NAME} 2>&1) + CMD_CD=$? + [[ $DEBUG -ge 1 ]] && echo "AWS CLI Read Response Code: ${CMD_CD}" + [[ $DEBUG -ge 2 ]] && echo "AWS CLI Read Response: ${WAF_CONFIG}" + + #If the retries exceeded error code is returned, try again, otherwise exit the loop + [[ $CMD_CD -eq $AWS_RETRY_ERROR ]] || break + + SLEEP_FOR=$(jitter ${j}) + echo "CLI retries exceed. Waiting for ${SLEEP_FOR} seconds to execute read again...$({j})" + sleep ${SLEEP_FOR} + done + + #Unable to get the lock tocken and IP set so there isn't any point in attempting the write op + [[ $j -ge $CIRCUIT_BREAKER ]] && echo “Attempts to read WAF IPSet exceeded” && sleep $(jitter ${i}) && continue + + #The loop was short circuited with an error code other than 0, so something is wrong + [[ $CMD_CD -eq 0 ]] || ( echo "An unexpected read error occurred: ${CMD_CD}" && exit 2 ) + + echo "Read was successful." + + if [ ${OP} == "append" ]; then + ##If this is used to whitelist individual ips or cidrs, using an additive approach is what is required + #Parse out IP set addresses to array + IP_ADDRESSES=($(jq -r '.IPSet.Addresses | .[]' <<< ${WAF_CONFIG})) + + #If CIDR is already present in IP set, eject + grep -q $RUNNER_CIDRS <<< ${IP_ADDRESSES} + [[ $? -ne 0 ]] || ( echo "CIDR is present in IP Set." && exit 0 ) + + #Add runner CIDR to array + IP_ADDRESSES+=("$RUNNER_CIDRS") + else + ##If this is used to hard set the IP set, just clobber it + IP_ADDRESSES=("$RUNNER_CIDRS") + fi + + #Stringify IPs + STRINGIFIED=$(echo $(IFS=" " ; echo "${IP_ADDRESSES[*]}")) + [[ $DEBUG -ge 2 ]] && echo "Ip Addresses: ${STRINGIFIED}" + + #Parse out optimistic concurrency control token + OCC_TOKEN=$(jq -r '.LockToken' <<< ${WAF_CONFIG}) + [[ $DEBUG -ge 2 ]] && echo "LockToken: ${OCC_TOKEN}" + + #This loop is ONLY for retrying if the retries exceeded exception is thrown + for ((k=1; k <= $CIRCUIT_BREAKER; k++)); do + #Write updated WAF configuration to AWS + OUTPUT=$(aws wafv2 update-ip-set --scope CLOUDFRONT --id ${ID} --name ${NAME} --lock-token ${OCC_TOKEN} --addresses ${STRINGIFIED} 2>&1) + CMD_CD=$? + [[ $DEBUG -ge 1 ]] && echo "AWS CLI Write Response Code: ${CMD_CD}" + [[ $DEBUG -ge 2 ]] && echo "AWS CLI Write Response: ${OUTPUT}" + + #If the retries exceeded error code is returned, try again, otherwise exit the loop + [[ $CMD_CD -eq $AWS_RETRY_ERROR ]] || break + #If WAFOptimisticLockException error code is returned, exit the loop + [[ "$OUTPUT" =~ "WAFOptimisticLockException" ]] && break + + SLEEP_FOR=$(jitter ${k}) + echo "CLI retries exceed. Waiting for ${SLEEP_FOR} seconds to execute write again...(${k})" + sleep ${SLEEP_FOR} + done + + [[ $CMD_CD -ne 0 ]] || break + #Still not having success, so try again + + echo "Exit Code: ${CMD_CD}" + + SLEEP_FOR=$(jitter ${i}) + echo "Waiting for ${SLEEP_FOR} seconds to execute main loop again...(${i})" + sleep ${SLEEP_FOR} +done + +[[ $DEBUG -ge 1 ]] && echo "Attempts to update ip set: $i" + +[[ $i -gt $CIRCUIT_BREAKER ]] && echo “Attempts to update WAF IPSet exceeded, exiting.” && exit 2 + +echo "Applied the IP Set successfully." + +#Things should not have made it this far without being able to successfully write the IP Set +exit $CMD_CD diff --git a/.github/workflows/cypress-workflow.yml b/.github/workflows/cypress-workflow.yml index 6297aae216..b20bc320c0 100644 --- a/.github/workflows/cypress-workflow.yml +++ b/.github/workflows/cypress-workflow.yml @@ -46,7 +46,7 @@ jobs: runs-on: ubuntu-latest container: image: cypress/browsers:node16.16.0-chrome107-ff107 - options: --user 1001 + options: --user root needs: setup strategy: fail-fast: false diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c8415d296a..2cc84ab594 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -38,6 +38,16 @@ jobs: echo "branch_name=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV fi - uses: actions/checkout@v3 + - name: 'Setup jq' + uses: dcarbone/install-jq-action@v2.1.0 + with: + version: '${{ inputs.version }}' + force: '${{ inputs.force }}' + - name: 'Check jq' + # language=sh + run: | + which jq + jq --version - name: Validate branch name run: ./.github/branchNameValidation.sh $STAGE_PREFIX$branch_name - name: set branch specific variable names @@ -80,6 +90,8 @@ jobs: working-directory: services outputs: application_endpoint: ${{ steps.endpoint.outputs.application_endpoint}} + BRANCH_SPECIFIC_VARNAME_AWS_DEFAULT_REGION: ${{ steps.set_names.outputs.BRANCH_SPECIFIC_VARNAME_AWS_DEFAULT_REGION }} + BRANCH_SPECIFIC_VARNAME_AWS_OIDC_ROLE_TO_ASSUME: ${{ steps.set_names.outputs.BRANCH_SPECIFIC_VARNAME_AWS_OIDC_ROLE_TO_ASSUME }} # run e2e tests after deploy completes e2e-tests-init: name: Initialize End To End Tests @@ -96,14 +108,19 @@ jobs: - uses: actions/checkout@v3 - name: set branch_name run: | - if [[ "$GITHUB_REF" =~ ^refs/heads/dependabot/.* ]]; then # Dependabot builds very long branch names. This is a switch to make it shorter. - echo "branch_name=`echo ${GITHUB_REF#refs/heads/} | md5sum | head -c 10 | sed 's/^/x/'`" >> $GITHUB_ENV + echo "GITHUB_REF=${GITHUB_REF}" + if [[ "$GITHUB_REF" =~ ^refs/heads/dependabot/.* ]]; then + echo "branch_name=`echo ${GITHUB_REF##*/*-} | md5sum | head -c 10 | sed 's/^/x/'`" >> $GITHUB_ENV + elif [[ "$GITHUB_REF" =~ ^refs/.*/snyk-* ]]; then + echo "branch_name=`echo ${GITHUB_REF##*/*-} | head -c 10 | sed 's/^/s/'`" >> $GITHUB_ENV else echo "branch_name=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV fi - name: set branch specific variable names + id: set_names run: ./.github/build_vars.sh set_names - name: set variable values + id: set_values run: ./.github/build_vars.sh set_values env: AWS_DEFAULT_REGION: ${{ secrets[env.BRANCH_SPECIFIC_VARNAME_AWS_DEFAULT_REGION] || secrets.AWS_DEFAULT_REGION }} @@ -139,13 +156,63 @@ jobs: - name: set path run: | echo "PATH=$(pwd)/node_modules/.bin/:$PATH" >> $GITHUB_ENV + - name: Get Runner IP + id: get-ip + run: | + #!/bin/bash + # Get the IP address of the runner + IP_ADDRESS=$(curl https://api.ipify.org) + echo "Runner IP address: $IP_ADDRESS" + # Store the IP address as an output variable + echo "RUNNER_IP=$IP_ADDRESS/32" >> $GITHUB_OUTPUT + - name: Get Github Actions CIDR Blocks + id: get-gha-cidrs + shell: bash + run: | + #!/bin/bash + GHA_RESP=$(curl --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' https://api.github.com/meta) + echo "Response for GHA runner CIDR blocks: $GHA_RESP" + IPV4_CIDR_ARR=($(echo $GHA_RESP | jq -r '.actions | .[]' | grep -v ':')) + GHA_CIDRS_IPV4=$(echo $(IFS=" "; echo ${IPV4_CIDR_ARR[*]})) + echo "GHA_CIDRS_IPV4=$GHA_CIDRS_IPV4" >> $GITHUB_OUTPUT + - name: Generate IP Set Name + id: gen-ip-set-name + run: | + STAGE_GH_IPSET_NAME=$STAGE_PREFIX$branch_name-gh-ipset + echo "Github IP Set name: $STAGE_GH_IPSET_NAME" + echo "STAGE_GH_IPSET_NAME=$STAGE_GH_IPSET_NAME" >> $GITHUB_OUTPUT + - name: Fetch AWS IP set Metadata + id: fetch-ip-set-info + run: | + #!/bin/bash + # Fetch AWS IP set ARNs using AWS CLI and store them in a variable + AWS_IP_SET_INFO=$(aws wafv2 list-ip-sets --scope=CLOUDFRONT) + # Store the IP set ARNs in an output variable using GITHUB_OUTPUT + IPSET_NAME=${{ steps.gen-ip-set-name.outputs.STAGE_GH_IPSET_NAME }} + IPSET=$(jq '.IPSets | map(select(.Name == "'${IPSET_NAME}'")) | .[]' <<< ${AWS_IP_SET_INFO}) + [ -z "$IPSET" ] && echo "IP Set with name ${IPSET_NAME} was not located. Exiting..." && exit 1 + echo "IP Set metadata: ${IPSET}" + #Get Values from the IP SET + IPSET_ID=$(jq -r '.Id' <<< ${IPSET}) + echo "IPSET_ARN=$IPSET_ARN" >> $GITHUB_OUTPUT + echo "IPSET_NAME=$IPSET_NAME" >> $GITHUB_OUTPUT + echo "IPSET_ID=$IPSET_ID" >> $GITHUB_OUTPUT + - name: Update IP Set + id: update-ip-set + run: ./.github/waf-controller.sh set ${{ steps.fetch-ip-set-info.outputs.IPSET_NAME }} ${{ steps.fetch-ip-set-info.outputs.IPSET_ID }} ${{ steps.get-gha-cidrs.outputs.GHA_CIDRS_IPV4 }} + env: + AWS_RETRY_MODE: adaptive + AWS_MAX_ATTEMPTS: 10 outputs: application_endpoint: ${{ needs.deploy.outputs.application_endpoint }} + ipset_name: ${{ steps.fetch-ip-set-info.outputs.IPSET_NAME }} + ipset_id: ${{ steps.fetch-ip-set-info.outputs.IPSET_ID }} setup-tests: name: "Setup End To End Tests" uses: ./.github/workflows/cypress-workflow.yml - needs: e2e-tests-init + needs: + - e2e-tests-init with: test-path: "init" test-endpoint: "${{ needs.e2e-tests-init.outputs.application_endpoint }}" @@ -244,3 +311,31 @@ jobs: cypress-user3: ${{ secrets.CYPRESS_TEST_USER_3 }} cypress-user4: ${{ secrets.CYPRESS_TEST_USER_4 }} cypress-password: ${{ secrets.CYPRESS_TEST_PASSWORD_1 }} + + cleanup: + name: Deslist GHA Runner CIDR Blocks + if: ${{ github.ref != 'refs/heads/master' && github.ref != 'refs/heads/val' && github.ref != 'refs/heads/prod' }} + runs-on: ubuntu-latest + needs: + - e2e-tests-init + - e2e-feature-tests + - child-e2e-measure-tests + - adult-e2e-measure-tests + - health-home-e2e-measure-tests + - deploy + - a11y-tests + env: + SLS_DEPRECATION_DISABLE: "*" # Turn off deprecation warnings in the pipeline + steps: + - uses: actions/checkout@v3 + - name: Configure AWS credentials for GitHub Actions + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets[env.BRANCH_SPECIFIC_VARNAME_AWS_OIDC_ROLE_TO_ASSUME] || secrets.AWS_OIDC_ROLE_TO_ASSUME }} + aws-region: ${{ secrets[env.BRANCH_SPECIFIC_VARNAME_AWS_DEFAULT_REGION] || secrets.AWS_DEFAULT_REGION }} + - name: clean-up-iplist + id: reset-ip-set + run: ./.github/waf-controller.sh set ${{ needs.e2e-tests-init.outputs.ipset_name }} ${{ needs.e2e-tests-init.outputs.ipset_id }} '[]' + env: + AWS_RETRY_MODE: adaptive + AWS_MAX_ATTEMPTS: 10 diff --git a/services/ui/package.json b/services/ui/package.json index 90324d6171..dd977525f2 100644 --- a/services/ui/package.json +++ b/services/ui/package.json @@ -10,5 +10,8 @@ "license": "CC0-1.0", "devDependencies": { "serverless-s3-bucket-helper": "Enterprise-CMCS/serverless-s3-bucket-helper#0.1.1" + }, + "dependencies": { + "@enterprise-cmcs/serverless-waf-plugin": "^1.3.1" } } diff --git a/services/ui/serverless.yml b/services/ui/serverless.yml index 5831f7dcf9..ea40b9195e 100644 --- a/services/ui/serverless.yml +++ b/services/ui/serverless.yml @@ -17,6 +17,7 @@ plugins: - serverless-stack-termination-protection - serverless-iam-helper - serverless-s3-bucket-helper + - "@enterprise-cmcs/serverless-waf-plugin" custom: stage: ${opt:stage, self:provider.stage} @@ -30,18 +31,75 @@ custom: route53DomainName: ${ssm:/configuration/${self:custom.stage}/route53/domainName, ""} cloudfrontCertificateArn: ${ssm:/configuration/${self:custom.stage}/cloudfront/certificateArn, ssm:/configuration/default/cloudfront/certificateArn, ""} cloudfrontDomainName: ${ssm:/configuration/${self:custom.stage}/cloudfront/domainName, ""} - webAclName: ${self:service}-${self:custom.stage}-webacl + vpnIpSetArn: ${ssm:/configuration/${self:custom.stage}/vpnIpSetArn, ssm:/configuration/default/vpnIpSetArn, ""} + # + wafExcludeRules: + wafScope: CLOUDFRONT + wafPlugin: + name: ${self:service}-${self:custom.stage}-webacl + rules: + - enable: ${param:restrictToVpn} + rule: + Name: vpn-only + Priority: 0 + Action: + Allow: {} + VisibilityConfig: + SampledRequestsEnabled: true + CloudWatchMetricsEnabled: true + MetricName: ${self:service}-${self:custom.stage}-webacl-vpn-only + Statement: + IPSetReferenceStatement: + ARN: ${self:custom.vpnIpSetArn} + - enable: ${param:restrictToVpn} + rule: + Name: github-only + Priority: 1 + Action: + Allow: {} + VisibilityConfig: + SampledRequestsEnabled: true + CloudWatchMetricsEnabled: true + MetricName: ${self:service}-${self:custom.stage}-tmp-gh-runner + Statement: + IPSetReferenceStatement: + ARN: !GetAtt GitHubIPSet.Arn + - enable: ${param:restrictToVpn} + rule: + Name: block-all-other-traffic + Priority: 3 + Action: + Block: + CustomResponse: + ResponseCode: 403 + VisibilityConfig: + SampledRequestsEnabled: true + CloudWatchMetricsEnabled: true + MetricName: ${self:service}-${self:custom.stage}-block-traffic + Statement: + NotStatement: + Statement: + IPSetReferenceStatement: + ARN: ${self:custom.vpnIpSetArn} firehoseStreamName: aws-waf-logs-${self:service}-${self:custom.stage}-firehose scripts: hooks: # Associate the WAF ACL with the Firehose Delivery Stream deploy:finalize: | - wafAclArn=`aws wafv2 list-web-acls --scope CLOUDFRONT | jq -r '.WebACLs | .[] | select(.Name=="${self:custom.webAclName}") | .ARN'` + wafAclArn=`aws wafv2 list-web-acls --scope CLOUDFRONT | jq -r '.WebACLs | .[] | select(.Name=="${self:custom.wafPlugin.name}") | .ARN'` firehoseStreamArn=`aws firehose describe-delivery-stream --delivery-stream-name ${self:custom.firehoseStreamName} | jq -r '.DeliveryStreamDescription.DeliveryStreamARN'` aws wafv2 put-logging-configuration \ --logging-configuration ResourceArn=$wafAclArn,LogDestinationConfigs=$firehoseStreamArn \ --region ${self:provider.region} - +params: + prod: + restrictToVpn: false + val: + restrictToVpn: false + master: + restrictToVpn: true + default: + restrictToVpn: true resources: Conditions: CreateDnsRecord: @@ -65,6 +123,13 @@ resources: - "" - ${self:custom.cloudfrontDomainName} Resources: + GitHubIPSet: + Type: AWS::WAFv2::IPSet + Properties: + Name: ${self:custom.stage}-gh-ipset + Scope: CLOUDFRONT + IPAddressVersion: IPV4 + Addresses: [] S3Bucket: Type: AWS::S3::Bucket Properties: @@ -137,35 +202,6 @@ resources: Bool: aws:SecureTransport: false Bucket: !Ref CloudFrontLoggingBucket - CloudFrontWebAcl: - Type: AWS::WAFv2::WebACL - Properties: - Name: ${self:custom.webAclName} - DefaultAction: - Block: {} - Rules: - - Action: - Allow: {} - Name: ${self:custom.webAclName}-allow-usa-plus-territories - Priority: 0 - Statement: - GeoMatchStatement: - CountryCodes: - - GU # Guam - - PR # Puerto Rico - - US # USA - - UM # US Minor Outlying Islands - - VI # US Virgin Islands - - MP # Northern Mariana Islands - VisibilityConfig: - SampledRequestsEnabled: true - CloudWatchMetricsEnabled: true - MetricName: WafWebAcl - Scope: CLOUDFRONT - VisibilityConfig: - CloudWatchMetricsEnabled: true - SampledRequestsEnabled: true - MetricName: ${self:custom.stage}-webacl CloudFrontOriginAccessIdentity: Type: AWS::CloudFront::CloudFrontOriginAccessIdentity Properties: @@ -214,7 +250,7 @@ resources: - ErrorCode: 403 ResponseCode: 403 ResponsePagePath: /index.html - WebACLId: !GetAtt CloudFrontWebAcl.Arn + WebACLId: !GetAtt WafPluginAcl.Arn Logging: Bucket: !Sub "${CloudFrontLoggingBucket}.s3.amazonaws.com" Prefix: AWSLogs/CLOUDFRONT/${self:custom.stage}/ diff --git a/services/ui/yarn.lock b/services/ui/yarn.lock index 8886796bb4..a38b95ee47 100644 --- a/services/ui/yarn.lock +++ b/services/ui/yarn.lock @@ -2,6 +2,11 @@ # yarn lockfile v1 +"@enterprise-cmcs/serverless-waf-plugin@^1.3.1": + version "1.3.2" + resolved "https://registry.yarnpkg.com/@enterprise-cmcs/serverless-waf-plugin/-/serverless-waf-plugin-1.3.2.tgz#66efd0b91326b7d1b045ab7ea7ba5826ed2e635d" + integrity sha512-577MWRddWK2uPEaeUMorOFQq6rhUhGwbdmz+tuKaU9+v77/bDQPqoc6cmhF2oYMswqpxvMgW0P07HAAcmKtquw== + serverless-s3-bucket-helper@Enterprise-CMCS/serverless-s3-bucket-helper#0.1.1: version "1.0.0" resolved "https://codeload.github.com/Enterprise-CMCS/serverless-s3-bucket-helper/tar.gz/f0f6d6a1ffe54e292f0afc93777764bce16a4037"