Merge pull request #74 from tgroshon/master

Add Suspending/Resuming ASG Processes to ELBv2

load-balancing/elb-v2/README.md

@@ -21,6 +21,8 @@ The following requirements must be met for the register and deregister scripts t
     autoscaling:EnterStandby
     autoscaling:ExitStandby
     autoscaling:UpdateAutoScalingGroup
+    autoscaling:SuspendProcesses
+    autoscaling:ResumeProcesses
 ```
 
 For information about creating an IAM instance profile to use with AWS CodeDeploy, see [Create an IAM Instance Profile for Your Amazon EC2 Instances](http://docs.aws.amazon.com/codedeploy/latest/userguide/how-to-create-iam-instance-profile.html).
@@ -73,3 +75,20 @@ PORT="80"
 ```
 
 7. Deploy your application. For information, see [Create a Deployment with AWS CodeDeploy](http://docs.aws.amazon.com/codedeploy/latest/userguide/deployments-create.html).
+
+## Important notice about handling AutoScaling processes
+
+When using AutoScaling with CodeDeploy you have to consider some edge cases during the deployment time window:
+
+1. If you have a scale up event, the new instance(s) will get the latest successful *Revision*, and not the one you are currently deploying. You will end up with a fleet of mixed revisions.
+2. If you have a scale down event, instances are going to be terminated, and your deployment will (probably) fail.
+3. If your instances are not balanced accross Availability Zones **and you are** using these scripts, AutoScaling may terminate some instances or create new ones to maintain balance (see [this doc](http://docs.aws.amazon.com/autoscaling/latest/userguide/as-suspend-resume-processes.html#process-types)), interfering with your deployment.
+4. If you have the health checks of your AutoScaling Group based off the ELB's ([documentation](http://docs.aws.amazon.com/autoscaling/latest/userguide/healthcheck.html)) **and you are not** using these scripts, then instances will be marked as unhealthy and terminated, interfering with your deployment.
+
+In an effort to solve these cases, the scripts can suspend some AutoScaling processes (AZRebalance, AlarmNotification, ScheduledActions and ReplaceUnhealthy) while deploying, to avoid those events happening in the middle of your deployment. You only have to set up `HANDLE_PROCS=true` in `common_functions.sh`.
+
+A record of the previously (to the start of the deployment) suspended process is kept by the scripts (on each instance), so when finishing the deployment the status of the processes on the AutoScaling Group should be returned to the same status as before. I.e. if AZRebalance was suspended manually it will not be resumed. However, if the scripts don't run (failed deployment) you may end up with stale suspended processes.
+
+Disclaimer: There's a small chance that an event is triggered while the deployment is progressing from one instance to another. The only way to avoid that completely would be to monitor the deployment externally to CodeDeploy/AutoScaling and act accordingly. The effort on doing that compared to this depends on the each use case.
+
+**WARNING**: If you are using this functionality you should only use *CodeDepoyDefault.OneAtATime* deployment configuration to ensure a serial execution of the scripts. Concurrent runs are not supported.

load-balancing/elb-v2/common_functions.sh

@@ -50,6 +50,12 @@ WAITER_INTERVAL_ALB=10
 # AutoScaling Standby features at minimum require this version to work.
 MIN_CLI_VERSION='1.10.55'
 
+# Create a flagfile for each deployment
+FLAGFILE="/tmp/asg_codedeploy_flags-$DEPLOYMENT_GROUP_ID-$DEPLOYMENT_ID"
+
+# Handle ASG processes
+HANDLE_PROCS=false
+
 #
 # Performs CLI command and provides expotential backoff with Jitter between any failed CLI commands
 # FullJitter algorithm taken from: https://www.awsarchitectureblog.com/2015/03/backoff.html
@@ -111,6 +117,111 @@ get_instance_region() {
 
 AWS_CLI="exec_with_fulljitter_retry aws --region $(get_instance_region)"
 
+# Usage: set_flag <flag> <value>
+#
+#   Writes <flag>=<value> to FLAGFILE
+set_flag() {
+    if echo "$1=$2" >> $FLAGFILE; then
+        return 0
+    else
+        error_exit "Unable to write flag \"$1=$2\" to $FLAGFILE"
+    fi
+}
+
+# Usage: get_flag <flag>
+#
+#   Checks for <flag> in FLAGFILE. Echoes it's value and returns 0 on success or non-zero if it fails to read the file.
+get_flag() {
+    if [ -r $FLAGFILE ]; then
+        local result=$(awk -F= -v flag="$1" '{if ( $1 == flag ) {print $2}}' $FLAGFILE | tail -1)
+        echo "${result}"
+        return 0
+    else
+        # FLAGFILE doesn't exist
+        return 1
+    fi
+}
+
+# Usage: check_suspended_processes
+#
+#   Checks processes suspended on the ASG before beginning and store them in
+#   the FLAGFILE to avoid resuming afterwards. Also abort if Launch process
+#   is suspended.
+check_suspended_processes() {
+  # Get suspended processes in an array
+  local suspended=($($AWS_CLI autoscaling describe-auto-scaling-groups \
+      --auto-scaling-group-name \"${asg_name}\" \
+      --query \'AutoScalingGroups[].SuspendedProcesses\' \
+      --output text \| awk \'{printf \$1\" \"}\'))
+
+  if [ ${#suspended[@]} -eq 0 ]; then
+    msg "No processes were suspended on the ASG before starting."
+  else
+    msg "This processes were suspended on the ASG before starting: ${suspended[*]}"
+  fi
+
+  # If "Launch" process is suspended abort because we will not be able to recover from StandBy. Note the "[[ ... =~" bashism.
+  if [[ "${suspended[@]}" =~ "Launch" ]]; then
+    error_exit "'Launch' process of AutoScaling is suspended which will not allow us to recover the instance from StandBy. Aborting."
+  fi
+
+  for process in ${suspended[@]}; do
+    set_flag "$process" "true"
+  done
+}
+
+# Usage: suspend_processes
+#
+#   Suspend processes known to cause problems during deployments.
+#   The API call is idempotent so it doesn't matter if any were previously suspended.
+suspend_processes() {
+  local -a processes=(AZRebalance AlarmNotification ScheduledActions ReplaceUnhealthy)
+
+  msg "Suspending ${processes[*]} processes"
+  $AWS_CLI autoscaling suspend-processes \
+    --auto-scaling-group-name \"${asg_name}\" \
+    --scaling-processes ${processes[@]}
+  if [ $? != 0 ]; then
+    error_exit "Failed to suspend ${processes[*]} processes for ASG ${asg_name}. Aborting as this may cause issues."
+  fi
+}
+
+# Usage: resume_processes
+#
+#   Resume processes suspended, except for the one suspended before deployment.
+resume_processes() {
+  local -a processes=(AZRebalance AlarmNotification ScheduledActions ReplaceUnhealthy)
+  local -a to_resume
+
+  for p in ${processes[@]}; do
+    if ! local tmp_flag_value=$(get_flag "$p"); then
+        error_exit "$FLAGFILE doesn't exist or is unreadable"
+    elif [ ! "$tmp_flag_value" = "true" ] ; then
+      to_resume=("${to_resume[@]}" "$p")
+    fi
+  done
+
+  msg "Resuming ${to_resume[*]} processes"
+  $AWS_CLI autoscaling resume-processes \
+    --auto-scaling-group-name "${asg_name}" \
+    --scaling-processes ${to_resume[@]}
+  if [ $? != 0 ]; then
+    error_exit "Failed to resume ${to_resume[*]} processes for ASG ${asg_name}. Aborting as this may cause issues."
+  fi
+}
+
+# Usage: remove_flagfile
+#
+#   Removes FLAGFILE. Returns non-zero if failure.
+remove_flagfile() {
+  if rm $FLAGFILE; then
+      msg "Successfully removed flagfile $FLAGFILE"
+      return 0
+  else
+      msg "WARNING: Failed to remove flagfile $FLAGFILE."
+  fi
+}
+
 # Usage: autoscaling_group_name <EC2 instance ID>
 #
 #    Prints to STDOUT the name of the AutoScaling group this instance is a part of and returns 0. If
@@ -163,6 +274,14 @@ autoscaling_enter_standby() {
         return 0
     fi
 
+    if [ "$HANDLE_PROCS" = "true" ]; then
+        msg "Checking ASG ${asg_name} suspended processes"
+        check_suspended_processes
+
+        # Suspend troublesome processes while deploying
+        suspend_processes
+    fi
+
     msg "Checking to see if ASG ${asg_name} will let us decrease desired capacity"
     local min_desired=$($AWS_CLI autoscaling describe-auto-scaling-groups \
         --auto-scaling-group-name \"${asg_name}\" \
@@ -185,9 +304,9 @@ autoscaling_enter_standby() {
             msg "Failed to reduce ASG ${asg_name}'s minimum size to $new_min. Cannot put this instance into Standby."
             return 1
         else
-            msg "ASG ${asg_name}'s minimum size has been decremented, creating flag file /tmp/asgmindecremented"
-            # Create a "flag" file to denote that the ASG min has been decremented
-            touch /tmp/asgmindecremented
+            msg "ASG ${asg_name}'s minimum size has been decremented, creating flag in file $FLAGFILE"
+            # Create a "flag" to denote that the ASG min has been decremented
+            set_flag "asgmindecremented" "true"
         fi
     fi
 
@@ -254,7 +373,9 @@ autoscaling_exit_standby() {
         return 1
     fi
 
-    if [ -a /tmp/asgmindecremented ]; then
+    if ! local tmp_flag_value=$(get_flag "asgmindecremented"); then
+        error_exit "$FLAGFILE doesn't exist or is unreadable"
+    elif [ "$tmp_flag_value" = "true" ]; then
         local min_desired=$($AWS_CLI autoscaling describe-auto-scaling-groups \
             --auto-scaling-group-name \"${asg_name}\" \
             --query \'AutoScalingGroups[0].[MinSize, DesiredCapacity]\' \
@@ -269,16 +390,20 @@ autoscaling_exit_standby() {
             --min-size $new_min)
         if [ $? != 0 ]; then
             msg "Failed to increase ASG ${asg_name}'s minimum size to $new_min."
+            remove_flagfile
             return 1
         else
             msg "Successfully incremented ASG ${asg_name}'s minimum size"
-            msg "Removing /tmp/asgmindecremented flag file"
-            rm -f /tmp/asgmindecremented
         fi
     else
         msg "Auto scaling group was not decremented previously, not incrementing min value"
     fi
 
+    if [ "$HANDLE_PROCS" = "true" ]; then
+        resume_processes
+    fi
+
+    remove_flagfile
     return 0
 }