restart fetch only when it is idling

.github/workflows/restart.yml

@@ -76,15 +76,15 @@ jobs:
       - name: restart gather
         uses: cloud-gov/cg-cli-tools@main
         with:
-          command: tools/restart_gather.sh
+          command: tools/restart_harvester.sh catalog-gather
           cf_org: gsa-datagov
           cf_space: prod
           cf_username: ${{secrets.CF_SERVICE_USER}}
           cf_password: ${{secrets.CF_SERVICE_AUTH}}
       - name: restart fetch
         uses: cloud-gov/cg-cli-tools@main
         with:
-          command: cf restart catalog-fetch
+          command: tools/restart_harvester.sh catalog-fetch
           cf_org: gsa-datagov
           cf_space: prod
           cf_username: ${{secrets.CF_SERVICE_USER}}

tools/restart_harvester.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Custom script to ensure harvester processes are safe to restart
+# Based on
+#  - https://github.com/GSA/data.gov/issues/3796
+#  - https://github.com/GSA/data.gov/issues/3962
+
+# Input either catalog-gather or catalog-fetch
+app_to_restart=$1
+
+###################
+# Utility Functions
+# Get the number of lines in the log (int)
+log_count () { cf logs --recent $app_to_restart | wc -l; }
+
+# Get the time of the last line in the log (int)
+current_time=$(date --utc +%s)
+log_last_time () {
+  date --utc --date="$(cf logs --recent $app_to_restart | tail -n 1 | awk '{split($0,time," "); print time[1]}')" +%s
+}
+
+# Restart if CPU usage is < 1
+cpu_restart () {
+  instances=$(cf app $app_to_restart | grep '^instances:' | sed 's/.*\///')
+  i=0
+  cf app $app_to_restart | tail -n $instances | awk '{ split($4,cpu,"."); print cpu[1]}' | while read -r cpu ; do
+    if [[ $cpu < 1 ]]; then
+      cf restart-app-instance $app_to_restart $i
+    fi
+    i=$(($i + 1))
+  done;
+}
+
+############
+# Main Logic
+# Check the age of the last log
+# If there are only two lines, there are no logs
+if [[ $((`log_count` > 2)) == '1' ]]; then
+  # if the timestamp is younger than 15 mins from start of script
+  if [[ $((current_time - `log_last_time` < 900)) == '1' ]]; then
+    echo "Logs are too new!  Not going to restart"
+    exit 0
+  fi
+fi
+
+# if CPU status shows it is not busy, we do the restart
+$(cpu_restart)