profiling.yaml

scripts:
  run-pidstat:
  # WAIT_FOR: which signal should it wait before starting 
  # WAIT_END: which signal should it wait before stopping
  # FILE_ID: a name to identify the generated file if multiple runs
  - js: ${{PIDSTAT_ENABLED}}
    then:
    - wait-for: ${{WAIT_FOR:HF_BENCHMARK_STARTED}}
    - js: ${{HEROES_ENABLED}}
      then:
      - log: starting pidstat against ${{HOST.HEROES_REST_PID}}..
      - sh: pidstat -p ${{HOST.HEROES_REST_PID}} ${{PIDSTAT_PARAMS}} > /tmp/HEROES_pidstat_${{FILE_ID:all}}.log & export HEROES_PIDSTAT_PID="$!"
      - queue-download: /tmp/HEROES_pidstat_${{FILE_ID:all}}.log
    - js: ${{VILLAINS_ENABLED}}
      then:
      - log: starting pidstat against ${{HOST.VILLAINS_REST_PID}}..
      - sh: pidstat -p ${{HOST.VILLAINS_REST_PID}} ${{PIDSTAT_PARAMS}} > /tmp/VILLAINS_pidstat_${{FILE_ID:all}}.log & export VILLAINS_PIDSTAT_PID="$!"
      - queue-download: /tmp/VILLAINS_pidstat_${{FILE_ID:all}}.log
    - js: ${{LOCATIONS_ENABLED}}
      then:
      - log: starting pidstat against ${{HOST.LOCATIONS_GRPC_PID}}..
      - sh: pidstat -p ${{HOST.LOCATIONS_GRPC_PID}} ${{PIDSTAT_PARAMS}} > /tmp/LOCATIONS_pidstat_${{FILE_ID:all}}.log & export LOCATIONS_PIDSTAT_PID="$!"
      - queue-download: /tmp/LOCATIONS_pidstat_${{FILE_ID:all}}.log
    - js: ${{FIGHTS_ENABLED}}
      then:
      - log: starting pidstat against ${{HOST.FIGHTS_REST_PID}}..
      - sh: pidstat -p ${{HOST.FIGHTS_REST_PID}} ${{PIDSTAT_PARAMS}} > /tmp/FIGHTS_pidstat_${{FILE_ID:all}}.log & export FIGHTS_PIDSTAT_PID="$!"
      - queue-download: /tmp/FIGHTS_pidstat_${{FILE_ID:all}}.log
    - wait-for: ${{WAIT_END:HF_BENCHMARK_TERMINATED}}
      then:
      - sh: kill ${HEROES_PIDSTAT_PID} || echo "pidstat already stopped"
      - sh: kill ${VILLAINS_PIDSTAT_PID} || echo "pidstat already stopped"
      - sh: kill ${LOCATIONS_PIDSTAT_PID} || echo "pidstat already stopped"
      - sh: kill ${FIGHTS_PIDSTAT_PID} || echo "pidstat already stopped"
  
  # measure RSS
  run-pmap:
  # WAIT_FOR: which signal should it wait before starting 
  # WAIT_END: which signal should it wait before stopping
  # MAIN_SERVICE_PID_STATE: the pid of the JVM process to profile
  # MAIN_SERVICE_ID: service to monitor FIGHTS, HEROES, VILLAINS or LOCATIONS
  # FILE_ID: a name to identify the generated file if multiple runs
  - js: ${{PMAP_ENABLED}}
    then:
    - wait-for: ${{WAIT_FOR:HF_BENCHMARK_STARTED}}
    - queue-download: /tmp/${{MAIN_SERVICE_ID}}_rss_${{FILE_ID:all}}.log
    - repeat-until: ${{WAIT_END:HF_BENCHMARK_TERMINATED}}
      then:
      - sh: echo -n "$(date +"%I:%M:%S %p")  " >> /tmp/${{MAIN_SERVICE_ID}}_rss_${{FILE_ID:all}}.log
      - sh:
          # NOTE: with docker this requires starting the containers with the same host user, this is now sets by default with
          # --user "$(id -u):$(id -g)"
          command: pmap -x ${{${{MAIN_SERVICE_PID_STATE}}}} ${{PMAP_PARAMS:}} 2>/dev/null | grep total >> /tmp/${{MAIN_SERVICE_ID}}_rss_${{FILE_ID:all}}.log
          ignore-exit-code: true
      - sleep: 1s

  run-strace:
  # WAIT_FOR: which signal should it wait before starting 
  # WAIT_END: which signal should it wait before stopping
  # FILE_ID: a name to identify the generated file if multiple runs
  - js: ${{STRACE_ENABLED}}
    then:
    - wait-for: ${{WAIT_FOR:HF_BENCHMARK_STARTED}}
    - js: ${{FIGHTS_ENABLED}}
      then:
      - log: starting strace against ${{HOST.FIGHTS_REST_PID}}..
      - sh: strace -p ${{HOST.FIGHTS_REST_PID}} ${{STRACE_PARAMS:}} 2> /tmp/FIGHTS_strace_${{FILE_ID:all}}.log & export FIGHTS_STRACE_PID="$!"
      - queue-download: /tmp/FIGHTS_strace_${{FILE_ID:all}}.log
    - wait-for: ${{WAIT_END:HF_BENCHMARK_TERMINATED}}
      then:
      - sh: kill ${FIGHTS_STRACE_PID} || echo "strace already stopped"

  run-perfstat:
  # WAIT_FOR: which signal should it wait before starting 
  # WAIT_END: which signal should it wait before stopping
  # MAIN_SERVICE_ID: service to monitor with perf stat FIGHTS, HEROES, VILLAINS or LOCATIONS
  # FILE_ID: a name to identify the generated file if multiple runs
  - js: ${{PERFSTAT_ENABLED}} && ${{${{MAIN_SERVICE_ID}}_ENABLED}}
    then:
    - wait-for: ${{WAIT_FOR:HF_BENCHMARK_STARTED}}
    # - js: ${{FIGHTS_ENABLED}}
      # then:
    - queue-download: /tmp/${{MAIN_SERVICE_ID}}_perfstat_${{FILE_ID:all}}.log
    - log: starting perf stat against ${{HOST.${{MAIN_SERVICE_ID}}_REST_PID}}..
    - sh: perf stat -d -p ${{HOST.${{MAIN_SERVICE_ID}}_REST_PID}} ${{PERFSTAT_PARAMS:}} -o /tmp/${{MAIN_SERVICE_ID}}_perfstat_${{FILE_ID:all}}.log
      silent: true # Prevent the 'Nanny found idle' warnings
      watch:
        - regex: Problems finding threads of monitor
          then:
            - abort: Failed to start perf, unable to find provided pid ${{HOST.${{MAIN_SERVICE_ID}}_REST_PID}}
      on-signal:
        ${{WAIT_END:HF_BENCHMARK_TERMINATED}}:
          - log: Stopping perf
          - ctrlC

  run-vmstat:
  # WAIT_FOR: which signal should it wait before starting 
  # WAIT_END: which signal should it wait before stopping
  - js: ${{VMSTAT_ENABLED}}
    then:
    - wait-for: ${{WAIT_FOR:HF_BENCHMARK_STARTED}}
    - log: starting vmstat..
    - sh: vmstat ${{VMSTAT_PARAMS}} > /tmp/vmstat.log & export VMSTAT_PID="$!"
    - queue-download: /tmp/vmstat.log
    - wait-for: ${{WAIT_END:HF_BENCHMARK_TERMINATED}}
      then:
      - sh: kill ${VMSTAT_PID} || echo "vmstat already stopped"

  run-mpstat:
  # WAIT_FOR: which signal should it wait before starting 
  # WAIT_END: which signal should it wait before stopping
  - js: ${{MPSTAT_ENABLED}}
    then:
    - wait-for: ${{WAIT_FOR:HF_BENCHMARK_STARTED}}
    - log: starting mpstat..
    - sh: mpstat ${{MPSTAT_PARAMS}} > /tmp/mpstat.log & export MPSTAT_PID="$!"
    - queue-download: /tmp/mpstat.log
    - wait-for: ${{WAIT_END:HF_BENCHMARK_TERMINATED}}
      then:
      - sh: kill ${MPSTAT_PID} || echo "mpstat already stopped"
  
  # application profiling
  app-prof-setup:
  - js: ${{APP_PROF_ENABLED}}
    then:
    - sh: ${{SUDOER}} sh -c "echo 1 > /proc/sys/kernel/perf_event_paranoid" || echo "Unable to set perf_event_paranoid"
    - sh: ${{SUDOER}} sh -c "echo 0 > /proc/sys/kernel/kptr_restrict" || echo "Unable to unset kptr_restrict"
    - js: ${{IS_NATIVE}}
      else:
      - script: asprof-setup

  # async profiler
  asprof-setup:
  - js: ${{APP_PROF_ENABLED}}
    then:
    # override container runtimes to include asprof volumes
    - set-state: CONTAINER_RUNTIME_PARAMS ${{CONTAINER_RUNTIME_PARAMS}} -v ${{ASPROF_DIR}}:${{ASPROF_DIR}}:rw,Z -v ${{ASPROF_OUT_DIR}}:${{ASPROF_OUT_DIR}}:rw,Z 
    - sh: if [ -d ${{ASPROF_DIR}} ]; then rm -rf ${{ASPROF_DIR}}; fi
    - sh: curl -L ${{ASPROF_RELEASE_URL}} > /tmp/async-profiler.tgz
    - sh: tar -xvf /tmp/async-profiler.tgz -C /tmp
    - sh: mv /tmp/async-profiler-${{ASPROF_VERSION:3.0}}-linux-x64 ${{ASPROF_DIR:/tmp/async-profiler}}
    # cleanup downloaded tar
    - sh: rm -rf /tmp/async-profiler.tgz
    - set-state: RUN.ASPROF_EXEC ${{ASPROF_DIR:/tmp/async-profiler}}/bin/asprof
    - sh: if [ -d ${{ASPROF_OUT_DIR}} ]; then rm -rf ${{ASPROF_OUT_DIR}}; fi
    - sh: mkdir -p ${{ASPROF_OUT_DIR}} && chmod 777 ${{ASPROF_OUT_DIR}} || echo "Cannot create ${{ASPROF_OUT_DIR}} directory"
  
  run-app-prof:
  - js: ${{APP_PROF_ENABLED}}
    then:
    - js: ${{IS_NATIVE}}
      then:
      - script: run-perfrecord
      else:
      - script: run-asprof

  run-perfrecord:
  # WAIT_FOR: which signal should it wait before starting 
  # WAIT_END: which signal should it wait before stopping
  # MAIN_SERVICE_PID_STATE: the pid of the JVM process to profile
  # MAIN_SERVICE_ID: service to monitor FIGHTS, HEROES, VILLAINS or LOCATIONS
  # FILE_ID: a name to identify the generated file if multiple runs
  - wait-for: ${{WAIT_FOR:HF_BENCHMARK_STARTED}}
  - log: Starting application level profiling for native ${{${{MAIN_SERVICE_PID_STATE}}}}...
  - sh: 
      command: ${{SUDOER}} ls -la /work
      ignore-exit-code: true
  - sh: ${{SUDOER}} perf record ${{PERFRECORD_PARAMS}} -o /tmp/${{MAIN_SERVICE_ID}}_perfrecord.data -p ${{${{MAIN_SERVICE_PID_STATE}}}}
    on-signal:
      # stop perf when the HF load terminates
      ${{WAIT_END:HF_BENCHMARK_TERMINATED}}:
      - ctrlC
  - sh: ${{SUDOER}} perf script -i /tmp/${{MAIN_SERVICE_ID}}_perfrecord.data >> /tmp/${{MAIN_SERVICE_ID}}_perfrecord-script.txt
  # - sh: ${{SUDOER}} perf report -i /tmp/${{MAIN_SERVICE_ID}}_perfrecord.data >> /tmp/${{MAIN_SERVICE_ID}}_perfrecord-report.txt
  - queue-download: /tmp/${{MAIN_SERVICE_ID}}_perfrecord.data
  - queue-download: /tmp/${{MAIN_SERVICE_ID}}_perfrecord-script.txt
  # - queue-download: /tmp/${{MAIN_SERVICE_ID}}_perfrecord-report.txt
  # remove the /work emulation if present
  - sh: if [ -d /work ]; then rm -rf /work || echo "Cannot remove /work directory"; fi

  run-asprof:
  # Relies on existing async-profiler exectuable located at ASPROF_EXEC
  # MAIN_SERVICE_PID_STATE: the pid of the JVM process to profile
  # MAIN_SERVICE_ID: service to monitor FIGHTS, HEROES, VILLAINS or LOCATIONS
  # FILE_ID: a name to identify the generated file if multiple runs
  - wait-for: ${{WAIT_FOR:HF_BENCHMARK_STARTED}}
  - log: Starting application level profiling for JVM ${{${{MAIN_SERVICE_PID_STATE}}}}...
  - sh: ${{SUDOER}} ${{ASPROF_EXEC:/tmp/async-profiler/bin/asprof}} start -e ${{ASPROF_EVENT}} --fdtransfer ${{ASPROF_PARAMS:}} ${{${{MAIN_SERVICE_PID_STATE}}}}
  - log: Async profiler started, waiting for ${{WAIT_END:HF_BENCHMARK_TERMINATED}} stop signal...
  - wait-for: ${{WAIT_END:HF_BENCHMARK_TERMINATED}}
  - sh: ${{SUDOER}} ${{ASPROF_EXEC:/tmp/async-profiler/bin/asprof}} stop -f ${{ASPROF_OUT_DIR}}/${{MAIN_SERVICE_ID:service}}_asprof_${{FILE_ID:all}} -o ${{ASPROF_FORMAT:flamegraph}} --title '${{MAIN_SERVICE_ID:service}} ${{ASPROF_EVENT}} profiling' ${{${{MAIN_SERVICE_PID_STATE}}}}
  - log: Stopped application level profiling
  # Rename the output file based on the provided format file extension
  - set-state: FILE_EXTENSION ${{= ${{ASPROF_FORMATS}}['${{ASPROF_FORMAT:flamegraph}}'] }}
  - sh: mv ${{ASPROF_OUT_DIR}}/${{MAIN_SERVICE_ID:service}}_asprof_${{FILE_ID:all}} ${{ASPROF_OUT_DIR}}/${{MAIN_SERVICE_ID:service}}_asprof_${{FILE_ID:all}}.${{FILE_EXTENSION}}
  - queue-download: ${{ASPROF_OUT_DIR}}/${{MAIN_SERVICE_ID:service}}_asprof_${{FILE_ID:all}}.${{FILE_EXTENSION}}
  
  capture-first-response:
  - wait-for: HF_BENCHMARK_TERMINATED
  # capture this if and only if LOOP_SERVICE_FIRST_RESP_TIMESTAMP is set
  - read-state: ${{RUN.LOOP_SERVICE_FIRST_RESP_TIMESTAMP}}
    then:
    - log: Capturing first response time
    - sh: ${{CONTAINER_RUNTIME}} inspect ${{ ${{MAIN_SERVICE_POD_ID_STATE}} }} | grep StartedAt | awk '{print $2}'| awk '{gsub("\"", " "); print $1}'| awk '{gsub("T"," "); print}'|awk '{print substr($0, 1, length($0)-6)}'
      then:
      - set-state: LOOP_SERVICE_START_TIME
    - sh: echo $(($(date "+%s%N" -d "${{LOOP_SERVICE_START_TIME}} $(date "+%z")")/1000000))
      then:
      - set-state: LOOP_SERVICE_START_TIMESTAMP
    - sh: echo $((${{LOOP_SERVICE_FIRST_RESP_TIMESTAMP}} - ${{LOOP_SERVICE_START_TIMESTAMP}}))
      then:
      - set-state: FIRST_RESPONSE_TIME
    

  # parse the metrics from the generated profiling files and set them to 
  # qDup states so that they can easily be exported
  # RSS at the end
  # RSS (min, max, mean)
  # CPU usage (min, max, mean)
  # vmstat? mpstat?
  export-metrics:
  # stats for pidstat
  - js: ${{PIDSTAT_ENABLED}}
    then:
    - wait-for: ${{WAIT_FOR:HF_BENCHMARK_STARTED}}
    - js: ${{HEROES_ENABLED}}
      then:
      - script: parse-pidstat-results
        with:
          PIDSTAT_FILE: /tmp/HEROES_pidstat_${{FILE_ID:all}}.log
          MAX_STATE: ${{FILE_ID:all}}_HEROES_CPU_USAGE_MAX
          AVG_STATE: ${{FILE_ID:all}}_HEROES_CPU_USAGE_AVG
    - js: ${{VILLAINS_ENABLED}}
      then:
      - script: parse-pidstat-results
        with:
          PIDSTAT_FILE: /tmp/VILLAINS_pidstat_${{FILE_ID:all}}.log
          MAX_STATE: ${{FILE_ID:all}}_VILLAINS_CPU_USAGE_MAX
          AVG_STATE: ${{FILE_ID:all}}_VILLAINS_CPU_USAGE_AVG
    - js: ${{LOCATIONS_ENABLED}}
      then:
      - script: parse-pidstat-results
        with:
          PIDSTAT_FILE: /tmp/LOCATIONS_pidstat_${{FILE_ID:all}}.log
          MAX_STATE: ${{FILE_ID:all}}_LOCATIONS_CPU_USAGE_MAX
          AVG_STATE: ${{FILE_ID:all}}_LOCATIONS_CPU_USAGE_AVG
    - js: ${{FIGHTS_ENABLED}}
      then:
      - script: parse-pidstat-results
        with:
          PIDSTAT_FILE: /tmp/FIGHTS_pidstat_${{FILE_ID:all}}.log
          MAX_STATE: ${{FILE_ID:all}}_FIGHTS_CPU_USAGE_MAX
          AVG_STATE: ${{FILE_ID:all}}_FIGHTS_CPU_USAGE_AVG
  # rss
  - js: ${{PMAP_ENABLED}}
    then:
    - script: parse-rss-results
      with:
        RSS_FILE: /tmp/${{MAIN_SERVICE_ID}}_rss_${{FILE_ID:all}}.log
        MAX_STATE: ${{FILE_ID:all}}_${{MAIN_SERVICE_ID}}_RSS_MAX
        AVG_STATE: ${{FILE_ID:all}}_${{MAIN_SERVICE_ID}}_RSS_AVG

  cleanup-profiling:
  - sh:
      command: rm -rf /tmp/vmstat.log /tmp/mpstat.log /tmp/*_rss*.log /tmp/*_pidstat*.log /tmp/*_perfstat_${{FILE_ID:all}}.log 
      ignore-exit-code: true
  - sh: 
      command: if [ -d ${{ASPROF_DIR}} ]; then rm -rf ${{ASPROF_DIR}}; fi
      ignore-exit-code: true
  - sh: 
      command: if [ -d ${{ASPROF_OUT_DIR}} ]; then rm -rf ${{ASPROF_OUT_DIR}}; fi
      ignore-exit-code: true

states:
  SUDOER: 

  PIDSTAT_ENABLED: true
  PIDSTAT_PARAMS: "1"

  PMAP_ENABLED: true
  PMAP_PARAMS: 

  STRACE_ENABLED: false
  STRACE_PARAMS: -s 99 -ff

  PERFSTAT_ENABLED: true
  PERFSTAT_PARAMS: 

  VMSTAT_ENABLED: true
  VMSTAT_PARAMS: "1" 

  MPSTAT_ENABLED: true
  MPSTAT_PARAMS: "-P ALL 1"

  # in according to the stack it will use either perf record or asprof
  APP_PROF_ENABLED: false

  ASPROF_RELEASE_URL: https://github.com/async-profiler/async-profiler/releases/download/v${{ASPROF_VERSION:3.0}}/async-profiler-${{ASPROF_VERSION:3.0}}-linux-x64.tar.gz
  ASPROF_VERSION: "3.0"
  ASPROF_DIR: /tmp/async-profiler
  ASPROF_PARAMS: 
  ASPROF_EVENT: cpu
  ASPROF_FORMAT: flamegraph
  ASPROF_OUT_DIR: /tmp/async-profiler-output
  ASPROF_FORMATS: |
    {
      "jfr": "jfr",
      "flamegraph": "html"
    }

  PERFRECORD_PARAMS: -e cpu-clock -F 100 --call-graph dwarf