diff --git a/README.md b/README.md index d48097f..6997355 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Following packages are expected: - `curl` - `jq` - `awk` -- `date` with microsecond support. On *alpine* this means installing `coreutils` +- `date` with microsecond support. On *alpine* and *macOS* this means installing `coreutils` - `bash` > 4.0 - `git` only if you use GitLab @@ -312,7 +312,9 @@ Please look at [.gitlab-ci.yml.example](.gitlab-ci.yml.example) + `saas-linux-small-amd64` (GitLab) + Generic + `alpine` (Install dependencies before - See above) - + Also Windows and macOS are currently not supported. + + *macOS* is working on our local test machines (see install details below). + + Runners on GitHub are untested, but should work. You need to create a power profile though (see Cloud Energy below). We are happy for beta testers! contact us :) + + Windows is currently only supported with WSL2 - If you use Alpine, you must install coreutils so that time logging with date is possible with an accuracy of microseconds (`apk add coreutils`) @@ -326,7 +328,7 @@ See also our [work on analysing fixed frequency in Cloud Providers and CI/CD](ht ### Note on the integration / Auto-Updates - If you want the extension to automatically update within a version number, use the convenient @vX form. - + `uses: green-coding-solutions/eco-ci-energy-estimation@v4 # will pick the latest minor v3.x` + + `uses: green-coding-solutions/eco-ci-energy-estimation@v4 # will pick the latest minor v4.x` + In case of a major change from @v4 to @v5 you need to upgrade manually. The upside is: If you use dependabot it will create a PR for you as it understands the notation - If you want to pin the dependency and want to audit every release we recommend using the hash notation @@ -338,7 +340,38 @@ See also our [work on analysing fixed frequency in Cloud Providers and CI/CD](ht + We do **not** recommend this as it might contain beta features. We recommend using the releases and tagged versions only -### Testing +### macOS + +*macOS* requires the *GNU* `date` tool so it can properly create a microsecond timestamp. + +Install it with the package manager of your choice and then add its binary first in the `PATH` variable, so that it precedes the *BSD* `date`. + +Example for `homebrew`: +```bash +brew install coreutils +export PATH="/opt/homebrew/opt/coreutils/libexec/gnubin:$PATH" +# then you can run: +bash local_ci.example.sh +``` + +### Local CI / Running in docker + +Although initially designed for use in *GitHub Actions* and *GitLab Pipelines* the Eco-CI tool works everywhere where `bash` works. + +This means you can just use it locally by following it's general 3-step interface: +- Start +- Measure (optionally repeat if you want to lap multiple steps) +- End & Display + +As an example we have set up a full example pipeline in the form of a `bash` file under `local_ci.example.sh`. + +In this file you find the needed calls along with some fake activity like calls to `sleep` and `ls` etc. + +You just need to slice the file to you needs and bring the code that you want to encapsulate with Eco-CI into the positions where currently the `sleep` and `ls` calls are. + + + +### Trying out with Docker and Circle-CI image For local testing you can just run in the docker container of your choice, directly from the root of the repository. @@ -350,7 +383,8 @@ docker run --network host --rm -it -v ./:/tmp/data:ro cimg/base:current bash /tm In case you are testing with a local installation of the GMT append `--network host` to access `api.green-coding.internal` -### Testing for KDE pipelines +### Trying out with Docker and KDE pipelines ```bash docker run --rm -it -v ./:/tmp/data:ro invent-registry.kde.org/sysadmin/ci-images/suse-qt67:latest bash /tmp/data/local_ci.example.sh ``` + diff --git a/scripts/cpu-utilization.sh b/scripts/cpu-utilization-linux.sh similarity index 100% rename from scripts/cpu-utilization.sh rename to scripts/cpu-utilization-linux.sh diff --git a/scripts/cpu-utilization-macos.sh b/scripts/cpu-utilization-macos.sh new file mode 100644 index 0000000..eda47da --- /dev/null +++ b/scripts/cpu-utilization-macos.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# By ChatGPT and masterfully adapted by Arne Tarara. No rights reserved :) + +iostat -w 1 | awk -v date_cmd="gdate +%s%N" ' +NR > 3 { # skips first 3 rows, which contain header data and first average-only measurement + # Extract user, system, and idle CPU percentages + user = $3 + sys = $4 + idle = $6 + + # Calculate total CPU usage + usage = 100 - idle + + # Get the current time in seconds with microseconds + cmd = date_cmd + cmd | getline current_time_ns + close(cmd) + + # Calculate the time difference + if (last_time_ns != "") { + time_diff_ns = current_time_ns - last_time_ns + } else { + time_diff_ns = 1000000000 # No difference for the first line + } + + # Print the time and CPU usage + printf "%.10f %.2f\n", (time_diff_ns / 1000000000), usage + + + # Store the current time as the last time for the next iteration + last_time_ns = current_time_ns +}' \ No newline at end of file diff --git a/scripts/make_measurement.sh b/scripts/make_measurement.sh index 7817034..7830842 100755 --- a/scripts/make_measurement.sh +++ b/scripts/make_measurement.sh @@ -13,11 +13,13 @@ function make_inference() { # clear energy file for step because we fill it later anew echo > /tmp/eco-ci/energy-step.txt + power_data_file_path="$(dirname "$0")/../machine-power-data/${ECO_CI_MACHINE_POWER_DATA}" + # bash mode inference is slower in initi= 4 )); then echo "Using bash mode inference" - source "$(dirname "$0")/../machine-power-data/${ECO_CI_MACHINE_POWER_DATA}" # will set cloud_energy_hashmap + source "${power_data_file_path}" # will set cloud_energy_hashmap while read -r read_var_time read_var_util; do echo "${read_var_time} ${cloud_energy_hashmap[$read_var_util]}" | awk '{printf "%.9f\n", $1 * $2}' >> /tmp/eco-ci/energy-step.txt @@ -26,7 +28,8 @@ function make_inference() { echo 'Using legacy mode inference' while read -r read_var_time read_var_util; do # The pattern contains a . and [ ] but this no problem as no other dot appears anywhere - power_value=$(awk -F "=" -v pattern="cloud_energy_hashmap\\[${read_var_util}\\]" ' $0 ~ pattern { print $2 }' $ECO_CI_MACHINE_POWER_DATA) + + power_value=$(awk -F "=" -v pattern="cloud_energy_hashmap\\\\[${read_var_util}\\\\]" ' $0 ~ pattern { print $2 }' "${power_data_file_path}") echo "${read_var_time} ${power_value}" | awk '{printf "%.9f\n", $1 * $2}' >> /tmp/eco-ci/energy-step.txt done < /tmp/eco-ci/cpu-util-temp.txt fi diff --git a/scripts/setup.sh b/scripts/setup.sh index e04528c..880672b 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -83,11 +83,19 @@ function lap_measurement { # start writing cpu utilization with actual sleep durations end_measurement - bash "$(dirname "$0")/cpu-utilization.sh" > /tmp/eco-ci/cpu-util-step.txt 2> /dev/null < /dev/null & + if [[ $(uname) == "Darwin" ]]; then + bash "$(dirname "$0")/cpu-utilization-macos.sh" > /tmp/eco-ci/cpu-util-step.txt 2> /dev/null < /dev/null & + else + bash "$(dirname "$0")/cpu-utilization-linux.sh" > /tmp/eco-ci/cpu-util-step.txt 2> /dev/null < /dev/null & + fi } function end_measurement { - pkill -SIGTERM -f "$(dirname "$0")/cpu-utilization.sh" || true; + if [[ $(uname) == "Darwin" ]]; then + pkill -SIGTERM -f "$(dirname "$0")/cpu-utilization-macos.sh" || true; + else + pkill -SIGTERM -f "$(dirname "$0")/cpu-utilization-linux.sh" || true; + fi } diff --git a/scripts/vars.sh b/scripts/vars.sh index 89792b2..bdb572e 100755 --- a/scripts/vars.sh +++ b/scripts/vars.sh @@ -23,15 +23,24 @@ function cpu_vars { machine_power_data="$1" - model_name=$(cat /proc/cpuinfo | grep "model name" || true) + if [[ -f '/proc/cpuinfo' ]]; then + model_name=$(cat /proc/cpuinfo | grep 'model name' || true) - echo "Machine has following CPU Model ${model_name}" + echo "Machine has following CPU Model ${model_name}" - echo "Full CPU Info" - cat /proc/cpuinfo + echo 'Full CPU Info' + cat /proc/cpuinfo + else + echo '/proc/cpuinfo is not accessible ... cannot dump CPU model info' + model_name='UNKNOWN' + fi - echo "Full memory info" - cat /proc/meminfo + if [[ -f '/proc/meminfo' ]]; then + echo 'Full memory info' + cat /proc/meminfo + else + echo '/proc/meminfo does not exist. Cannot dump memory info' + fi if [[ "$machine_power_data" == "github_EPYC_7763_4_CPU_shared.sh" ]]; then