Better overhead and api

.github/workflows/data-json-test.yml

@@ -12,24 +12,29 @@ on:
     # Reason being that we pull our ML model and this could have changed in the meantime
     - cron: '22 4 * * 6'
   workflow_dispatch:
-  
+
 jobs:
   test-data-output-action:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Initialize Energy Estimation
         uses: ./
         with:
           task: start-measurement
+          json-output: true
 
       - uses: actions/setup-node@v3
         with:
           node-version: '18'
 
+      - name: Sleep 2
+        run: |
+          sleep 2
+
       - name: Node Setup Energy Measurment
         id: data-node-setup
         uses: ./

.github/workflows/overhead_test.yml

@@ -0,0 +1,34 @@
+name: test-action
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: read
+  pull-requests: write
+
+jobs:
+  test-action:
+    runs-on: ${{ matrix.os }}
+    continue-on-error: false
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [ubuntu-22.04, ubuntu-24.04, ubuntu-20.04]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Sleep step
+        run: sleep 2
+
+      - name: Filesystem
+        run: timeout 10s ls -alhR /usr/lib
+        continue-on-error: true
+
+
+      - name: Sleep 3
+        run: sleep 3
+

.github/workflows/test.yml

@@ -20,14 +20,12 @@ permissions:
 
 jobs:
   test-action:
-    runs-on: ubuntu-latest
-#    runs-on: ${matrix.os}
-#    continue-on-error: false
-#    strategy:
-#      fail-fast: true
-#      matrix:
-#        os: [ubuntu-latest, ubuntu-24.04, ubuntu-20.04, ubuntu-18.04, ubuntu-20.04-16core]
-#        # It might be the case
+    runs-on: ${{ matrix.os }}
+    continue-on-error: false
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [ubuntu-22.04, ubuntu-24.04, ubuntu-20.04]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -45,7 +43,11 @@ jobs:
       - name: Sleep step
         run: sleep 2
 
-      - name: Dump ECO-CI CPU
+      - name: Dump ECO-CI CPU Step before
+        run: |
+          cat /tmp/eco-ci/cpu-util-step.txt
+
+      - name: Dump ECO-CI CPU before
         run: |
           cat /tmp/eco-ci/cpu-util-total.txt
 
@@ -56,6 +58,15 @@ jobs:
           task: get-measurement
           label: "Sleep 3s"
 
+      - name: Dump ECO-CI CPU Step actual processed
+        run: |
+          cat /tmp/eco-ci/cpu-util-temp.txt
+
+      - name: Dump ECO-CI Energy Step actual processed
+        run: |
+          cat /tmp/eco-ci/energy-step.txt
+
+
       - name: Filesystem
         run: timeout 10s ls -alhR /usr/lib
         continue-on-error: true

.gitlab-ci.yml.example

@@ -2,28 +2,39 @@ image: ubuntu:22.04
 include:
   remote: 'https://raw.githubusercontent.com/green-coding-solutions/eco-ci-energy-estimation/main/eco-ci-gitlab.yml'
 
+# trigger rebuild
 stages:
   - test
 
-test-job:
-  stage: test 
+
+test-job:   # This job runs in the test stage.
+  stage: test    # It only starts when the job in the build stage completes successfully.
   script:
-    # Please input valid UUIDs here if you want to use CarbonDB (https://www.green-coding.io/projects/carbondb/)
-    # Generate one here for example: https://www.freecodeformat.com/validate-uuid-guid.php
     #- export ECO_CI_COMPANY_UUID="YOUR COMPANY UUID"
     #- export ECO_CI_PROJECT_UUID="YOUR PROJECT UUID"
+    #- export ECO_CI_MACHINE_UUID="YOUR MACHINE UUID"
+
+    - apt-get update
+    - apt-get install git jq curl -y # git, jq and curl are a depedency for Eco-CI as we need to clone the gitlab repo
 
-    - !reference [.initialize_energy_estimator, script]
     - !reference [.start_measurement, script]
-    - sleep 10s
-    - export ECO_CI_LABEL="measurement 1 (script)"
+    - sleep 2
+    - export ECO_CI_LABEL="Sleep 2"
+    - ls -alh /tmp/eco-ci
+    - echo "Step"
+    - cat /tmp/eco-ci/cpu-util-step.txt
     - !reference [.get_measurement, script]
-    - sleep 3s
-    - export ECO_CI_LABEL="measurement 2 (script)"
+    - echo "Total"
+    - cat /tmp/eco-ci/cpu-util-total.txt
+    - echo "Energy"
+    - cat /tmp/eco-ci/energy-step.txt
+
+    - timeout 10s grep -R xxx / 1> /dev/null 2>/dev/null || true
+    - export ECO_CI_LABEL="grep with timeout"
     - !reference [.get_measurement, script]
-    - export ECO_CI_SHOW_CARBON="true"
     - !reference [.display_results, script]
 
   artifacts:
     paths:
-      - eco-ci-output.txt
+      - eco-ci-output.txt
+      - metrics.txt

action.yml

@@ -12,44 +12,49 @@ inputs:
     description: 'Label for the get-measurement task, to mark what this measurement correlates to in your workflow'
     default: null
     required: false
+  machine-power-data:
+    description: 'The file to read the machine power data from. Default will be 4 core AMD EPYC 7763 Github Runner'
+    default: "github_EPYC_7763_4_CPU_shared.sh"
+    required: false
   send-data:
     description: 'Send metrics data to metrics.green-coding.io to create and display badge, and see an overview of the energy of your CI runs. Set to false to send no data.'
     default: true
     required: false
-  show-carbon:
+  calculate-co2:
     description: 'Uses the grid carbon intensity of the run location to estimate the emitted carbon for the measurements.'
     default: true
     required: false
   display-table:
     description: 'Show the energy reading results in a table during display-results step'
     default: true
     required: false
-  display-graph:
-    description: 'Show the graph of the energy use over time during display-results step'
-    default: true
-    required: false
   display-badge:
     description: 'Shows the badge for the ci run during display-results step'
     default: true
     required: false
   pr-comment:
     description: 'Add a comment to the PR with the results during display-results step'
     default: false
-  api-base:
+    required: false
+  json-output:
+    description: 'Output measurement data also as JSON artifacts'
+    default: false
+    required: false
+  gh-api-base:
       description: 'Base URL of the Github API to send data to. Default is api.github.com, but can be changed to your hostname if you have Github Enterprise'
-      default: 'api.github.com'
+      default: ${{ github.api_url }}
       required: false
   company-uuid:
       description: 'If you want to add data to the CarbonDB you can set this to your company UUID'
-      default: ''
+      default: '-'
       required: false
   project-uuid:
       description: 'If you want to add data to the CarbonDB you can set this to your project UUID'
-      default: ''
+      default: '-'
       required: false
   machine-uuid:
       description: 'If you want to add data to the CarbonDB you can set this to your machine UUID.'
-      default: ''
+      default: '-'
       required: false
 
 outputs:
@@ -70,87 +75,22 @@ runs:
           echo "Eco-CI task was called with ${{inputs.task}} instead" >> $GITHUB_STEP_SUMMARY
           fail('Invalid task name specified')
 
-    - id: initialize
-      if: inputs.task == 'start-measurement' && env.ECO_CI_INIT != 'DONE'
-      name: Setup
-      shell: bash
-      run: |
-        if command -v python3 &>/dev/null; then
-            echo "Python is already installed."
-        else
-            echo "Python is not installed. Installing..."
-            apt-get update
-            apt-get install -y python3.12 python3.12-venv || apt-get install -y python3.10 python3.10-venv
-            echo "Python has been installed."
-        fi
-
-        python_version=$(python3 --version 2>&1)
-        python_major_version=$(python3 -c 'import sys; print(sys.version_info[0])')
-        python_minor_version=$(python3 -c 'import sys; print(sys.version_info[1])')
-        python_cache_path="/tmp/eco-ci/venv/lib/python${python_major_version}.${python_minor_version}/site-packages"
-        echo "python_cache_path=$python_cache_path" >> $GITHUB_OUTPUT
-
-
-        # call the initialize function of setup.sh
-        ${{github.action_path}}/scripts/setup.sh initialize -g ${{inputs.display-graph}}
-
-      # To identify the hash for our cache we cannot use the classic mechansim of
-      # hashFiles('/tmp/eco-ci/spec-power-model/requirements.txt')
-      # hashFiles is restricted to ONLY work in the GITHUB_WORKSPACE which is for the calling action
-      # therefore we need to construct the hash ourselfs beforehand and save it to an output variable
-    - if:  inputs.task == 'start-measurement' && env.ECO_CI_INIT != 'DONE'
-      name: Hash requirements file
-      id: hash-requirements
-      shell: bash
-      run: echo "myhash=$(md5sum /tmp/eco-ci/spec-power-model/requirements.txt | cut -d ' ' -f1)" >> $GITHUB_OUTPUT;
-
-    - if:  inputs.task == 'start-measurement' && env.ECO_CI_INIT != 'DONE'
-      name: Cache pip packages
-      id: cache-pip
-      uses: actions/cache@v4
-      env:
-        cache-name: cache-pip-packages
-      with:
-        # npm cache files are stored in `~/.npm` on Linux/macOS
-        path: ${{ steps.initialize.outputs.python_cache_path }}
-        key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ steps.hash-requirements.outputs.myhash }}
-        restore-keys: |
-          ${{ runner.os }}-build-${{ env.cache-name }}-${{ steps.hash-requirements.outputs.myhash }}
-
-    - if: inputs.task == 'start-measurement' && env.ECO_CI_INIT != 'DONE' && steps.cache-pip.outputs.cache-hit == 'true'
-      name: Inform about cache hit
-      continue-on-error: true
-      shell: bash
-      run: |
-        echo "Cache hit succeeded! 😀"
-
-    - if: inputs.task == 'start-measurement' && env.ECO_CI_INIT != 'DONE' && steps.cache-pip.outputs.cache-hit != 'true'
-      name: Inform about cache hit
-      continue-on-error: true
-      shell: bash
-      run: |
-        echo "Cache hit failed! ❌"
-
-
-    - if:  inputs.task == 'start-measurement'
+    - id: start-measurement
+      if: inputs.task == 'start-measurement'
       name: Starting measurement
       shell: bash
-      # if measurement is started first time the reporter might not have run already
-      # we prefer this over manual startint / stopping as it is less error prone for users
       run: |
-        ${{github.action_path}}/scripts/setup.sh setup_python
+        workflow_id="${{github.workflow}}"
+        workflow_name="${{github.workflow}}"
 
         if ${{inputs.send-data}}; then
-          curl_response=$(curl -s -H "Authorization: Bearer ${{github.token}}" ${{ github.api_url }}/repos/${{ github.repository }}/actions/workflows)
-          workflow_id=$(echo $curl_response | jq '.workflows[] | select(.name == "${{ github.workflow }}") | .id')
-          ${{github.action_path}}/scripts/vars.sh add_var "WORKFLOW_ID" $workflow_id
-        else
-          ${{github.action_path}}/scripts/vars.sh add_var "WORKFLOW_ID" ${{github.workflow}}
+          workflow_id=$(curl -s -H "Authorization: Bearer ${{github.token}}" ${{ inputs.gh-api-base }}/repos/${{ github.repository }}/actions/runs/${{ github.run_id }} | jq -r .workflow_id)
         fi
 
-        ${{github.action_path}}/scripts/setup.sh start_measurement
-        echo "ECO_CI_INIT=DONE" >> $GITHUB_ENV;
+        workflow_id=${workflow_id:-"not_set"}
+        workflow_name=${workflow_name:-"not_set"}
 
+        ${{github.action_path}}/scripts/setup.sh start_measurement "${{inputs.machine-power-data}}" "${{ github.run_id }}" "${{inputs.branch}}" "${{ github.repository }}" "$workflow_id" "$workflow_name" "${{ github.sha }}" "github" "${{ inputs.send-data }}" "${{ inputs.company-uuid }}" "${{ inputs.project-uuid }}" "${{ inputs.machine-uuid }}" "${{ inputs.calculate-co2 }}" "${{ inputs.json-output }}"
 
     - if:  inputs.task == 'get-measurement'
       id: run-lap-model
@@ -159,28 +99,32 @@ runs:
         NAME: ${{ github.workflow }}
       shell: bash
       run: |
-        ${{github.action_path}}/scripts/make_measurement.sh -l "${{inputs.label}}" -r "${{ github.run_id }}" -b "${{inputs.branch}}" -R "${{ github.repository }}" -c ${{ github.sha }} -sd ${{inputs.send-data}} -s "github" -n "$NAME" -cbc "${{inputs.company-uuid}}" -cbp "${{inputs.project-uuid}}" -cbm "${{inputs.machine-uuid}}"
+        ${{github.action_path}}/scripts/make_measurement.sh make_measurement "${{inputs.label}}"
+        data_file="/tmp/test.bash"
         lap_data_file="/tmp/eco-ci/lap-data.json"
-        echo "data-lap-json=$(cat $lap_data_file)" >> $GITHUB_OUTPUT
-
+        if [[ -e $lap_data_file ]]; then
+          echo "data-lap-json=$(cat $lap_data_file)" >> $GITHUB_OUTPUT
+        fi
     - if: inputs.task == 'display-results'
       name: get estimation for total energy
       id: run-total-model
       shell: bash
       run: |
-        ${{github.action_path}}/scripts/display_results.sh -dt ${{inputs.display-table}} -dg ${{inputs.display-graph}} -db ${{inputs.display-badge}} -b "${{inputs.branch}}" -r ${{ github.run_id }} -R "${{ github.repository }}" -sd ${{inputs.send-data}} -sc ${{inputs.show-carbon}} -s "github"
+        ${{github.action_path}}/scripts/display_results.sh display_results "${{inputs.display-table}}" "${{inputs.display-badge}}"
         cat "/tmp/eco-ci/output.txt" >> $GITHUB_STEP_SUMMARY
         total_data_file="/tmp/eco-ci/total-data.json"
-        echo "data-total-json=$(cat $total_data_file)" >> $GITHUB_OUTPUT
+        if [[ -e $total_data_file ]]; then
+          echo "data-total-json=$(cat $total_data_file)" >> $GITHUB_OUTPUT
+        fi
 
-    - if: github.event_name == 'pull_request' && inputs.pr-comment == 'true'
+    - if: inputs.task == 'display-results' && inputs.pr-comment == 'true' && github.event_name == 'pull_request'
       name: Minimize Old Comment and Post New Comment
       id: pr-comment
       shell: bash
       env:
         PR_NUMBER: ${{ github.event.pull_request.number }}
       run: |
-        COMMENTS=$(curl -s -H  "Authorization: Bearer ${{github.token}}" "https://${{inputs.api-base}}/repos/${{ github.repository }}/issues/$PR_NUMBER/comments")
+        COMMENTS=$(curl -s -H  "Authorization: Bearer ${{github.token}}" "${{ inputs.gh-api-base }}/repos/${{ github.repository }}/issues/$PR_NUMBER/comments")
 
         echo "$COMMENTS" | jq -c --arg username "github-actions[bot]" '.[] | select(.user.login == $username and (.body | index("Eco-CI") // false))' | while read -r comment; do
             COMMENT_ID=$(echo "$comment" | jq -r '.id')
@@ -193,12 +137,12 @@ runs:
         $INNER_BODY
 
         </details>" '{"body": $body}')
-            curl -s -H "Authorization: Bearer ${{github.token}}" -X PATCH -d "$PAYLOAD" "https://${{inputs.api-base}}/repos/${{ github.repository }}/issues/comments/$COMMENT_ID"
+            curl -s -H "Authorization: Bearer ${{github.token}}" -X PATCH -d "$PAYLOAD" "${{ inputs.gh-api-base }}/repos/${{ github.repository }}/issues/comments/$COMMENT_ID"
             echo "Comment $COMMENT_ID collapsed."
         done
 
         NEW_COMMENT=$(cat "/tmp/eco-ci/output-pr.txt" | jq -Rs '.')
-        API_URL="https://${{inputs.api-base}}/repos/${{ github.repository }}/issues/${PR_NUMBER}/comments"
+        API_URL="${{ inputs.gh-api-base }}/repos/${{ github.repository }}/issues/${PR_NUMBER}/comments"
         curl -X POST -H "Authorization: Bearer ${{github.token}}" -d @- $API_URL <<EOF
         {
           "body": $NEW_COMMENT