From 0372acd2e7d0dfd20b61157423588fa4eb90b117 Mon Sep 17 00:00:00 2001 From: Arne Tarara Date: Wed, 17 Aug 2022 11:01:41 +0200 Subject: [PATCH] Version 0.5 release - Squashed commit of the following: commit b94d1cbbc9c66cac8901d0d79a7d1d63d4ac64ad Author: Arne Tarara Date: Tue Aug 16 16:47:45 2022 +0200 Differential moved to AVG. and disabled unused channels for now commit 5fcb5375e0b64c2abf163514caccf289053c4413 Author: Arne Tarara Date: Tue Aug 16 14:57:09 2022 +0200 Introduced measurement time as s in overview commit c4916d1ea37cd9c7cf338536f573f6975e268019 Author: Arne Tarara Date: Tue Aug 16 10:02:57 2022 +0200 setuid bit now used, so we can execute still with root rights but without calling sudo. chown for root in make. removed sudoers file usage; small fixes to printf error calls which were not on Stderr commit 16ef1de770e1e0b252b1918ee63ef5ac6a3841d3 Author: Arne Tarara Date: Mon Aug 15 19:43:09 2022 +0200 Added cpu_utilization_procfs_system_provider commit d44950f6804d9eb7ab2a83f65cd6a629c72cc0c9 Author: Arne Tarara Date: Mon Aug 15 19:41:08 2022 +0200 Renamed cpu_time_system provider to cpu_time_procfs_system commit 25330833b5789b6f8fd11f8fee99b15b6405f9c0 Merge: 4676f71 805016e Author: Arne Tarara Date: Mon Aug 15 15:40:23 2022 +0200 Merge branch 'main' into dev * main: Adding missing time and atx metrics to lookup Release 1.0: Squashed commit of the following: commit 4676f713f0d59bd7d88d54fe9ecee4dbcf489b7b Author: dan-mm <33732895+dan-mm@users.noreply.github.com> Date: Mon Aug 15 15:26:51 2022 +0200 Bugfixes rapl c improvements (#19) * small refactor of code; use enum for measurement modes; moved all setup call outside of main measurement loop * get userid dynamically; makefile now chmod +s on static-binary * changed enum to #define for measurement mode * missed a variable during refactor * implemented user_id fix to the rest of the metric providers which use cgroups Co-authored-by: Dan M commit 79bb522224769683519d7f0602ced13b7ff2d3b3 Author: Arne Tarara Date: Sat Aug 13 14:37:22 2022 +0200 Metrics have now dimension added in their chart displays commit ab2dc175fc9631d768be7b110a78e9170e34ee2e Author: Arne Tarara Date: Sat Aug 13 14:03:36 2022 +0200 RAM reporter renamed to Memory commit 66c68307017803126728dc08429266c15d0dc562 Author: Arne Tarara Date: Sat Aug 13 13:55:40 2022 +0200 Explanations added for network formulas and accounting commit ae2e6151847e6e4773cf378a37e95317444a205c Author: Arne Tarara Date: Sat Aug 13 13:55:23 2022 +0200 Time reporters inactive by default commit f590ec4a0476a5f15718d19ae43d005a1c30ff86 Author: Arne Tarara Date: Sat Aug 13 12:37:00 2022 +0200 Network IO was not correctly added up, but only last container was accounted for commit 6a3038834f1e15741344cef4c2e8ef8f9b86fcc4 Author: Arne Tarara Date: Sat Aug 13 11:08:18 2022 +0200 CLI is now very pretty commit f483dd9c7a9ab354eb96c10f94c886d6f29ba148 Author: Arne Tarara Date: Fri Aug 12 20:23:57 2022 +0200 Added all channels to importer commit ab486a4d5095086544fb6429a70b43bc84a771e3 Author: Arne Tarara Date: Fri Aug 12 13:35:08 2022 +0200 DC converter now operates with mV and new measurement resistor of 0.005 W commit e455ce160da703b90906696e199d9b65f40e60fe Author: Arne Tarara Date: Fri Aug 12 12:47:38 2022 +0200 Added Watt power reading and renamed atx channel with DC added commit ed9752e63774192bd23bb2e80e584e164ad9f1d5 Author: Arne Tarara Date: Fri Aug 12 12:31:51 2022 +0200 Moved dc_converter to mV commit 448ea1e49112cd77969ac3ac8ad3008a3fea7c56 Author: Arne Tarara Date: Fri Aug 12 12:30:09 2022 +0200 Clarifying comment commit e25c2733c588634bce9bde08f38bbed9d0545f94 Author: Arne Tarara Date: Sun Aug 7 22:42:44 2022 +0200 Removed obsolte TODOs and comments commit a895f7393dd8a9da68e555778d130078466f4ad5 Author: Arne Tarara Date: Sun Aug 7 22:41:22 2022 +0200 Clarifying comment commit a58c3ebf8951f7a75b43892c9ccb02b54d4738c5 Author: Arne Tarara Date: Sun Aug 7 15:12:47 2022 +0200 Introduced typo commit 4100461754a00f55e455a4444ea27c6d7bafc53f Author: Arne Tarara Date: Sun Aug 7 15:10:47 2022 +0200 also renamed field to MB commit 65a242cc6cc9477b63e6528806769d8d8d105bea Author: Arne Tarara Date: Sun Aug 7 15:09:51 2022 +0200 Network IO moved to MB commit f7a50689bbdbc5b7ad9052cb24bf5025fde7f19e Author: Arne Tarara Date: Sun Aug 7 15:09:01 2022 +0200 ATX is now using channel as identifier commit 573d199cd34291272891ec78a3b5fb76a0a2cf61 Author: Arne Tarara Date: Sun Aug 7 14:48:33 2022 +0200 DC measurement converter and display added --- config.yml.example | 7 +- frontend/js/stats.js | 43 ++++-- frontend/stats.html | 75 +++++++--- install.sh | 25 ---- tools/dc_converter.py | 36 ++--- .../cpu/energy/RAPL/MSR/system/Makefile | 4 +- .../cpu/energy/RAPL/MSR/system/provider.py | 2 +- .../cpu/energy/RAPL/MSR/system/source.c | 139 +++++++++++------- .../cpu/time/cgroup/container/source.c | 7 +- .../cpu/time/{ => procfs}/system/Makefile | 0 .../cpu/time/{ => procfs}/system/provider.py | 8 +- .../cpu/time/{ => procfs}/system/source.c | 0 .../cpu/utilization/cgroup/container/source.c | 7 +- .../cpu/utilization/procfs/system/Makefile | 4 + .../cpu/utilization/procfs/system/provider.py | 28 ++++ .../cpu/utilization/procfs/system/source.c | 128 ++++++++++++++++ .../memory/energy/RAPL/MSR/system/Makefile | 4 +- .../memory/energy/RAPL/MSR/system/provider.py | 2 +- .../memory/total/cgroup/container/source.c | 7 +- .../network/io/cgroup/container/Makefile | 4 +- .../network/io/cgroup/container/provider.py | 2 +- .../network/io/cgroup/container/source.c | 11 +- 22 files changed, 378 insertions(+), 165 deletions(-) rename tools/metric_providers/cpu/time/{ => procfs}/system/Makefile (100%) rename tools/metric_providers/cpu/time/{ => procfs}/system/provider.py (76%) rename tools/metric_providers/cpu/time/{ => procfs}/system/source.c (100%) create mode 100644 tools/metric_providers/cpu/utilization/procfs/system/Makefile create mode 100644 tools/metric_providers/cpu/utilization/procfs/system/provider.py create mode 100644 tools/metric_providers/cpu/utilization/procfs/system/source.c diff --git a/config.yml.example b/config.yml.example index 7c31ac72b..e59a4886b 100644 --- a/config.yml.example +++ b/config.yml.example @@ -23,12 +23,13 @@ measurement: metric-providers: cpu.utilization.cgroup.container.provider.CpuUtilizationCgroupContainerProvider: 100 cpu.energy.RAPL.MSR.system.provider.CpuEnergyRaplMsrSystemProvider: 100 - memory.total.cgroup.container.provider.MemoryTotalCgroupContainerProvider: 100 + cpu.utilization.procfs.system.provider.CpuUtilizationProcfsSystemProvider: 100 # cpu.time.cgroup.container.provider.CpuTimeCgroupContainerProvider: 100 # cpu.time.cgroup.system.provider.CpuTimeCgroupSystemProvider: 100 -# cpu.time.system.provider.CpuTimeSystemProvider: 100 - network.io.cgroup.container.provider.NetworkIoCgroupContainerProvider: 100 +# cpu.time.procfs.system.provider.CpuTimeProcfsSystemProvider: 100 + memory.total.cgroup.container.provider.MemoryTotalCgroupContainerProvider: 100 memory.energy.RAPL.MSR.system.provider.MemoryEnergyRaplMsrSystemProvider: 100 + network.io.cgroup.container.provider.NetworkIoCgroupContainerProvider: 100 admin: # This address will get an email, when a new project was added through the frontend diff --git a/frontend/js/stats.js b/frontend/js/stats.js index 5e1971abb..015e184e0 100644 --- a/frontend/js/stats.js +++ b/frontend/js/stats.js @@ -4,6 +4,11 @@ const metrics_info = { SI_conversion_factor: 100, // CPU comes as ratio, but since stored as integer is was multiplicated with 100 unit_after_conversion: '%' }, + cpu_utilization_procfs_system: { + unit: 'Ratio', + SI_conversion_factor: 100, // CPU comes as ratio, but since stored as integer is was multiplicated with 100 + unit_after_conversion: '%' + }, cpu_energy_rapl_msr_system: { unit: 'mJ', SI_conversion_factor: 1000, @@ -39,7 +44,7 @@ const metrics_info = { SI_conversion_factor: 1, unit_after_conversion: 'us' }, - cpu_time_system: { + cpu_time_procfs_system: { unit: 'us', SI_conversion_factor: 1, unit_after_conversion: 'us' @@ -122,7 +127,6 @@ const getEChartsOptions = () => { } const fillProjectData = (project, key = null) => { - for (item in project) { if (item == 'machine_specs') { fillProjectTab('#machine-specs', project[item]) @@ -132,6 +136,7 @@ const fillProjectData = (project, key = null) => { } else if(item == 'measurement_config') { fillProjectTab('#measurement-config', project[item]) } else { + document.querySelector('#project-data').insertAdjacentHTML('beforeend', `${item}${project?.[item]}`) } } @@ -148,7 +153,7 @@ const fillProjectTab = (selector, data) => { } const getMetrics = (stats_data, style='apex') => { - const metrics = {cpu_load: [], mem_total: [], network_io: {}, series: {}, atx_energy: 0, cpu_energy: 0, memory_energy: 0} + const metrics = {cpu_utilization_containers: [], cpu_utilization_system: [], mem_total: [], network_io: {}, series: {}, atx_energy: 0, cpu_energy: 0, memory_energy: 0} let accumulate = 0; @@ -172,7 +177,9 @@ const getMetrics = (stats_data, style='apex') => { value = value / metrics_info[metric_name].SI_conversion_factor; if (metric_name == 'cpu_utilization_cgroup_container') { - if (accumulate === 1) metrics.cpu_load.push(value); + if (accumulate === 1) metrics.cpu_utilization_containers.push(value); + } else if (metric_name == 'cpu_utilization_procfs_system') { + if (accumulate === 1) metrics.cpu_utilization_system.push(value); } else if (metric_name == 'cpu_energy_rapl_msr_system') { if (accumulate === 1) metrics.cpu_energy += value; } else if (metric_name == 'atx_energy_dc_channel') { @@ -338,9 +345,6 @@ const createGraph = (element, data, labels, title) => { const fillAvgContainers = (stats_data, metrics) => { - // timestamp is in microseconds, therefore divide by 10**6 - const measurement_duration_in_s = (stats_data.project.end_measurement - stats_data.project.start_measurement) / 1000000; - const atx_energy_in_mWh = ((metrics.atx_energy) / 3600) * 1000; const cpu_energy_in_mWh = ((metrics.cpu_energy) / 3600) * 1000; const memory_energy_in_mWh = ((metrics.memory_energy) / 3600) * 1000; @@ -367,21 +371,25 @@ const fillAvgContainers = (stats_data, metrics) => { if(memory_energy_in_mWh) document.querySelector("#memory-energy").innerText = memory_energy_in_mWh.toFixed(2) + " mWh" if(cpu_energy_in_mWh) document.querySelector("#total-energy").innerText = (cpu_energy_in_mWh+memory_energy_in_mWh+network_io_in_mWh).toFixed(2) + " mWh" - if(cpu_energy_in_mWh) document.querySelector("#component-power").innerText = ((metrics.cpu_energy+metrics.memory_energy)/measurement_duration_in_s).toFixed(2) + " W" - if(atx_energy_in_mWh) document.querySelector("#atx-power").innerText = (metrics.atx_energy / measurement_duration_in_s).toFixed(2) + " W" + if(cpu_energy_in_mWh) document.querySelector("#component-power").innerText = ((metrics.cpu_energy+metrics.memory_energy)/stats_data.project.measurement_duration_in_s).toFixed(2) + " W" + if(atx_energy_in_mWh) document.querySelector("#atx-power").innerText = (metrics.atx_energy / stats_data.project.measurement_duration_in_s).toFixed(2) + " W" if(network_io) document.querySelector("#network-io").innerText = network_io.toFixed(2) + " MB" if(network_io_in_mWh) document.querySelector("#network-energy").innerHTML = network_io_in_mWh.toFixed(2) + " mWh" - if(co2_display.value) document.querySelector("#total-co2-internal").innerHTML = `${(co2_display.value).toFixed(2)} ${co2_display.unit}` - if(co2_budget_utilization) document.querySelector("#co2-budget-utilization").innerHTML = (co2_budget_utilization).toFixed(2) + " %" + if (co2_display.value) document.querySelector("#total-co2-internal").innerHTML = `${(co2_display.value).toFixed(2)} ${co2_display.unit}` + if (co2_budget_utilization) document.querySelector("#co2-budget-utilization").innerHTML = (co2_budget_utilization).toFixed(2) + " %" - if(metrics.cpu_load.length) { - document.querySelector("#max-cpu-load").innerText = (Math.max.apply(null, metrics.cpu_load)) + " %" - document.querySelector("#avg-cpu-load").innerText = ((metrics.cpu_load.reduce((a, b) => a + b, 0) / metrics.cpu_load.length)).toFixed(2) + " %" + if (metrics.cpu_utilization_containers.length) { + document.querySelector("#max-cpu-load-containers").innerText = (Math.max.apply(null, metrics.cpu_utilization_containers)) + " %" + document.querySelector("#avg-cpu-load-containers").innerText = ((metrics.cpu_utilization_containers.reduce((a, b) => a + b, 0) / metrics.cpu_utilization_containers.length)).toFixed(2) + " %" + } + if (metrics.cpu_utilization_system.length) { + document.querySelector("#max-cpu-load-system").innerText = (Math.max.apply(null, metrics.cpu_utilization_system)) + " %" + document.querySelector("#avg-cpu-load-system").innerText = ((metrics.cpu_utilization_system.reduce((a, b) => a + b, 0) / metrics.cpu_utilization_system.length)).toFixed(2) + " %" } - if(metrics.mem_total.length) document.querySelector("#avg-mem-load").innerText = ((metrics.mem_total.reduce((a, b) => a + b, 0) / metrics.mem_total.length)).toFixed(2) + " MB" + if (metrics.mem_total.length) document.querySelector("#avg-mem-load").innerText = ((metrics.mem_total.reduce((a, b) => a + b, 0) / metrics.mem_total.length)).toFixed(2) + " MB" upscaled_CO2_in_kg = total_CO2_in_kg * 100 * 30 ; // upscaled by 30 days for 10.000 requests (or runs) per day @@ -411,6 +419,11 @@ $(document).ready( (e) => { $('.ui.secondary.menu .item').tab(); const metrics = getMetrics(stats_data, 'echarts'); + + // create new custom field + // timestamp is in microseconds, therefore divide by 10**6 + stats_data.project['measurement_duration_in_s'] = (stats_data.project?.end_measurement - stats_data.project?.start_measurement) / 1000000 + fillProjectData(stats_data.project) displayGraphs(metrics.series, notes_json.data, 'echarts'); fillAvgContainers(stats_data, metrics); diff --git a/frontend/stats.html b/frontend/stats.html index 537fe788c..1706bb5e3 100644 --- a/frontend/stats.html +++ b/frontend/stats.html @@ -88,6 +88,7 @@

Project Data

+

container level metrics

@@ -95,9 +96,10 @@

Project Data

- N/A + N/A
+
all containers
@@ -107,9 +109,10 @@

Project Data

- N/A + N/A
+
all containers
@@ -138,40 +141,43 @@

Project Data

+ +

system level metrics

+
-
CPU Energy
-
Intel RAPL CPU Package
+
AVG CPU Load
- N/A + N/A
+
system
-
Memory Energy
-
Intel RAPL DRAM
+
Max. CPU Load
- N/A + N/A
+
system
-
Component Energy
-
RAPL CPU+Memory
+
CPU Energy
+
Intel RAPL CPU Package
- N/A + N/A
@@ -179,12 +185,12 @@

Project Data

-
Component Power (avg.)
-
RAPL CPU+Memory
+
Memory Energy
+
Intel RAPL DRAM
- N/A + N/A
@@ -204,14 +210,17 @@

Project Data

+
+

compound metrics

+
-
Total Energy
-
CPU + Memory + Network
+
Component Energy
+
RAPL CPU+Memory
- N/A + N/A
@@ -219,12 +228,25 @@

Project Data

-
Total co2
+
Component Power (avg.)
+
RAPL CPU+Memory
+
+
+
+ N/A +
+
+
+
+
+
+
+
Total Energy
CPU + Memory + Network
- N/A + N/A
@@ -256,8 +278,21 @@

Project Data

+
+
+
Total co2
+
CPU + Memory + Network
+
+
+
+ N/A +
+
+
+
+
+
-
diff --git a/install.sh b/install.sh index 3fd64797e..8d6c7f6da 100755 --- a/install.sh +++ b/install.sh @@ -24,37 +24,12 @@ while IFS= read -r subdir; do echo "Installing $subdir/static-binary ..." rm -f $subdir/static-binary 2> /dev/null make -C $subdir - chmod +x $subdir/static-binary fi done -sudo_line="$USER ALL=(ALL) NOPASSWD: $PWD/tools/metric_providers/cpu/energy/RAPL/MSR/system/static-binary -i *" -sudo_line_2="$USER ALL=(ALL) NOPASSWD: $PWD/tools/metric_providers/memory/energy/RAPL/MSR/system/static-binary -i * -d" -sudo_line_3="$USER ALL=(ALL) NOPASSWD: $PWD/tools/metric_providers/network/io/cgroup/container/static-binary -i * -s *" - etc_hosts_line_1="127.0.0.1 green-coding-postgres-container" etc_hosts_line_2="127.0.0.1 api.green-coding.local metrics.green-coding.local" -echo "Writing to /etc/sudoers file..." -if ! sudo grep -Fxq "$sudo_line" /etc/sudoers; then - echo "$sudo_line" | sudo tee -a /etc/sudoers -else - echo "Entry was already present..." -fi - -if ! sudo grep -Fxq "$sudo_line_2" /etc/sudoers; then - echo "$sudo_line_2" | sudo tee -a /etc/sudoers -else - echo "Entry was already present..." -fi - -if ! sudo grep -Fxq "$sudo_line_3" /etc/sudoers; then - echo "$sudo_line_3" | sudo tee -a /etc/sudoers -else - echo "Entry was already present..." -fi - - echo "Writing to /etc/hosts file..." if ! sudo grep -Fxq "$etc_hosts_line_1" /etc/hosts; then echo "$etc_hosts_line_1" | sudo tee -a /etc/hosts diff --git a/tools/dc_converter.py b/tools/dc_converter.py index 880338116..e3395639b 100644 --- a/tools/dc_converter.py +++ b/tools/dc_converter.py @@ -17,27 +17,15 @@ def main(args): df = df.rename({"Unnamed: 0": "time"}, axis=1) - - df = df[['time', - 'Differential 1 - 2 Last (mV)', - 'Differential 3 - 4 Last (mV)', - 'Differential 5 - 6 Last (mV)', - 'Differential 7 - 8 Last (mV)', - 'Differential 9 - 10 Last (mV)', - 'Differential 11 - 12 Last (mV)', - 'Differential 13 - 14 Last (mV)', - 'Differential 15 - 16 Last (mV)',]] - df = df.rename({ - 'Differential 1 - 2 Last (mV)': 'ch_1_12V', - 'Differential 3 - 4 Last (mV)': 'ch_3_12V', - 'Differential 5 - 6 Last (mV)': 'ch_5_12V', - 'Differential 5 - 6 Last (mV)': 'ch_5_12V', - 'Differential 7 - 8 Last (mV)': 'ch_7_12V', - 'Differential 9 - 10 Last (mV)': 'ch_9_12V', - 'Differential 11 - 12 Last (mV)': 'ch_11_12V', - 'Differential 13 - 14 Last (mV)': 'ch_13_12V', - 'Differential 15 - 16 Last (mV)': 'ch_15_12V' + 'Differential 1 - 2 Ave. (mV)': 'ch_1_12V', + 'Differential 3 - 4 Ave. (mV)': 'ch_3_12V', + 'Differential 5 - 6 Ave. (mV)': 'ch_5_12V', + 'Differential 7 - 8 Ave. (mV)': 'ch_7_12V', + 'Differential 9 - 10 Ave. (mV)': 'ch_9_12V', + 'Differential 11 - 12 Ave. (mV)': 'ch_11_12V', +# 'Differential 13 - 14 Ave. (mV)': 'ch_13_12V', +# 'Differential 15 - 16 Ave. (mV)': 'ch_15_12V', }, axis=1) # bring timestamp to our used microsecond format @@ -53,12 +41,14 @@ def main(args): # Then multiply with constant voltage 12 V to get Power. # Then multiply with measurement_interval to get Joules # Then multiply by 10**3 to get millijoules + df.ch_1_12V = ((df.ch_1_12V / 1000) / 0.005) * 12 * measurement_interval * 10**3 + df.ch_3_12V = ((df.ch_3_12V / 1000) / 0.005) * 12 * measurement_interval * 10**3 df.ch_5_12V = ((df.ch_5_12V / 1000) / 0.005) * 12 * measurement_interval * 10**3 df.ch_7_12V = ((df.ch_7_12V / 1000) / 0.005) * 12 * measurement_interval * 10**3 df.ch_9_12V = ((df.ch_9_12V / 1000) / 0.005) * 12 * measurement_interval * 10**3 df.ch_11_12V = ((df.ch_11_12V / 1000) / 0.005) * 12 * measurement_interval * 10**3 - df.ch_13_12V = ((df.ch_13_12V / 1000) / 0.005) * 12 * measurement_interval * 10**3 - df.ch_15_12V = ((df.ch_15_12V / 1000) / 0.005) * 12 * measurement_interval * 10**3 +# df.ch_13_12V = ((df.ch_13_12V / 1000) / 0.005) * 12 * measurement_interval * 10**3 +# df.ch_15_12V = ((df.ch_15_12V / 1000) / 0.005) * 12 * measurement_interval * 10**3 df = df.astype(int) @@ -90,4 +80,4 @@ def main(args): args = parser.parse_args() - main(args) \ No newline at end of file + main(args) diff --git a/tools/metric_providers/cpu/energy/RAPL/MSR/system/Makefile b/tools/metric_providers/cpu/energy/RAPL/MSR/system/Makefile index 45c978065..b081b99f0 100644 --- a/tools/metric_providers/cpu/energy/RAPL/MSR/system/Makefile +++ b/tools/metric_providers/cpu/energy/RAPL/MSR/system/Makefile @@ -1,4 +1,6 @@ CFLAGS = -o3 -Wall -lm -static -static-libgcc static-binary: source.c - gcc $< $(CFLAGS) -o $@ \ No newline at end of file + gcc $< $(CFLAGS) -o $@ + sudo chown root $@ + sudo chmod u+s $@ \ No newline at end of file diff --git a/tools/metric_providers/cpu/energy/RAPL/MSR/system/provider.py b/tools/metric_providers/cpu/energy/RAPL/MSR/system/provider.py index 429636a80..96c279c38 100644 --- a/tools/metric_providers/cpu/energy/RAPL/MSR/system/provider.py +++ b/tools/metric_providers/cpu/energy/RAPL/MSR/system/provider.py @@ -9,7 +9,7 @@ def __init__(self, resolution): self._metric_name = "cpu_energy_rapl_msr_system" self._metrics = {"time":int, "value":int} self._resolution = resolution - super().__init__(sudo=True) + super().__init__() if __name__ == "__main__": import time diff --git a/tools/metric_providers/cpu/energy/RAPL/MSR/system/source.c b/tools/metric_providers/cpu/energy/RAPL/MSR/system/source.c index c78c39fb2..6089a352c 100644 --- a/tools/metric_providers/cpu/energy/RAPL/MSR/system/source.c +++ b/tools/metric_providers/cpu/energy/RAPL/MSR/system/source.c @@ -36,7 +36,6 @@ #include #include #include -#include /* AMD Support */ @@ -206,7 +205,7 @@ static int detect_cpu(void) { if (vendor==CPU_VENDOR_INTEL) { if (family!=6) { - printf("Wrong CPU family %d\n",family); + fprintf(stderr, "Wrong CPU family %d\n",family); return -1; } @@ -222,7 +221,7 @@ static int detect_cpu(void) { msr_pp0_energy_status=MSR_AMD_PP0_ENERGY_STATUS; if (family!=23) { - printf("Wrong CPU family %d\n",family); + fprintf(stderr, "Wrong CPU family %d\n",family); return -1; } model=CPU_AMD_FAM17H; @@ -266,25 +265,18 @@ static int detect_packages(void) { return 0; } -/*******************************/ -/* MSR code */ -/*******************************/ +#define MEASURE_ENERGY_PKG 1 +#define MEASURE_DRAM 2 -static int rapl_msr(int cpu_model, bool measure_energy_pkg, bool measure_dram) { +int dram_avail=0; +int different_units=0; +double cpu_energy_units[MAX_PACKAGES],dram_energy_units[MAX_PACKAGES]; +unsigned int energy_status; +double energy_units[MAX_PACKAGES]; - int fd; - long long result; - double power_units,time_units; - double cpu_energy_units[MAX_PACKAGES],dram_energy_units[MAX_PACKAGES]; - double package_before[MAX_PACKAGES],package_after[MAX_PACKAGES]; - double dram_before[MAX_PACKAGES],dram_after[MAX_PACKAGES]; - int j; - struct timeval now; - - int dram_avail=0; - int different_units=0; +static int check_availability(int cpu_model, int measurement_mode) { - if(measure_dram) { + if(measurement_mode == MEASURE_DRAM){ switch(cpu_model) { case CPU_SANDYBRIDGE_EP: case CPU_IVYBRIDGE_EP: @@ -342,7 +334,7 @@ static int rapl_msr(int cpu_model, bool measure_energy_pkg, bool measure_dram) { } } - if(measure_dram && !dram_avail) { + if(measurement_mode == MEASURE_DRAM && !dram_avail) { fprintf(stderr,"DRAM not available for your processer.\n"); exit(-1); } @@ -352,6 +344,14 @@ static int rapl_msr(int cpu_model, bool measure_energy_pkg, bool measure_dram) { exit(-1); } + + return 0; +} + +static int setup_measurement_units(int measurement_mode) { + int fd; + int j; + long long result; for(j=0;j energy status units // 16-19 -> time units // 4-7, 13-15, and 20-63 are all reserved bits - power_units=pow(0.5,(double)(result&0xf)); //multiplying by 0xf will give you the first 4 bits - cpu_energy_units[j]=pow(0.5,(double)((result>>8)&0x1f)); //multiplying by 0x1f will give you the first 5 bits + //power_units and time_units are not actually used... should we be using them? + //power_units=pow(0.5,(double)(result&0xf)); //multiplying by 0xf will give you the first 4 bits + //time_units=pow(0.5,(double)((result>>16)&0xf)); - time_units=pow(0.5,(double)((result>>16)&0xf)); + cpu_energy_units[j]=pow(0.5,(double)((result>>8)&0x1f)); //multiplying by 0x1f will give you the first 5 bits - if(measure_dram && different_units) { + if(measurement_mode == MEASURE_DRAM && different_units) { dram_energy_units[j]=pow(0.5,(double)16); } - else if (measure_dram && !different_units) { + else if (measurement_mode == MEASURE_DRAM && !different_units) { dram_energy_units[j]=cpu_energy_units[j]; } close(fd); + } + for(j=0;j=0) { + gettimeofday(&now, NULL); + printf("%ld%06ld %ld\n", now.tv_sec, now.tv_usec, (long int)(energy_output*1000)); + } + /* + else { + fprintf(stderr, "Energy reading had unexpected value: %f", energy_output); + exit(-1); + }*/ close(fd); } @@ -429,7 +459,7 @@ int main(int argc, char **argv) { int c; int cpu_model; - bool measure_dram=false; + int measure_mode = MEASURE_ENERGY_PKG; while ((c = getopt (argc, argv, "hi:d")) != -1) { switch (c) { @@ -443,7 +473,7 @@ int main(int argc, char **argv) { msleep_time = atoi(optarg); break; case 'd': - measure_dram=true; + measure_mode=MEASURE_DRAM; break; default: fprintf(stderr,"Unknown option %c\n",c); @@ -455,11 +485,10 @@ int main(int argc, char **argv) { cpu_model=detect_cpu(); detect_packages(); - + check_availability(cpu_model, measure_mode); + setup_measurement_units(measure_mode); while(1) { - // For now, either measure energy-pkg, or dram, but not both - // this is based on whether the -d flag was passed in - rapl_msr(cpu_model, !measure_dram, measure_dram); + rapl_msr(); } return 0; diff --git a/tools/metric_providers/cpu/time/cgroup/container/source.c b/tools/metric_providers/cpu/time/cgroup/container/source.c index 597943ddf..52f9ac896 100644 --- a/tools/metric_providers/cpu/time/cgroup/container/source.c +++ b/tools/metric_providers/cpu/time/cgroup/container/source.c @@ -16,7 +16,7 @@ typedef struct container_t { // struct is a specification and this static makes // between Threads. // TODO: If this code ever gets multi-threaded please review this assumption to // not pollute another threads state -static char *user_id = "1000"; //TODO: Figure out user_id dynamically, or request +static int user_id = 0; static long int user_hz; static unsigned int msleep_time=1000; static container_t *containers = NULL; @@ -51,6 +51,7 @@ int main(int argc, char **argv) { setvbuf(stdout, NULL, _IONBF, 0); user_hz = sysconf(_SC_CLK_TCK); + user_id = getuid(); while ((c = getopt (argc, argv, "i:s:h")) != -1) { switch (c) { @@ -82,7 +83,7 @@ int main(int argc, char **argv) { containers = realloc(containers, length * sizeof(container_t)); containers[length-1].id = id; sprintf(containers[length-1].path, - "/sys/fs/cgroup/user.slice/user-%s.slice/user@%s.service/user.slice/docker-%s.scope/cpu.stat", + "/sys/fs/cgroup/user.slice/user-%d.slice/user@%d.service/user.slice/docker-%s.scope/cpu.stat", user_id, user_id, id); FILE* fd = NULL; @@ -102,7 +103,7 @@ int main(int argc, char **argv) { } if(containers == NULL) { - printf("Please supply at least one container id with -s XXXX\n"); + fprintf(stderr, "Please supply at least one container id with -s XXXX\n"); exit(1); } diff --git a/tools/metric_providers/cpu/time/system/Makefile b/tools/metric_providers/cpu/time/procfs/system/Makefile similarity index 100% rename from tools/metric_providers/cpu/time/system/Makefile rename to tools/metric_providers/cpu/time/procfs/system/Makefile diff --git a/tools/metric_providers/cpu/time/system/provider.py b/tools/metric_providers/cpu/time/procfs/system/provider.py similarity index 76% rename from tools/metric_providers/cpu/time/system/provider.py rename to tools/metric_providers/cpu/time/procfs/system/provider.py index 22fbdf107..eefae8b04 100644 --- a/tools/metric_providers/cpu/time/system/provider.py +++ b/tools/metric_providers/cpu/time/procfs/system/provider.py @@ -1,12 +1,12 @@ import sys, os if __name__ == "__main__": - sys.path.append(os.path.dirname(os.path.abspath(__file__))+'/../..') + sys.path.append(os.path.dirname(os.path.abspath(__file__))+'/../../..') from metric_providers.base import BaseMetricProvider -class CpuTimeSystemProvider(BaseMetricProvider): +class CpuTimeProcfsSystemProvider(BaseMetricProvider): def __init__(self,resolution): self._current_dir = os.path.dirname(os.path.abspath(__file__)) - self._metric_name = "cpu_time_system" + self._metric_name = "cpu_time_procfs_system" self._metrics = {"time":int, "value":int} self._resolution = resolution super().__init__() @@ -14,7 +14,7 @@ def __init__(self,resolution): if __name__ == "__main__": import time - o = CpuTimeSystemProvider(resolution=100) + o = CpuTimeProcfsSystemProvider(resolution=100) print("Starting to profile") o.start_profiling() diff --git a/tools/metric_providers/cpu/time/system/source.c b/tools/metric_providers/cpu/time/procfs/system/source.c similarity index 100% rename from tools/metric_providers/cpu/time/system/source.c rename to tools/metric_providers/cpu/time/procfs/system/source.c diff --git a/tools/metric_providers/cpu/utilization/cgroup/container/source.c b/tools/metric_providers/cpu/utilization/cgroup/container/source.c index dd8ddd3dc..809c54f50 100644 --- a/tools/metric_providers/cpu/utilization/cgroup/container/source.c +++ b/tools/metric_providers/cpu/utilization/cgroup/container/source.c @@ -16,7 +16,7 @@ typedef struct container_t { // struct is a specification and this static makes // between Threads. // TODO: If this code ever gets multi-threaded please review this assumption to // not pollute another threads state -static char *user_id = "1000"; //TODO: Figure out user_id dynamically, or request +static int user_id = 0; static long int user_hz; static unsigned int msleep_time=1000; static container_t *containers = NULL; @@ -127,6 +127,7 @@ int main(int argc, char **argv) { setvbuf(stdout, NULL, _IONBF, 0); user_hz = sysconf(_SC_CLK_TCK); + user_id = getuid(); while ((c = getopt (argc, argv, "i:s:h")) != -1) { switch (c) { @@ -158,7 +159,7 @@ int main(int argc, char **argv) { containers = realloc(containers, length * sizeof(container_t)); containers[length-1].id = id; sprintf(containers[length-1].path, - "/sys/fs/cgroup/user.slice/user-%s.slice/user@%s.service/user.slice/docker-%s.scope/cpu.stat", + "/sys/fs/cgroup/user.slice/user-%d.slice/user@%d.service/user.slice/docker-%s.scope/cpu.stat", user_id, user_id, id); FILE* fd = NULL; @@ -178,7 +179,7 @@ int main(int argc, char **argv) { } if(containers == NULL) { - printf("Please supply at least one container id with -s XXXX\n"); + fprintf(stderr, "Please supply at least one container id with -s XXXX\n"); exit(1); } diff --git a/tools/metric_providers/cpu/utilization/procfs/system/Makefile b/tools/metric_providers/cpu/utilization/procfs/system/Makefile new file mode 100644 index 000000000..d6bdeba98 --- /dev/null +++ b/tools/metric_providers/cpu/utilization/procfs/system/Makefile @@ -0,0 +1,4 @@ +CFLAGS = -o3 -Wall -static -static-libgcc + +static-binary: source.c + gcc $< $(CFLAGS) -o $@ \ No newline at end of file diff --git a/tools/metric_providers/cpu/utilization/procfs/system/provider.py b/tools/metric_providers/cpu/utilization/procfs/system/provider.py new file mode 100644 index 000000000..84d243824 --- /dev/null +++ b/tools/metric_providers/cpu/utilization/procfs/system/provider.py @@ -0,0 +1,28 @@ +import sys, os +if __name__ == "__main__": + sys.path.append(os.path.dirname(os.path.abspath(__file__))+'/../../..') +from metric_providers.base import BaseMetricProvider + +class CpuUtilizationProcfsSystemProvider(BaseMetricProvider): + def __init__(self, resolution): + self._current_dir = os.path.dirname(os.path.abspath(__file__)) + self._metric_name = "cpu_utilization_procfs_system" + self._metrics = {"time":int, "value":int} + self._resolution = resolution + super().__init__() + +if __name__ == "__main__": + import time + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("container_id", help="Please provide the container_id") + args = parser.parse_args() + + o = CpuUtilizationProcfsSystemProvider(resolution=100) + + print("Starting to profile") + o.start_profiling({args.container_id: "test"}) + time.sleep(2) + o.stop_profiling() + print("Done, check ", o._filename) diff --git a/tools/metric_providers/cpu/utilization/procfs/system/source.c b/tools/metric_providers/cpu/utilization/procfs/system/source.c new file mode 100644 index 000000000..77db82e06 --- /dev/null +++ b/tools/metric_providers/cpu/utilization/procfs/system/source.c @@ -0,0 +1,128 @@ +#include +#include +#include +#include +#include +#include + +typedef struct procfs_time_t { // struct is a specification and this static makes no sense here + unsigned long user_time; + unsigned long nice_time; + unsigned long system_time; + unsigned long idle_time; + unsigned long iowait_time; + unsigned long irq_time; + unsigned long softirq_time; + unsigned long steal_time; + unsigned long guest_time; + unsigned long compute_time; // custom attr by us not in standard /proc/stat format +} procfs_time_t; + + +// All variables are made static, because we believe that this will +// keep them local in scope to the file and not make them persist in state +// between Threads. +// TODO: If this code ever gets multi-threaded please review this assumption to +// not pollute another threads state +static long int user_hz; +static unsigned int msleep_time=1000; + +static void read_cpu_proc(procfs_time_t* procfs_time_struct) { + + FILE* fd = NULL; + + fd = fopen("/proc/stat", "r"); + if ( fd == NULL) { + fprintf(stderr, "Error - file %s failed to open: errno: %d\n", "/proc/stat/", errno); + exit(1); + } + + fscanf(fd, "cpu %ld %ld %ld %ld %ld %ld %ld %ld %ld", &procfs_time_struct->user_time, &procfs_time_struct->nice_time, &procfs_time_struct->system_time, &procfs_time_struct->idle_time, &procfs_time_struct->iowait_time, &procfs_time_struct->irq_time, &procfs_time_struct->softirq_time, &procfs_time_struct->steal_time, &procfs_time_struct->guest_time); + + // debug + // printf("Read: cpu %ld %ld %ld %ld %ld %ld %ld %ld %ld\n", procfs_time_struct->user_time, procfs_time_struct->nice_time, procfs_time_struct->system_time, procfs_time_struct->idle_time, procfs_time_struct->iowait_time, procfs_time_struct->irq_time, procfs_time_struct->softirq_time, procfs_time_struct->steal_time, procfs_time_struct->guest_time); + + fclose(fd); + + // after this multiplication we are on microseconds + // integer division is deliberately, cause we don't loose precision as *1000000 is done before + + procfs_time_struct->user_time = (procfs_time_struct->user_time*1000000)/user_hz; + procfs_time_struct->nice_time = (procfs_time_struct->nice_time*1000000)/user_hz; + procfs_time_struct->system_time = (procfs_time_struct->system_time*1000000)/user_hz; + procfs_time_struct->idle_time = (procfs_time_struct->idle_time*1000000)/user_hz; + procfs_time_struct->iowait_time = (procfs_time_struct->iowait_time*1000000)/user_hz; + procfs_time_struct->irq_time = (procfs_time_struct->irq_time*1000000)/user_hz; + procfs_time_struct->softirq_time = (procfs_time_struct->softirq_time*1000000)/user_hz; + procfs_time_struct->steal_time = (procfs_time_struct->steal_time*1000000)/user_hz; + procfs_time_struct->guest_time = (procfs_time_struct->guest_time*1000000)/user_hz; + procfs_time_struct->compute_time = procfs_time_struct->user_time + procfs_time_struct->nice_time + procfs_time_struct->system_time + procfs_time_struct->iowait_time + procfs_time_struct->irq_time + procfs_time_struct->softirq_time + procfs_time_struct->steal_time + procfs_time_struct->guest_time; +} + + +static int output_stats() { + + long int idle_reading, compute_time_reading; + procfs_time_t main_cpu_reading_before; + procfs_time_t main_cpu_reading_after; + struct timeval now; + + gettimeofday(&now, NULL); // will set now + read_cpu_proc(&main_cpu_reading_before); // will set main_cpu_reading_before + + usleep(msleep_time*1000); + + read_cpu_proc(&main_cpu_reading_after); // will set main_cpu_reading_before + + idle_reading = main_cpu_reading_after.idle_time - main_cpu_reading_before.idle_time; + compute_time_reading = main_cpu_reading_after.compute_time - main_cpu_reading_before.compute_time; + + // debug + // printf("Main CPU Idle Reading: %ld\nMain CPU Compute Time Reading: %ld\n", idle_reading, compute_time_reading); + // printf("%ld%06ld %f\n", now.tv_sec, now.tv_usec, (double)compute_time_reading / (double)(compute_time_reading+idle_reading)); + + // main output to Stdout + printf("%ld%06ld %ld\n", now.tv_sec, now.tv_usec, (compute_time_reading*10000) / (compute_time_reading+idle_reading) ); // Deliberate integer conversion. Precision with 0.01% is good enough + + return 1; +} + +int main(int argc, char **argv) { + + int c; + + setvbuf(stdout, NULL, _IONBF, 0); + user_hz = sysconf(_SC_CLK_TCK); + + while ((c = getopt (argc, argv, "i:h")) != -1) { + switch (c) { + case 'h': + printf("Usage: %s [-i msleep_time] [-h]\n\n",argv[0]); + printf("\t-h : displays this help\n"); + printf("\t-i : specifies the milliseconds sleep time that will be slept between measurements\n\n"); + + struct timespec res; + double resolution; + + printf("\tEnvironment variables:\n"); + printf("\tUserHZ\t\t%ld\n", user_hz); + clock_getres(CLOCK_REALTIME, &res); + resolution = res.tv_sec + (((double)res.tv_nsec)/1.0e9); + printf("\tSystemHZ\t%ld\n", (unsigned long)(1/resolution + 0.5)); + printf("\tCLOCKS_PER_SEC\t%ld\n", CLOCKS_PER_SEC); + exit(0); + case 'i': + msleep_time = atoi(optarg); + break; + default: + fprintf(stderr,"Unknown option %c\n",c); + exit(-1); + } + } + + while(1) { + output_stats(); + } + + return 0; +} diff --git a/tools/metric_providers/memory/energy/RAPL/MSR/system/Makefile b/tools/metric_providers/memory/energy/RAPL/MSR/system/Makefile index 45c978065..b081b99f0 100644 --- a/tools/metric_providers/memory/energy/RAPL/MSR/system/Makefile +++ b/tools/metric_providers/memory/energy/RAPL/MSR/system/Makefile @@ -1,4 +1,6 @@ CFLAGS = -o3 -Wall -lm -static -static-libgcc static-binary: source.c - gcc $< $(CFLAGS) -o $@ \ No newline at end of file + gcc $< $(CFLAGS) -o $@ + sudo chown root $@ + sudo chmod u+s $@ \ No newline at end of file diff --git a/tools/metric_providers/memory/energy/RAPL/MSR/system/provider.py b/tools/metric_providers/memory/energy/RAPL/MSR/system/provider.py index 755407248..7cc112ca0 100644 --- a/tools/metric_providers/memory/energy/RAPL/MSR/system/provider.py +++ b/tools/metric_providers/memory/energy/RAPL/MSR/system/provider.py @@ -10,7 +10,7 @@ def __init__(self, resolution): self._metrics = {"time":int, "value":int} self._resolution = resolution self._extra_switches = ['-d'] - super().__init__(sudo=True) + super().__init__() if __name__ == "__main__": import time diff --git a/tools/metric_providers/memory/total/cgroup/container/source.c b/tools/metric_providers/memory/total/cgroup/container/source.c index ecf954ac8..85db5219d 100644 --- a/tools/metric_providers/memory/total/cgroup/container/source.c +++ b/tools/metric_providers/memory/total/cgroup/container/source.c @@ -15,7 +15,7 @@ typedef struct container_t { // struct is a specification and this static makes // between Threads. // TODO: If this code ever gets multi-threaded please review this assumption to // not pollute another threads state -static const char *user_id = "1000"; //TODO: Figure out user_id dynamically, or request +static int user_id = 0; static unsigned int msleep_time=1000; static container_t *containers = NULL; @@ -61,6 +61,7 @@ int main(int argc, char **argv) { int length = 0; setvbuf(stdout, NULL, _IONBF, 0); + user_id = getuid(); while ((c = getopt (argc, argv, "i:s:h")) != -1) { switch (c) { @@ -82,7 +83,7 @@ int main(int argc, char **argv) { containers = realloc(containers, length * sizeof(container_t)); containers[length-1].id = id; sprintf(containers[length-1].path, - "/sys/fs/cgroup/user.slice/user-%s.slice/user@%s.service/user.slice/docker-%s.scope/memory.current", + "/sys/fs/cgroup/user.slice/user-%d.slice/user@%d.service/user.slice/docker-%s.scope/memory.current", user_id, user_id, id); FILE* fd = NULL; @@ -102,7 +103,7 @@ int main(int argc, char **argv) { } if(containers == NULL) { - printf("Please supply at least one container id with -s XXXX\n"); + fprintf(stderr, "Please supply at least one container id with -s XXXX\n"); exit(1); } diff --git a/tools/metric_providers/network/io/cgroup/container/Makefile b/tools/metric_providers/network/io/cgroup/container/Makefile index e4cfcdb0d..74fe92acd 100644 --- a/tools/metric_providers/network/io/cgroup/container/Makefile +++ b/tools/metric_providers/network/io/cgroup/container/Makefile @@ -1,4 +1,6 @@ CFLAGS = -o3 -Wall -lc static-binary: source.c - gcc $< $(CFLAGS) -o $@ \ No newline at end of file + gcc $< $(CFLAGS) -o $@ + sudo chown root $@ + sudo chmod u+s $@ \ No newline at end of file diff --git a/tools/metric_providers/network/io/cgroup/container/provider.py b/tools/metric_providers/network/io/cgroup/container/provider.py index 813f974b2..3fe011a30 100644 --- a/tools/metric_providers/network/io/cgroup/container/provider.py +++ b/tools/metric_providers/network/io/cgroup/container/provider.py @@ -9,7 +9,7 @@ def __init__(self, resolution): self._metric_name = "network_io_cgroup_container" self._metrics = {"time":int, "value":int, "container_id":str} self._resolution = resolution - super().__init__(sudo=True) + super().__init__() if __name__ == "__main__": import time diff --git a/tools/metric_providers/network/io/cgroup/container/source.c b/tools/metric_providers/network/io/cgroup/container/source.c index 86eee8801..fe8833717 100644 --- a/tools/metric_providers/network/io/cgroup/container/source.c +++ b/tools/metric_providers/network/io/cgroup/container/source.c @@ -20,7 +20,7 @@ typedef struct container_t { // struct is a specification and this static makes // between Threads. // TODO: If this code ever gets multi-threaded please review this assumption to // not pollute another threads state -static const char *user_id = "1000"; //TODO: Figure out user_id dynamically, or request +static int user_id = 0; static unsigned int msleep_time=1000; static container_t *containers = NULL; @@ -53,14 +53,14 @@ static unsigned long int get_network_cgroup(unsigned int pid) { int fd_ns = open(ns_path, O_RDONLY); /* Get descriptor for namespace */ if (fd_ns == -1) { - printf("open failed"); + fprintf(stderr, "open failed"); exit(1); } // printf("Entering namespace /proc/%u/ns/net \n", pid); if (setns(fd_ns, 0) == -1) { // argument 0 means that any type of NS (IPC, Network, UTS) is allowed - printf("setns failed"); + fprintf(stderr, "setns failed"); exit(1); } @@ -117,6 +117,7 @@ int main(int argc, char **argv) { int length = 0; setvbuf(stdout, NULL, _IONBF, 0); + user_id = getuid(); // because the file is run without sudo but has the suid bit set we only need getuid and not geteuid while ((c = getopt (argc, argv, "i:s:h")) != -1) { switch (c) { @@ -138,7 +139,7 @@ int main(int argc, char **argv) { containers = realloc(containers, length * sizeof(container_t)); containers[length-1].id = id; sprintf(containers[length-1].path, - "/sys/fs/cgroup/user.slice/user-%s.slice/user@%s.service/user.slice/docker-%s.scope/cgroup.procs", + "/sys/fs/cgroup/user.slice/user-%d.slice/user@%d.service/user.slice/docker-%s.scope/cgroup.procs", user_id, user_id, id); FILE* fd = NULL; @@ -159,7 +160,7 @@ int main(int argc, char **argv) { } if(containers == NULL) { - printf("Please supply at least one container id with -s XXXX\n"); + fprintf(stderr, "Please supply at least one container id with -s XXXX\n"); exit(1); }