From 6b85a8df39fca7dabbc786d515e693fd1d0dca95 Mon Sep 17 00:00:00 2001 From: David Poliakoff Date: Tue, 30 Jul 2019 15:47:35 -0600 Subject: [PATCH 1/3] Initial push --- src/tools/sanitizer-utility-library/Makefile | 14 ++ .../kp_kernel_info.h | 196 ++++++++++++++++++ .../kp_kernel_timer.cpp | 184 ++++++++++++++++ .../kp_memory_events.hpp | 54 +++++ 4 files changed, 448 insertions(+) create mode 100644 src/tools/sanitizer-utility-library/Makefile create mode 100644 src/tools/sanitizer-utility-library/kp_kernel_info.h create mode 100644 src/tools/sanitizer-utility-library/kp_kernel_timer.cpp create mode 100644 src/tools/sanitizer-utility-library/kp_memory_events.hpp diff --git a/src/tools/sanitizer-utility-library/Makefile b/src/tools/sanitizer-utility-library/Makefile new file mode 100644 index 000000000..17ab3c3ef --- /dev/null +++ b/src/tools/sanitizer-utility-library/Makefile @@ -0,0 +1,14 @@ +CXX=g++ +CXXFLAGS=-O3 -std=c++11 -g +SHARED_CXXFLAGS=-shared -fPIC + +all: kp_sanitizer_utility.so + +kp_reader: kp_reader.cpp kp_sanitizer_utility.so + $(CXX) $(CXXFLAGS) -o kp_reader kp_reader.cpp + +kp_sanitizer_utility.so: kp_kernel_timer.cpp kp_kernel_info.h + $(CXX) $(SHARED_CXXFLAGS) $(CXXFLAGS) -o $@ kp_kernel_timer.cpp + +clean: + rm *.so kp_reader diff --git a/src/tools/sanitizer-utility-library/kp_kernel_info.h b/src/tools/sanitizer-utility-library/kp_kernel_info.h new file mode 100644 index 000000000..28365711b --- /dev/null +++ b/src/tools/sanitizer-utility-library/kp_kernel_info.h @@ -0,0 +1,196 @@ + +#ifndef _H_KOKKOSP_KERNEL_INFO +#define _H_KOKKOSP_KERNEL_INFO + +#include +#include +#include +#if defined(HAVE_GCC_ABI_DEMANGLE) +#include +#endif +double seconds() { + struct timeval now; + gettimeofday(&now, NULL); + + return (double) (now.tv_sec + (now.tv_usec * 1.0e-6)); +} + +enum KernelExecutionType { + PARALLEL_FOR = 0, + PARALLEL_REDUCE = 1, + PARALLEL_SCAN = 2, + REGION = 3 +}; + +class KernelPerformanceInfo { + public: + KernelPerformanceInfo(std::string kName, KernelExecutionType kernelType) : + kType(kernelType) { + + kernelName = (char*) malloc(sizeof(char) * (kName.size() + 1)); + strcpy(kernelName, kName.c_str()); + + callCount = 0; + time = 0; + } + + ~KernelPerformanceInfo() { + free(kernelName); + } + + KernelExecutionType getKernelType() { + return kType; + } + + void incrementCount() { + callCount++; + } + + void addTime(double t) { + time += t; + timeSq += (t*t); + } + + void addFromTimer() { + addTime(seconds() - startTime); + + incrementCount(); + } + + void startTimer() { + startTime = seconds(); + } + + uint64_t getCallCount() { + return callCount; + } + + double getTime() { + return time; + } + + double getTimeSq() { + return timeSq; + } + + char* getName() { + return kernelName; + } + + void addCallCount(const uint64_t newCalls) { + callCount += newCalls; + } + + bool readFromFile(FILE* input) { + uint32_t recordLen = 0; + uint32_t actual_read = fread(&recordLen, sizeof(recordLen), 1, input); + if(actual_read != 1) return false; + + char* entry = (char*) malloc(recordLen); + fread(entry, recordLen, 1, input); + + uint32_t nextIndex = 0; + uint32_t kernelNameLength; + copy((char*) &kernelNameLength, &entry[nextIndex], sizeof(kernelNameLength)); + nextIndex += sizeof(kernelNameLength); + + if(strlen(kernelName) > 0) { + free(kernelName); + } + + kernelName = (char*) malloc( sizeof(char) * (kernelNameLength + 1)); + copy(kernelName, &entry[nextIndex], kernelNameLength); + kernelName[kernelNameLength] = '\0'; +#if defined(HAVE_GCC_ABI_DEMANGLE) + { + int status = -1; + char* demangledKernelName = abi::__cxa_demangle(kernelName, NULL, NULL, &status); + if (status==0) { + free(kernelName); + kernelName = demangledKernelName; + } + } +#endif // HAVE_GCC_ABI_DEMANGLE + nextIndex += kernelNameLength; + + copy((char*) &callCount, &entry[nextIndex], sizeof(callCount)); + nextIndex += sizeof(callCount); + + copy((char*) &time, &entry[nextIndex], sizeof(time)); + nextIndex += sizeof(time); + + copy((char*) &timeSq, &entry[nextIndex], sizeof(timeSq)); + nextIndex += sizeof(timeSq); + + uint32_t kernelT = 0; + copy((char*) &kernelT, &entry[nextIndex], sizeof(kernelT)); + nextIndex += sizeof(kernelT); + + if(kernelT == 0) { + kType = PARALLEL_FOR; + } else if(kernelT == 1) { + kType = PARALLEL_REDUCE; + } else if(kernelT == 2) { + kType = PARALLEL_SCAN; + } else if(kernelT == 3) { + kType = REGION; + } + + free(entry); + return true; + } + + void writeToFile(FILE* output) { + const uint32_t kernelNameLen = (uint32_t) strlen(kernelName); + + const uint32_t recordLen = + sizeof(uint32_t) + + sizeof(char) * kernelNameLen + + sizeof(uint64_t) + + sizeof(double) + + sizeof(double) + + sizeof(uint32_t); + + uint32_t nextIndex = 0; + char* entry = (char*) malloc(recordLen); + + copy(&entry[nextIndex], (char*) &kernelNameLen, sizeof(kernelNameLen)); + nextIndex += sizeof(kernelNameLen); + + copy(&entry[nextIndex], kernelName, kernelNameLen); + nextIndex += kernelNameLen; + + copy(&entry[nextIndex], (char*) &callCount, sizeof(callCount)); + nextIndex += sizeof(callCount); + + copy(&entry[nextIndex], (char*) &time, sizeof(time)); + nextIndex += sizeof(time); + + copy(&entry[nextIndex], (char*) &timeSq, sizeof(timeSq)); + nextIndex += sizeof(timeSq); + + uint32_t kernelTypeOutput = (uint32_t) kType; + copy(&entry[nextIndex], (char*) &kernelTypeOutput, sizeof(kernelTypeOutput)); + nextIndex += sizeof(kernelTypeOutput); + + fwrite(&recordLen, sizeof(uint32_t), 1, output); + fwrite(entry, recordLen, 1, output); + free(entry); + } + + private: + void copy(char* dest, const char* src, uint32_t len) { + for(uint32_t i = 0; i < len; i++) { + dest[i] = src[i]; + } + } + + char* kernelName; + uint64_t callCount; + double time; + double timeSq; + double startTime; + KernelExecutionType kType; +}; + +#endif diff --git a/src/tools/sanitizer-utility-library/kp_kernel_timer.cpp b/src/tools/sanitizer-utility-library/kp_kernel_timer.cpp new file mode 100644 index 000000000..5e3b04385 --- /dev/null +++ b/src/tools/sanitizer-utility-library/kp_kernel_timer.cpp @@ -0,0 +1,184 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "kp_kernel_info.h" +#include "kp_memory_events.hpp" +#include + +#include + +struct NamedPointer { + std::uintptr_t ptr; + const char* name; + std::uint64_t size; + const SpaceHandle space; +}; + +namespace std { + template<> + struct less { + static std::less comp; + bool operator()(const NamedPointer& n1, const NamedPointer& n2) const{ + return !comp(n1.ptr, n2.ptr); + } + }; +}; + +std::vector kokkos_stack; +std::set tracked_pointers; +std::map> space_map; + +using poisonFunctionType = void(*)(void*, std::size_t); + +poisonFunctionType poisonFunction; +poisonFunctionType unpoisonFunction; + +void poisonSpace(std::string spaceName){ + std::cout << "Poisoning "<(); + space_map["HBW"] = std::set(); + poisonSpace("Host"); + poisonSpace("HBW"); + +} + +extern "C" void kokkosp_finalize_library() { +} + +extern "C" void kokkosp_begin_parallel_for(const char* name, const uint32_t devID, uint64_t* kID) { + kokkos_stack.push_back(name); + const char* iter = nullptr; + iter = strstr(name,"Host"); + if(iter != nullptr) { + poisonSpace("HBW"); + } + else{ + iter = strstr(name,"Device"); + if(iter!=nullptr){ + poisonSpace("Host"); + } + } +} + +extern "C" void kokkosp_end_parallel_for(const uint64_t kID) { + std::string name = kokkos_stack.back(); + auto iter = name.find("Host"); + if(iter != std::string::npos) { + unpoisonSpace("HBW"); + } + else{ + iter = name.find("Device"); + if(iter!=std::string::npos){ + unpoisonSpace("Host"); + } + } + kokkos_stack.pop_back(); +} + +extern "C" void kokkosp_begin_parallel_scan(const char* name, const uint32_t devID, uint64_t* kID) { + kokkos_stack.push_back(name); +} + +extern "C" void kokkosp_end_parallel_scan(const uint64_t kID) { + kokkos_stack.pop_back(); +} + +extern "C" void kokkosp_begin_parallel_reduce(const char* name, const uint32_t devID, uint64_t* kID) { + kokkos_stack.push_back(name); +} + +extern "C" void kokkosp_end_parallel_reduce(const uint64_t kID) { + kokkos_stack.pop_back(); +} + +extern "C" void kokkosp_push_profile_region(char* regionName) { + kokkos_stack.push_back(regionName); +} + +extern "C" void kokkosp_pop_profile_region() { + kokkos_stack.pop_back(); +} + +extern "C" void kokkosp_deallocate_data(const SpaceHandle space, const char* label, const void* const ptr_raw, const uint64_t size) { + auto ptr = reinterpret_cast(ptr_raw); + //unpoisonFunction((void*)ptr, size); + auto key = NamedPointer{ptr}; + tracked_pointers.erase(key); + std::string lazy_hack = space.name; + auto iter = space_map.find(lazy_hack); + if(iter != space_map.end()){ + space_map[lazy_hack] = std::set(); + } + space_map[lazy_hack].erase(key); +} + +extern "C" void kokkosp_allocate_data(const SpaceHandle space, const char* label, const void* const ptr_raw, const uint64_t size) { + auto ptr = reinterpret_cast(ptr_raw); + std::size_t length = strlen(label); + char* name = (char*)malloc(sizeof(char) * (length+1)); + strncpy(name, label, length+1); + auto key = NamedPointer { ptr, name, size, space }; + tracked_pointers.insert(key); + std::string lazy_hack = space.name; + auto iter = space_map.find(lazy_hack); + if(iter != space_map.end()){ + space_map[lazy_hack] = std::set(); + } + space_map[lazy_hack].insert(key); + // poisonFunction((void*)ptr, size); +} + +extern "C" const char* parallel_runtime_get_pointer_name(const void* ptr_raw){ + auto ptr = reinterpret_cast(ptr_raw); + auto nearest = tracked_pointers.upper_bound(NamedPointer{ptr, ""}); + return nearest->name; +} + +extern "C" const char** parallel_runtime_get_callstack() { + + const char** stack = (const char**)malloc(sizeof(const char*) * (kokkos_stack.size() + 1)); + for(int i=0;i +#include + +struct SpaceHandle { + char name[64]; +}; From 90173da1763e9084b58730308c946d2798bc7b60 Mon Sep 17 00:00:00 2001 From: David Poliakoff Date: Tue, 6 Aug 2019 07:29:29 -0600 Subject: [PATCH 2/3] Commits before putting this up as a draft --- src/tools/sanitizer-utility-library/kp_kernel_timer.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tools/sanitizer-utility-library/kp_kernel_timer.cpp b/src/tools/sanitizer-utility-library/kp_kernel_timer.cpp index 5e3b04385..631d2bc16 100644 --- a/src/tools/sanitizer-utility-library/kp_kernel_timer.cpp +++ b/src/tools/sanitizer-utility-library/kp_kernel_timer.cpp @@ -176,7 +176,6 @@ extern "C" const char** parallel_runtime_get_callstack() { const char** stack = (const char**)malloc(sizeof(const char*) * (kokkos_stack.size() + 1)); for(int i=0;i Date: Tue, 10 Sep 2019 13:06:04 -0600 Subject: [PATCH 3/3] Now keying off of deviceId parameter --- .../kp_kernel_timer.cpp | 46 ++++++++----------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/src/tools/sanitizer-utility-library/kp_kernel_timer.cpp b/src/tools/sanitizer-utility-library/kp_kernel_timer.cpp index 5e3b04385..1583611ca 100644 --- a/src/tools/sanitizer-utility-library/kp_kernel_timer.cpp +++ b/src/tools/sanitizer-utility-library/kp_kernel_timer.cpp @@ -83,32 +83,28 @@ extern "C" void kokkosp_init_library(const int loadSeq, extern "C" void kokkosp_finalize_library() { } + + extern "C" void kokkosp_begin_parallel_for(const char* name, const uint32_t devID, uint64_t* kID) { + std::cout << "Running kernel on device "<(ptr_raw); - //unpoisonFunction((void*)ptr, size); auto key = NamedPointer{ptr}; tracked_pointers.erase(key); - std::string lazy_hack = space.name; - auto iter = space_map.find(lazy_hack); + std::string space_name_as_string = space.name; + auto iter = space_map.find(space_name_as_string); if(iter != space_map.end()){ - space_map[lazy_hack] = std::set(); + space_map[space_name_as_string] = std::set(); } - space_map[lazy_hack].erase(key); + space_map[space_name_as_string].erase(key); } extern "C" void kokkosp_allocate_data(const SpaceHandle space, const char* label, const void* const ptr_raw, const uint64_t size) { auto ptr = reinterpret_cast(ptr_raw); std::size_t length = strlen(label); - char* name = (char*)malloc(sizeof(char) * (length+1)); - strncpy(name, label, length+1); - auto key = NamedPointer { ptr, name, size, space }; + auto key = NamedPointer { ptr, label, size, space }; tracked_pointers.insert(key); - std::string lazy_hack = space.name; - auto iter = space_map.find(lazy_hack); + std::string space_name_as_string = space.name; + auto iter = space_map.find(space_name_as_string); if(iter != space_map.end()){ - space_map[lazy_hack] = std::set(); + space_map[space_name_as_string] = std::set(); } - space_map[lazy_hack].insert(key); - // poisonFunction((void*)ptr, size); + space_map[space_name_as_string].insert(key); } extern "C" const char* parallel_runtime_get_pointer_name(const void* ptr_raw){