Merge pull request #13018 from cwpearson/feature/tpetra-test-apply-us…

…es-tpls

Tpetra: check that CrsMatrix::apply uses TPLs when expected

packages/tpetra/core/src/Tpetra_Details_KokkosCounter.cpp

@@ -38,11 +38,13 @@
 // ************************************************************************
 // @HEADER
 */
+// clang-format off
 #include "Tpetra_Details_KokkosCounter.hpp"
 #include "TpetraCore_config.h"
 #include "Kokkos_Core.hpp"
 #include "Teuchos_TestForException.hpp"
 #include <cstring>
+#include <string>
 
 namespace Tpetra {
 namespace Details {
@@ -201,6 +203,54 @@ namespace Details {
     TEUCHOS_TEST_FOR_EXCEPTION(1,std::runtime_error,std::string("Error: ") + device + std::string(" is not a device known to Tpetra"));
   }
 
+// clang-format on
+namespace KokkosRegionCounterDetails {
+std::vector<std::string> regions;
+
+void push_region_callback(const char *label) { regions.push_back(label); }
+static_assert(std::is_same_v<decltype(&push_region_callback),
+                             Kokkos_Profiling_pushFunction>,
+              "Unexpected Kokkos profiling interface API. This is an internal "
+              "Tpetra developer error, please report this.");
+
+} // namespace KokkosRegionCounterDetails
+
+void KokkosRegionCounter::start() {
+  Kokkos::Tools::Experimental::set_push_region_callback(
+      KokkosRegionCounterDetails::push_region_callback);
+}
+
+void KokkosRegionCounter::reset() {
+  KokkosRegionCounterDetails::regions.clear();
+}
+
+void KokkosRegionCounter::stop() {
+  Kokkos::Tools::Experimental::set_push_region_callback(nullptr);
+}
+
+size_t
+KokkosRegionCounter::get_count_region_contains(const std::string &needle) {
+  size_t count = 0;
+  for (const auto &region : KokkosRegionCounterDetails::regions) {
+    count += (region.find(needle) != std::string::npos);
+  }
+  return count;
+}
+
+void KokkosRegionCounter::dump_regions(Teuchos::FancyOStream &os) {
+  for (const auto &region : KokkosRegionCounterDetails::regions) {
+    os << region << "\n";
+  }
+}
+
+void KokkosRegionCounter::dump_regions(std::ostream &os) {
+  for (const auto &region : KokkosRegionCounterDetails::regions) {
+    os << region << "\n";
+  }
+}
+
+
+// clang-format off
 
 
 } // namespace Details

packages/tpetra/core/src/Tpetra_Details_KokkosCounter.hpp

@@ -38,6 +38,7 @@
 // ************************************************************************
 // @HEADER
 */
+// clang-format off
 #ifndef TPETRA_DETAILS_KOKKOS_COUNTER_HPP
 #define TPETRA_DETAILS_KOKKOS_COUNTER_HPP
 
@@ -46,6 +47,7 @@
 /// types using the Kokkos Profiling Library
 
 #include <string>
+#include <Teuchos_FancyOStream.hpp>
 
 namespace Tpetra {
 namespace Details {
@@ -87,6 +89,30 @@ namespace FenceCounter {
   size_t get_count_global(const std::string & device);
 }
 
+// clang-format on
+
+/// \brief Counter for Kokkos regions representing third-party library usage
+namespace KokkosRegionCounter {
+/// \brief Start the counter
+void start();
+
+/// \brief Reset the counter
+void reset();
+
+/// \brief Stop the counter
+void stop();
+
+/// \brief How many regions containing `substr` have been seen
+size_t get_count_region_contains(const std::string &substr);
+
+/// \brief Print all observed region labels, separated by newline
+void dump_regions(std::ostream &os);
+void dump_regions(Teuchos::FancyOStream &os);
+} // namespace KokkosRegionCounter
+
+// clang-format off
+
+
 
 } // namespace Details
 } // namespace Tpetra

packages/tpetra/core/test/CrsMatrix/CMakeLists.txt

@@ -490,8 +490,29 @@ TRIBITS_ADD_EXECUTABLE_AND_TEST(
   STANDARD_PASS_OUTPUT
   )
 
+if (
+  # supported TPLs
+  (
+    (Tpetra_ENABLE_CUDA AND TPL_ENABLE_CUSPARSE ) OR
+    (Tpetra_ENABLE_HIP  AND TPL_ENABLE_ROCSPARSE)
+  )
 
+  AND
 
+  # supported type combos
+  (
+    (Tpetra_INST_DOUBLE OR Tpetra_INST_FLOAT)
+  )
+)
+  TRIBITS_ADD_EXECUTABLE_AND_TEST(
+    CrsMatrix_ApplyUsesTPLs
+    SOURCES
+      CrsMatrix_ApplyUsesTPLs.cpp
+      ${TEUCHOS_STD_UNIT_TEST_MAIN}
+    COMM serial mpi
+    STANDARD_PASS_OUTPUT
+    )
+endif()
 
 SET(TIMING_INSTALLS "")