From bd3ac0453d42c79132b33cf3dff2fac86e323fd6 Mon Sep 17 00:00:00 2001
From: Nicolas 'Pixel' Noble <pixel@nobis-crew.org>
Date: Wed, 15 Jan 2025 16:27:02 -0800
Subject: [PATCH 01/12] Starting to work on coroutines.

---
 src/CMakeLists.txt     |   3 +
 src/coroutines.h       | 173 +++++++++++++++++++++++++++++++++++++++++
 src/test_coroutines.cc |  56 +++++++++++++
 3 files changed, 232 insertions(+)
 create mode 100644 src/coroutines.h
 create mode 100644 src/test_coroutines.cc

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index f835f7bb..4fa76951 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -194,6 +194,7 @@ set(
   periodic_concurrency_worker.h
   thread_config.h
   cuda_runtime_library_manager.h
+  coroutines.h
 )
 
 add_executable(
@@ -338,6 +339,7 @@ add_executable(
   test_ctx_id_tracker.cc
   test_profile_data_collector.cc
   test_profile_data_exporter.cc
+  test_coroutines.cc
   ${TEST_HTTP_CLIENT}
 )
 
@@ -361,6 +363,7 @@ target_compile_options(
   perf_analyzer_unit_tests
   PRIVATE
     -Wno-write-strings
+    -fcoroutines
 )
 
 target_link_libraries(
diff --git a/src/coroutines.h b/src/coroutines.h
new file mode 100644
index 00000000..5bf2782f
--- /dev/null
+++ b/src/coroutines.h
@@ -0,0 +1,173 @@
+// Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//  * Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//  * Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//  * Neither the name of NVIDIA CORPORATION nor the names of its
+//    contributors may be used to endorse or promote products derived
+//    from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#pragma once
+
+#include <coroutine>
+#include <type_traits>
+#include <utility>
+
+namespace triton::perfanalyzer {
+
+template <typename T = void>
+class Coroutine {
+ public:
+  struct Empty {};
+  typedef
+      typename std::conditional<std::is_void<T>::value, Empty, T>::type SafeT;
+
+  Coroutine() = default;
+  Coroutine(Coroutine&& other) = default;
+  Coroutine& operator=(Coroutine&& other) = default;
+  Coroutine(Coroutine const&) = delete;
+  Coroutine& operator=(Coroutine const&) = delete;
+
+  class Awaiter {
+   public:
+    Awaiter(Awaiter&& other) = default;
+    Awaiter& operator=(Awaiter&& other) = default;
+    Awaiter(Awaiter const&) = default;
+    Awaiter& operator=(Awaiter const&) = default;
+    constexpr bool await_ready() const noexcept
+    {
+      bool ret = coroutine_->earlyResume_;
+      coroutine_->earlyResume_ = false;
+      return ret;
+    }
+    constexpr void await_suspend(std::coroutine_handle<> h)
+    {
+      coroutine_->suspended_ = true;
+    }
+    constexpr void await_resume() const noexcept {}
+
+   private:
+    Awaiter(Coroutine* coroutine) : coroutine_(coroutine) {}
+    Coroutine* coroutine_;
+    friend struct Coroutine;
+  };
+
+  Awaiter awaiter() { return Awaiter(this); }
+
+  void resume()
+  {
+    if (!handle_)
+      return;
+    if (!suspended_) {
+      earlyResume_ = true;
+      return;
+    }
+    suspended_ = false;
+    handle_.resume();
+  }
+
+  bool done()
+  {
+    if (!handle_)
+      return true;
+    bool isDone = handle_.done();
+    if (isDone) {
+      if constexpr (!std::is_void<T>::value) {
+        value_ = std::move(handle_.promise().value_);
+      }
+      handle_.destroy();
+      handle_ = nullptr;
+    }
+    return isDone;
+  }
+
+  const SafeT& value() const { return value_; }
+
+ private:
+  struct PromiseVoid {
+    Coroutine<> get_return_object()
+    {
+      return Coroutine<>{
+          std::move(std::coroutine_handle<Promise>::from_promise(*this))};
+    }
+    std::suspend_always initial_suspend() { return {}; }
+    std::suspend_always final_suspend() noexcept { return {}; }
+    void unhandled_exception() {}
+    void return_void()
+    {
+      auto awaitingCoroutine = awaitingCoroutine_;
+      if (awaitingCoroutine) {
+        __builtin_coro_resume(awaitingCoroutine);
+      }
+    }
+    [[no_unique_address]] Empty value_;
+    void* awaitingCoroutine_ = nullptr;
+  };
+  struct PromiseValue {
+    Coroutine<T> get_return_object()
+    {
+      return Coroutine{
+          std::move(std::coroutine_handle<Promise>::from_promise(*this))};
+    }
+    std::suspend_always initial_suspend() { return {}; }
+    std::suspend_always final_suspend() noexcept { return {}; }
+    void unhandled_exception() {}
+    void return_value(T&& value)
+    {
+      value_ = std::move(value);
+      auto awaitingCoroutine = awaitingCoroutine_;
+      if (awaitingCoroutine) {
+        __builtin_coro_resume(awaitingCoroutine);
+      }
+    }
+    T value_;
+    void* awaitingCoroutine_ = nullptr;
+  };
+  typedef typename std::conditional<
+      std::is_void<T>::value, PromiseVoid, PromiseValue>::type Promise;
+  Coroutine(std::coroutine_handle<Promise>&& handle)
+      : handle_(std::move(handle))
+  {
+  }
+  std::coroutine_handle<Promise> handle_;
+  [[no_unique_address]] SafeT value_;
+  void* awaitingCoroutine_ = nullptr;
+  bool suspended_ = true;
+  bool earlyResume_ = false;
+
+ public:
+  using promise_type = Promise;
+
+  constexpr bool await_ready() { return handle_.done(); }
+  template <typename U>
+  constexpr void await_suspend(std::coroutine_handle<U> h)
+  {
+    auto& promise = handle_.promise();
+    promise.awaitingCoroutine_ = h.address();
+    resume();
+  }
+  constexpr SafeT await_resume()
+  {
+    SafeT value = std::move(handle_.promise().value_);
+    handle_.destroy();
+    return value;
+  }
+};
+
+}  // namespace triton::perfanalyzer
diff --git a/src/test_coroutines.cc b/src/test_coroutines.cc
new file mode 100644
index 00000000..8740d7c6
--- /dev/null
+++ b/src/test_coroutines.cc
@@ -0,0 +1,56 @@
+// Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//  * Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//  * Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//  * Neither the name of NVIDIA CORPORATION nor the names of its
+//    contributors may be used to endorse or promote products derived
+//    from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "coroutines.h"
+#include "doctest.h"
+
+namespace triton::perfanalyzer {
+
+Coroutine<int>
+CoroutineTest()
+{
+  co_await std::suspend_always{};
+  co_return 42;
+}
+
+TEST_CASE("testing the Coroutine class")
+{
+  auto coroutine = CoroutineTest();
+
+  unsigned rounds = 0;
+  while (!coroutine.done()) {
+    coroutine.resume();
+    rounds++;
+  }
+
+  auto result = coroutine.value();
+
+  CHECK(rounds == 2);
+  CHECK(result == 42);
+  CHECK(coroutine.done());
+}
+
+}  // namespace triton::perfanalyzer

From 4fc5b5aa92f859bd2b9b7154fb6bdc90b8887e99 Mon Sep 17 00:00:00 2001
From: "Nicolas \"Pixel\" Noble" <pixel@nobis-crew.org>
Date: Wed, 15 Jan 2025 19:36:00 -0800
Subject: [PATCH 02/12] Adding some more tests.

---
 src/test_coroutines.cc | 45 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/src/test_coroutines.cc b/src/test_coroutines.cc
index 8740d7c6..3794121f 100644
--- a/src/test_coroutines.cc
+++ b/src/test_coroutines.cc
@@ -53,4 +53,49 @@ TEST_CASE("testing the Coroutine class")
   CHECK(coroutine.done());
 }
 
+Coroutine<>
+CoroutineVoidTest()
+{
+  co_await std::suspend_always{};
+}
+
+TEST_CASE("testing the Coroutine class with void")
+{
+  auto coroutine = CoroutineVoidTest();
+
+  unsigned rounds = 0;
+  while (!coroutine.done()) {
+    coroutine.resume();
+    rounds++;
+  }
+
+  CHECK(rounds == 2);
+  CHECK(coroutine.done());
+}
+
+Coroutine<int>
+CascadeCoroutines()
+{
+  co_await CoroutineVoidTest();
+  auto result = co_await CoroutineTest();
+  co_return result;
+}
+
+TEST_CASE("testing the Coroutine class with cascading coroutines")
+{
+  auto coroutine = CascadeCoroutines();
+
+  unsigned rounds = 0;
+  while (!coroutine.done()) {
+    coroutine.resume();
+    rounds++;
+  }
+
+  auto result = coroutine.value();
+
+  CHECK(rounds == 4);
+  CHECK(result == 42);
+  CHECK(coroutine.done());
+}
+
 }  // namespace triton::perfanalyzer

From b59752b4405f386b20ab51268910dd02cd1d089f Mon Sep 17 00:00:00 2001
From: Nicolas 'Pixel' Noble <pixel@nobis-crew.org>
Date: Thu, 16 Jan 2025 09:25:20 -0800
Subject: [PATCH 03/12] Code style + file renaming.

---
 src/CMakeLists.txt                |  2 +-
 src/{coroutines.h => coroutine.h} | 16 +++++++++-------
 src/test_coroutines.cc            | 26 ++++++++++++++------------
 3 files changed, 24 insertions(+), 20 deletions(-)
 rename src/{coroutines.h => coroutine.h} (96%)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 4fa76951..11324f36 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -194,7 +194,7 @@ set(
   periodic_concurrency_worker.h
   thread_config.h
   cuda_runtime_library_manager.h
-  coroutines.h
+  coroutine.h
 )
 
 add_executable(
diff --git a/src/coroutines.h b/src/coroutine.h
similarity index 96%
rename from src/coroutines.h
rename to src/coroutine.h
index 5bf2782f..ed41437f 100644
--- a/src/coroutines.h
+++ b/src/coroutine.h
@@ -68,12 +68,13 @@ class Coroutine {
     friend struct Coroutine;
   };
 
-  Awaiter awaiter() { return Awaiter(this); }
+  Awaiter CreateAwaiter() { return Awaiter(this); }
 
-  void resume()
+  void Resume()
   {
-    if (!handle_)
+    if (!handle_) {
       return;
+    }
     if (!suspended_) {
       earlyResume_ = true;
       return;
@@ -82,10 +83,11 @@ class Coroutine {
     handle_.resume();
   }
 
-  bool done()
+  bool Done()
   {
-    if (!handle_)
+    if (!handle_) {
       return true;
+    }
     bool isDone = handle_.done();
     if (isDone) {
       if constexpr (!std::is_void<T>::value) {
@@ -97,7 +99,7 @@ class Coroutine {
     return isDone;
   }
 
-  const SafeT& value() const { return value_; }
+  const SafeT& Value() const { return value_; }
 
  private:
   struct PromiseVoid {
@@ -160,7 +162,7 @@ class Coroutine {
   {
     auto& promise = handle_.promise();
     promise.awaitingCoroutine_ = h.address();
-    resume();
+    Resume();
   }
   constexpr SafeT await_resume()
   {
diff --git a/src/test_coroutines.cc b/src/test_coroutines.cc
index 3794121f..db87f7b5 100644
--- a/src/test_coroutines.cc
+++ b/src/test_coroutines.cc
@@ -24,7 +24,9 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-#include "coroutines.h"
+#include <coroutine>
+
+#include "coroutine.h"
 #include "doctest.h"
 
 namespace triton::perfanalyzer {
@@ -41,16 +43,16 @@ TEST_CASE("testing the Coroutine class")
   auto coroutine = CoroutineTest();
 
   unsigned rounds = 0;
-  while (!coroutine.done()) {
-    coroutine.resume();
+  while (!coroutine.Done()) {
+    coroutine.Resume();
     rounds++;
   }
 
-  auto result = coroutine.value();
+  auto result = coroutine.Value();
 
   CHECK(rounds == 2);
   CHECK(result == 42);
-  CHECK(coroutine.done());
+  CHECK(coroutine.Done());
 }
 
 Coroutine<>
@@ -64,13 +66,13 @@ TEST_CASE("testing the Coroutine class with void")
   auto coroutine = CoroutineVoidTest();
 
   unsigned rounds = 0;
-  while (!coroutine.done()) {
-    coroutine.resume();
+  while (!coroutine.Done()) {
+    coroutine.Resume();
     rounds++;
   }
 
   CHECK(rounds == 2);
-  CHECK(coroutine.done());
+  CHECK(coroutine.Done());
 }
 
 Coroutine<int>
@@ -86,16 +88,16 @@ TEST_CASE("testing the Coroutine class with cascading coroutines")
   auto coroutine = CascadeCoroutines();
 
   unsigned rounds = 0;
-  while (!coroutine.done()) {
-    coroutine.resume();
+  while (!coroutine.Done()) {
+    coroutine.Resume();
     rounds++;
   }
 
-  auto result = coroutine.value();
+  auto result = coroutine.Value();
 
   CHECK(rounds == 4);
   CHECK(result == 42);
-  CHECK(coroutine.done());
+  CHECK(coroutine.Done());
 }
 
 }  // namespace triton::perfanalyzer

From 11acea92e1602cc386a3ee5e22463def519c6f2b Mon Sep 17 00:00:00 2001
From: "Nicolas \"Pixel\" Noble" <pixel@nobis-crew.org>
Date: Thu, 16 Jan 2025 15:53:34 -0800
Subject: [PATCH 04/12] More shuffling, simplification, and proper tests.

---
 src/coroutine.h        | 135 ++++++++++++++++++++++-------------------
 src/test_coroutines.cc |  50 +++++++++++----
 2 files changed, 111 insertions(+), 74 deletions(-)

diff --git a/src/coroutine.h b/src/coroutine.h
index ed41437f..2917dab6 100644
--- a/src/coroutine.h
+++ b/src/coroutine.h
@@ -35,6 +35,65 @@ template <typename T = void>
 class Coroutine {
  public:
   struct Empty {};
+
+ private:
+  struct PromiseVoid {
+    Coroutine<> get_return_object()
+    {
+      return Coroutine<>{
+          std::move(std::coroutine_handle<Promise>::from_promise(*this))};
+    }
+    std::suspend_always initial_suspend()
+    {
+      suspended_ = true;
+      return {};
+    }
+    std::suspend_always final_suspend() noexcept { return {}; }
+    void unhandled_exception() {}
+    void return_void()
+    {
+      auto awaitingCoroutine = awaitingCoroutine_;
+      if (awaitingCoroutine) {
+        awaitingCoroutine_ = nullptr;
+        std::coroutine_handle<>::from_address(awaitingCoroutine).resume();
+      }
+    }
+    [[no_unique_address]] Empty value_{};
+    void* awaitingCoroutine_ = nullptr;
+    bool earlyResume_ = false;
+    bool suspended_ = false;
+  };
+  struct PromiseValue {
+    Coroutine<T> get_return_object()
+    {
+      return Coroutine{
+          std::move(std::coroutine_handle<Promise>::from_promise(*this))};
+    }
+    std::suspend_always initial_suspend()
+    {
+      suspended_ = true;
+      return {};
+    }
+    std::suspend_always final_suspend() noexcept { return {}; }
+    void unhandled_exception() {}
+    void return_value(T&& value)
+    {
+      value_ = std::move(value);
+      auto awaitingCoroutine = awaitingCoroutine_;
+      if (awaitingCoroutine) {
+        awaitingCoroutine_ = nullptr;
+        std::coroutine_handle<>::from_address(awaitingCoroutine).resume();
+      }
+    }
+    T value_{};
+    void* awaitingCoroutine_ = nullptr;
+    bool earlyResume_ = false;
+    bool suspended_ = false;
+  };
+  typedef typename std::conditional<
+      std::is_void<T>::value, PromiseVoid, PromiseValue>::type Promise;
+
+ public:
   typedef
       typename std::conditional<std::is_void<T>::value, Empty, T>::type SafeT;
 
@@ -50,36 +109,32 @@ class Coroutine {
     Awaiter& operator=(Awaiter&& other) = default;
     Awaiter(Awaiter const&) = default;
     Awaiter& operator=(Awaiter const&) = default;
-    constexpr bool await_ready() const noexcept
-    {
-      bool ret = coroutine_->earlyResume_;
-      coroutine_->earlyResume_ = false;
-      return ret;
-    }
-    constexpr void await_suspend(std::coroutine_handle<> h)
+    constexpr bool await_ready() const noexcept { return false; }
+    template<typename U>
+    constexpr bool await_suspend(std::coroutine_handle<U> h)
     {
-      coroutine_->suspended_ = true;
+      auto& promise = h.promise();
+      bool ret = promise.earlyResume_;
+      promise.earlyResume_ = false;
+      if (!ret) {
+        promise.suspended_ = true;
+      }
+      return !ret;
     }
     constexpr void await_resume() const noexcept {}
-
-   private:
-    Awaiter(Coroutine* coroutine) : coroutine_(coroutine) {}
-    Coroutine* coroutine_;
-    friend struct Coroutine;
   };
 
-  Awaiter CreateAwaiter() { return Awaiter(this); }
-
   void Resume()
   {
     if (!handle_) {
       return;
     }
-    if (!suspended_) {
-      earlyResume_ = true;
+    auto& promise = handle_.promise();
+    if (!promise.suspended_) {
+      promise.earlyResume_ = true;
       return;
     }
-    suspended_ = false;
+    promise.suspended_ = false;
     handle_.resume();
   }
 
@@ -102,47 +157,6 @@ class Coroutine {
   const SafeT& Value() const { return value_; }
 
  private:
-  struct PromiseVoid {
-    Coroutine<> get_return_object()
-    {
-      return Coroutine<>{
-          std::move(std::coroutine_handle<Promise>::from_promise(*this))};
-    }
-    std::suspend_always initial_suspend() { return {}; }
-    std::suspend_always final_suspend() noexcept { return {}; }
-    void unhandled_exception() {}
-    void return_void()
-    {
-      auto awaitingCoroutine = awaitingCoroutine_;
-      if (awaitingCoroutine) {
-        __builtin_coro_resume(awaitingCoroutine);
-      }
-    }
-    [[no_unique_address]] Empty value_;
-    void* awaitingCoroutine_ = nullptr;
-  };
-  struct PromiseValue {
-    Coroutine<T> get_return_object()
-    {
-      return Coroutine{
-          std::move(std::coroutine_handle<Promise>::from_promise(*this))};
-    }
-    std::suspend_always initial_suspend() { return {}; }
-    std::suspend_always final_suspend() noexcept { return {}; }
-    void unhandled_exception() {}
-    void return_value(T&& value)
-    {
-      value_ = std::move(value);
-      auto awaitingCoroutine = awaitingCoroutine_;
-      if (awaitingCoroutine) {
-        __builtin_coro_resume(awaitingCoroutine);
-      }
-    }
-    T value_;
-    void* awaitingCoroutine_ = nullptr;
-  };
-  typedef typename std::conditional<
-      std::is_void<T>::value, PromiseVoid, PromiseValue>::type Promise;
   Coroutine(std::coroutine_handle<Promise>&& handle)
       : handle_(std::move(handle))
   {
@@ -150,8 +164,6 @@ class Coroutine {
   std::coroutine_handle<Promise> handle_;
   [[no_unique_address]] SafeT value_;
   void* awaitingCoroutine_ = nullptr;
-  bool suspended_ = true;
-  bool earlyResume_ = false;
 
  public:
   using promise_type = Promise;
@@ -168,6 +180,7 @@ class Coroutine {
   {
     SafeT value = std::move(handle_.promise().value_);
     handle_.destroy();
+    handle_ = nullptr;
     return value;
   }
 };
diff --git a/src/test_coroutines.cc b/src/test_coroutines.cc
index db87f7b5..a65eb074 100644
--- a/src/test_coroutines.cc
+++ b/src/test_coroutines.cc
@@ -24,7 +24,10 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+#include <stdio.h>
+
 #include <coroutine>
+#include <queue>
 
 #include "coroutine.h"
 #include "doctest.h"
@@ -34,11 +37,11 @@ namespace triton::perfanalyzer {
 Coroutine<int>
 CoroutineTest()
 {
-  co_await std::suspend_always{};
+  co_await Coroutine<int>::Awaiter{};
   co_return 42;
 }
 
-TEST_CASE("testing the Coroutine class")
+TEST_CASE("coroutine:testing the Coroutine class")
 {
   auto coroutine = CoroutineTest();
 
@@ -58,10 +61,10 @@ TEST_CASE("testing the Coroutine class")
 Coroutine<>
 CoroutineVoidTest()
 {
-  co_await std::suspend_always{};
+  co_await Coroutine<>::Awaiter{};
 }
 
-TEST_CASE("testing the Coroutine class with void")
+TEST_CASE("coroutine:testing the Coroutine class with void")
 {
   auto coroutine = CoroutineVoidTest();
 
@@ -75,29 +78,50 @@ TEST_CASE("testing the Coroutine class with void")
   CHECK(coroutine.Done());
 }
 
+std::queue<std::coroutine_handle<>> pendingCoroutines;
+
+struct QueuedAwaiter {
+  bool await_ready() { return false; }
+  void await_suspend(std::coroutine_handle<> h)
+  {
+    pendingCoroutines.push(h);
+  }
+  void await_resume() {}
+};
+
+Coroutine<int>
+CascadeCoroutine() {
+  co_await QueuedAwaiter{};
+  co_return 42;
+}
+
 Coroutine<int>
-CascadeCoroutines()
+CascadeCoroutinesTest()
 {
-  co_await CoroutineVoidTest();
-  auto result = co_await CoroutineTest();
-  co_return result;
+  co_return co_await CascadeCoroutine() * 2;
 }
 
-TEST_CASE("testing the Coroutine class with cascading coroutines")
+TEST_CASE("coroutine:testing the Coroutine class with cascading coroutines")
 {
-  auto coroutine = CascadeCoroutines();
+  auto coroutine = CascadeCoroutinesTest();
+  coroutine.Resume();
 
   unsigned rounds = 0;
   while (!coroutine.Done()) {
-    coroutine.Resume();
+    CHECK(!pendingCoroutines.empty());
+    auto pending = pendingCoroutines.front();
+    pendingCoroutines.pop();
+    pending.resume();
+    printf("%i\n", rounds);
     rounds++;
   }
 
   auto result = coroutine.Value();
 
-  CHECK(rounds == 4);
-  CHECK(result == 42);
+  CHECK(rounds == 1);
+  CHECK(result == 84);
   CHECK(coroutine.Done());
+  CHECK(pendingCoroutines.empty());
 }
 
 }  // namespace triton::perfanalyzer

From 738118ace2e59fee2667f448c7dcb61db56202fe Mon Sep 17 00:00:00 2001
From: "Nicolas \"Pixel\" Noble" <pixel@nobis-crew.org>
Date: Thu, 16 Jan 2025 16:15:07 -0800
Subject: [PATCH 05/12] More cleanup, and documentation.

---
 src/coroutine.h        | 97 +++++++++++++++++++++++++++++++++++++-----
 src/test_coroutines.cc | 17 ++++++--
 2 files changed, 101 insertions(+), 13 deletions(-)

diff --git a/src/coroutine.h b/src/coroutine.h
index 2917dab6..57d3c85f 100644
--- a/src/coroutine.h
+++ b/src/coroutine.h
@@ -28,15 +28,44 @@
 #include <coroutine>
 #include <type_traits>
 #include <utility>
+#include <variant>
 
 namespace triton::perfanalyzer {
 
+/**
+ * @brief A C++20 coroutine implementation that supports both void and
+ * value-returning coroutines
+ *
+ * @details This class implements a coroutine that can be used to create
+ * cooperative multitasking functionality. It supports both void coroutines and
+ * coroutines that return a value of type T.
+ *
+ * The coroutine starts in a suspended state and must be explicitly resumed
+ * using Resume(). The coroutine's completion status can be checked using
+ * Done(), and for value-returning coroutines, the result can be retrieved using
+ * Value().
+ *
+ * Key features:
+ * - Support for void and value-returning coroutines.
+ * - Manual resume control via Resume().
+ * - Status checking via Done().
+ * - Value retrieval via Value() for non-void coroutines.
+ * - Awaitable interface for use in other coroutines, enabling cascading
+ * coroutines.
+ *
+ * @tparam T The type of value returned by the coroutine. Use void for
+ * coroutines that don't return a value.
+ */
 template <typename T = void>
 class Coroutine {
- public:
-  struct Empty {};
-
- private:
+  // The Promise class is used to manage the coroutine's state and control flow.
+  // We need one to handle void coroutines and another for value-returning
+  // coroutines. Their implementations are very similar, but the value-returning
+  // version stores the return value. The names of the methods are based on the
+  // C++20 coroutine specification, and cannot be changed. The two classes
+  // eventually coalesce into the Promise type alias below. The Promise class is
+  // completely internal to the Coroutine class and is not meant to be used
+  // directly.
   struct PromiseVoid {
     Coroutine<> get_return_object()
     {
@@ -58,7 +87,7 @@ class Coroutine {
         std::coroutine_handle<>::from_address(awaitingCoroutine).resume();
       }
     }
-    [[no_unique_address]] Empty value_{};
+    [[no_unique_address]] std::monostate value_{};
     void* awaitingCoroutine_ = nullptr;
     bool earlyResume_ = false;
     bool suspended_ = false;
@@ -90,19 +119,42 @@ class Coroutine {
     bool earlyResume_ = false;
     bool suspended_ = false;
   };
-  typedef typename std::conditional<
-      std::is_void<T>::value, PromiseVoid, PromiseValue>::type Promise;
+  typedef
+      typename std::conditional_t<std::is_void_v<T>, PromiseVoid, PromiseValue>
+          Promise;
 
  public:
-  typedef
-      typename std::conditional<std::is_void<T>::value, Empty, T>::type SafeT;
+  // The SafeT alias is used to handle the case where T is void. In this case,
+  // we use std::monostate as the type of the value_ member of the Promise
+  // class. This allows us to use a single Coroutine class for both void and
+  // value-returning coroutines.
+  typedef typename std::conditional_t<std::is_void_v<T>, std::monostate, T>
+      SafeT;
 
   Coroutine() = default;
   Coroutine(Coroutine&& other) = default;
   Coroutine& operator=(Coroutine&& other) = default;
+  // The copy constructor and copy assignment operator are deleted copying
+  // coroutines doesn't make sense.
   Coroutine(Coroutine const&) = delete;
   Coroutine& operator=(Coroutine const&) = delete;
 
+  /**
+   * @brief A helper class used to implement the coroutine awaitable interface.
+   *
+   * @details The Awaiter class provides the necessary methods for the co_await
+   * operator to work with coroutines. Its interface is based on the C++20
+   * coroutine specification, and the names of the methods cannot be changed.
+   *
+   * This class enables proper synchronization between coroutines and allows for
+   * optimization through early resume functionality to avoid unnecessary
+   * suspensions.
+   *
+   * Its main purpose is to be a generic awaiter mechanism, in case creating a
+   * specific awaiter for an asychronous operation would be too cumbersome, but
+   * in general, it is recommended to create a specific awaiter for each
+   * asynchronous operation instead.
+   */
   class Awaiter {
    public:
     Awaiter(Awaiter&& other) = default;
@@ -110,7 +162,7 @@ class Coroutine {
     Awaiter(Awaiter const&) = default;
     Awaiter& operator=(Awaiter const&) = default;
     constexpr bool await_ready() const noexcept { return false; }
-    template<typename U>
+    template <typename U>
     constexpr bool await_suspend(std::coroutine_handle<U> h)
     {
       auto& promise = h.promise();
@@ -124,6 +176,13 @@ class Coroutine {
     constexpr void await_resume() const noexcept {}
   };
 
+  /**
+   * @brief Resumes the coroutine.
+   *
+   * @details This method resumes the coroutine if it is suspended using the
+   * Awaiter mechanism above, or for its initial execution. If the coroutine is
+   * not suspended, it sets a flag to resume it early when it is next suspended.
+   */
   void Resume()
   {
     if (!handle_) {
@@ -138,6 +197,17 @@ class Coroutine {
     handle_.resume();
   }
 
+  /**
+   * @brief Checks if the coroutine has completed.
+   *
+   * @details This method checks if the coroutine has completed its execution.
+   * If the coroutine has completed, it will free the internal coroutine
+   * resources, and the coroutine object will be in a state where it can be
+   * safely destroyed. The internal Promise will be resolved, and its value will
+   * become available through the Value() method.
+   *
+   * @return true if the coroutine has completed, false otherwise.
+   */
   bool Done()
   {
     if (!handle_) {
@@ -154,6 +224,11 @@ class Coroutine {
     return isDone;
   }
 
+  /**
+   * @brief Retrieves the value returned by the coroutine.
+   *
+   * @return const SafeT& The value returned by the coroutine.
+   */
   const SafeT& Value() const { return value_; }
 
  private:
@@ -166,6 +241,8 @@ class Coroutine {
   void* awaitingCoroutine_ = nullptr;
 
  public:
+  // While the remainder of the class is public, the following methods are
+  // the necessary boilerplate to implement the cascade coroutine mechanism.
   using promise_type = Promise;
 
   constexpr bool await_ready() { return handle_.done(); }
diff --git a/src/test_coroutines.cc b/src/test_coroutines.cc
index a65eb074..5e8bf24d 100644
--- a/src/test_coroutines.cc
+++ b/src/test_coroutines.cc
@@ -24,8 +24,6 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-#include <stdio.h>
-
 #include <coroutine>
 #include <queue>
 
@@ -34,6 +32,10 @@
 
 namespace triton::perfanalyzer {
 
+// A simple coroutine that returns an integer.
+// It awaits for an Awaiter object and then returns 42,
+// meaning that it will resume twice before completing
+// as the coroutine is created in a suspended state.
 Coroutine<int>
 CoroutineTest()
 {
@@ -58,6 +60,9 @@ TEST_CASE("coroutine:testing the Coroutine class")
   CHECK(coroutine.Done());
 }
 
+// A simple coroutine that returns void.
+// This is the same as the previous test, but with a void return type,
+// in order to test the void specialization of the Coroutine class.
 Coroutine<>
 CoroutineVoidTest()
 {
@@ -78,8 +83,13 @@ TEST_CASE("coroutine:testing the Coroutine class with void")
   CHECK(coroutine.Done());
 }
 
+// This tests cascading coroutines, where one coroutine awaits another.
+// To do this without a global scheduler, we use a queue to store the
+// pending coroutines.
 std::queue<std::coroutine_handle<>> pendingCoroutines;
 
+// The specialized awaiter will simply queue the coroutine handle to
+// be resumed immediately in the pseudo-scheduler loop of the test.
 struct QueuedAwaiter {
   bool await_ready() { return false; }
   void await_suspend(std::coroutine_handle<> h)
@@ -89,12 +99,14 @@ struct QueuedAwaiter {
   void await_resume() {}
 };
 
+// A coroutine that schedules itself to be resumed later, and returns 42.
 Coroutine<int>
 CascadeCoroutine() {
   co_await QueuedAwaiter{};
   co_return 42;
 }
 
+// The main coroutine that awaits the previous one and returns its value.
 Coroutine<int>
 CascadeCoroutinesTest()
 {
@@ -112,7 +124,6 @@ TEST_CASE("coroutine:testing the Coroutine class with cascading coroutines")
     auto pending = pendingCoroutines.front();
     pendingCoroutines.pop();
     pending.resume();
-    printf("%i\n", rounds);
     rounds++;
   }
 

From d3eacf45e0d5133d23d463ba9556aa18ecb44b5b Mon Sep 17 00:00:00 2001
From: "Nicolas \"Pixel\" Noble" <pixel@nobis-crew.org>
Date: Thu, 16 Jan 2025 16:23:53 -0800
Subject: [PATCH 06/12] Formatting.

---
 src/test_coroutines.cc | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/test_coroutines.cc b/src/test_coroutines.cc
index 5e8bf24d..1c9aa3cd 100644
--- a/src/test_coroutines.cc
+++ b/src/test_coroutines.cc
@@ -92,16 +92,14 @@ std::queue<std::coroutine_handle<>> pendingCoroutines;
 // be resumed immediately in the pseudo-scheduler loop of the test.
 struct QueuedAwaiter {
   bool await_ready() { return false; }
-  void await_suspend(std::coroutine_handle<> h)
-  {
-    pendingCoroutines.push(h);
-  }
+  void await_suspend(std::coroutine_handle<> h) { pendingCoroutines.push(h); }
   void await_resume() {}
 };
 
 // A coroutine that schedules itself to be resumed later, and returns 42.
 Coroutine<int>
-CascadeCoroutine() {
+CascadeCoroutine()
+{
   co_await QueuedAwaiter{};
   co_return 42;
 }

From 74299ba430ee914e938c8460a056d9c8027cc646 Mon Sep 17 00:00:00 2001
From: "Nicolas \"Pixel\" Noble" <pixel@nobis-crew.org>
Date: Thu, 16 Jan 2025 19:46:27 -0800
Subject: [PATCH 07/12] Typo.

---
 src/coroutine.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/coroutine.h b/src/coroutine.h
index 57d3c85f..194d4d93 100644
--- a/src/coroutine.h
+++ b/src/coroutine.h
@@ -151,7 +151,7 @@ class Coroutine {
    * suspensions.
    *
    * Its main purpose is to be a generic awaiter mechanism, in case creating a
-   * specific awaiter for an asychronous operation would be too cumbersome, but
+   * specific awaiter for an asynchronous operation would be too cumbersome, but
    * in general, it is recommended to create a specific awaiter for each
    * asynchronous operation instead.
    */

From 777ec31f4435214c29a0995c206b978d18b53f9b Mon Sep 17 00:00:00 2001
From: Nicolas Noble <nicolasnoble@users.noreply.github.com>
Date: Wed, 22 Jan 2025 12:09:50 -0800
Subject: [PATCH 08/12] Renaming test_coroutines.cc to test_coroutine.cc

---
 src/CMakeLists.txt                            | 2 +-
 src/{test_coroutines.cc => test_coroutine.cc} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename src/{test_coroutines.cc => test_coroutine.cc} (100%)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 11324f36..0e6fa91c 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -339,7 +339,7 @@ add_executable(
   test_ctx_id_tracker.cc
   test_profile_data_collector.cc
   test_profile_data_exporter.cc
-  test_coroutines.cc
+  test_coroutine.cc
   ${TEST_HTTP_CLIENT}
 )
 
diff --git a/src/test_coroutines.cc b/src/test_coroutine.cc
similarity index 100%
rename from src/test_coroutines.cc
rename to src/test_coroutine.cc

From 557183eb84c54c7f2d8394bdeea0b135ffce8fbb Mon Sep 17 00:00:00 2001
From: "Nicolas \"Pixel\" Noble" <pixel@nobis-crew.org>
Date: Wed, 22 Jan 2025 20:42:39 +0000
Subject: [PATCH 09/12] Adding static assert for the type of the void
 coroutine.

---
 src/test_coroutine.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test_coroutine.cc b/src/test_coroutine.cc
index 1c9aa3cd..38a8efbd 100644
--- a/src/test_coroutine.cc
+++ b/src/test_coroutine.cc
@@ -81,6 +81,7 @@ TEST_CASE("coroutine:testing the Coroutine class with void")
 
   CHECK(rounds == 2);
   CHECK(coroutine.Done());
+  static_assert(std::is_same_v<std::decay_t<decltype(coroutine.Value())>, std::monostate>);
 }
 
 // This tests cascading coroutines, where one coroutine awaits another.

From d8b0b7091a2ac01be55fb2ba5e7769ef7c1b2ff8 Mon Sep 17 00:00:00 2001
From: "Nicolas \"Pixel\" Noble" <pixel@nobis-crew.org>
Date: Wed, 22 Jan 2025 20:44:34 +0000
Subject: [PATCH 10/12] Simplifying the first coroutine test.

---
 src/test_coroutine.cc | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/src/test_coroutine.cc b/src/test_coroutine.cc
index 38a8efbd..63cb9e59 100644
--- a/src/test_coroutine.cc
+++ b/src/test_coroutine.cc
@@ -47,17 +47,13 @@ TEST_CASE("coroutine:testing the Coroutine class")
 {
   auto coroutine = CoroutineTest();
 
-  unsigned rounds = 0;
-  while (!coroutine.Done()) {
-    coroutine.Resume();
-    rounds++;
-  }
+  REQUIRE(!coroutine.Done());  
 
-  auto result = coroutine.Value();
+  coroutine.Resume(); // resume from initial suspension  
+  coroutine.Resume(); // resume from suspension at end, coroutine completes  
 
-  CHECK(rounds == 2);
-  CHECK(result == 42);
-  CHECK(coroutine.Done());
+  CHECK(coroutine.Done());  
+  CHECK(coroutine.Value() == 42);
 }
 
 // A simple coroutine that returns void.

From 40a43015541c95369da35833d2c51bbafcadd82c Mon Sep 17 00:00:00 2001
From: "Nicolas \"Pixel\" Noble" <pixel@nobis-crew.org>
Date: Wed, 22 Jan 2025 20:46:55 +0000
Subject: [PATCH 11/12] Fixing include-what-you-use issue.

---
 src/test_coroutine.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test_coroutine.cc b/src/test_coroutine.cc
index 63cb9e59..5c657606 100644
--- a/src/test_coroutine.cc
+++ b/src/test_coroutine.cc
@@ -26,6 +26,7 @@
 
 #include <coroutine>
 #include <queue>
+#include <type_traits>
 
 #include "coroutine.h"
 #include "doctest.h"

From 5e19e616f3d91fc1b4d9ca1b3adea1bac8b859b2 Mon Sep 17 00:00:00 2001
From: "Nicolas \"Pixel\" Noble" <pixel@nobis-crew.org>
Date: Wed, 22 Jan 2025 14:59:51 -0800
Subject: [PATCH 12/12] Formatting.

---
 src/test_coroutine.cc | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/test_coroutine.cc b/src/test_coroutine.cc
index 5c657606..2f005e01 100644
--- a/src/test_coroutine.cc
+++ b/src/test_coroutine.cc
@@ -48,12 +48,12 @@ TEST_CASE("coroutine:testing the Coroutine class")
 {
   auto coroutine = CoroutineTest();
 
-  REQUIRE(!coroutine.Done());  
+  REQUIRE(!coroutine.Done());
 
-  coroutine.Resume(); // resume from initial suspension  
-  coroutine.Resume(); // resume from suspension at end, coroutine completes  
+  coroutine.Resume();  // resume from initial suspension
+  coroutine.Resume();  // resume from suspension at end, coroutine completes
 
-  CHECK(coroutine.Done());  
+  CHECK(coroutine.Done());
   CHECK(coroutine.Value() == 42);
 }
 
@@ -78,7 +78,8 @@ TEST_CASE("coroutine:testing the Coroutine class with void")
 
   CHECK(rounds == 2);
   CHECK(coroutine.Done());
-  static_assert(std::is_same_v<std::decay_t<decltype(coroutine.Value())>, std::monostate>);
+  static_assert(std::is_same_v<
+                std::decay_t<decltype(coroutine.Value())>, std::monostate>);
 }
 
 // This tests cascading coroutines, where one coroutine awaits another.