Replace loader handles with field at start of handle data

This replaces the handle logic in the loader from wrapped pointers
to a ddi table at the start of the handle struct itself.

scripts/templates/helper.py

@@ -1295,17 +1295,6 @@ def get_initial_null_set(obj):
         return 'if (nullptr != {0}) {{*{0} = nullptr;}}'.format(lvalue)
     return ""
 
-"""
-Public:
-    returns true if the function always wraps output pointers in loader handles
-"""
-def always_wrap_outputs(obj):
-    cname = obj_traits.class_name(obj)
-    return (cname, obj['name']) in [
-        ('$xProgram', 'Link'),
-        ('$xProgram', 'LinkExp'),
-    ]
-
 """
 Private:
     returns the list of parameters, filtering based on desc tags

scripts/templates/ur_interface_loader.cpp.mako

@@ -20,6 +20,7 @@ from templates import helper as th
 //===----------------------------------------------------------------------===//
 #include <${n}_api.h>
 #include <${n}_ddi.h>
+#include <mutex>
 
 #include "ur_interface_loader.hpp"
 
@@ -68,22 +69,45 @@ ${X}_APIEXPORT ${x}_result_t ${X}_APICALL ${tbl['export']['name']}(
 } // extern "C"
 #endif
 
-#ifdef UR_STATIC_ADAPTER_${Adapter}
-namespace ur::${adapter} {
-ur_result_t urAdapterGetDdiTables(ur_dditable_t *ddi) {
-  if (ddi == nullptr) {
+namespace {
+ur_result_t populateDdiTable(ur_dditable_t *ddi) {
+    if (ddi == nullptr) {
     return UR_RESULT_ERROR_INVALID_NULL_POINTER;
   }
 
   ur_result_t result;
 
+#ifdef UR_STATIC_ADAPTER_${Adapter}
+#define NAMESPACE_ ::ur::${adapter}
+#else
+#define NAMESPACE_
+#endif
+
 %for tbl in th.get_pfntables(specs, meta, n, tags):
-  result = ${n}::${adapter}::${tbl['export']['name']}( ${X}_API_VERSION_CURRENT, &ddi->${tbl['name']} );
+  result = NAMESPACE_::${tbl['export']['name']}( ${X}_API_VERSION_CURRENT, &ddi->${tbl['name']} );
   if (result != UR_RESULT_SUCCESS)
     return result;
 %endfor
 
+#undef NAMESPACE_
+
   return result;
 }
 }
+
+
+namespace ur::${adapter} {
+${x}_dditable_t *ddi_getter::value() {
+  static std::once_flag flag;
+  static ${x}_dditable_t table;
+
+  std::call_once(flag, []() { populateDdiTable(&table); });
+  return &table;
+}
+
+#ifdef UR_STATIC_ADAPTER_${Adapter}
+ur_result_t urAdapterGetDdiTables(${x}_dditable_t *ddi) {
+  return populateDdiTable(ddi);
+}
 #endif
+}

scripts/templates/ur_interface_loader.hpp.mako

@@ -18,6 +18,8 @@ from templates import helper as th
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+#pragma once
+
 #include <${n}_api.h>
 #include <${n}_ddi.h>
 
@@ -36,4 +38,8 @@ ${x}_result_t ${th.make_func_name(n, tags, obj)}(
 #ifdef UR_STATIC_ADAPTER_LEVEL_ZERO
 ur_result_t urAdapterGetDdiTables(ur_dditable_t *ddi);
 #endif
+
+struct ddi_getter {
+  static ${x}_dditable_t *value();
+};
 }

source/adapters/hip/adapter.cpp

@@ -15,7 +15,7 @@
 #include <atomic>
 #include <ur_api.h>
 
-struct ur_adapter_handle_t_ {
+struct ur_adapter_handle_t_ : ur_object_t_ {
   std::atomic<uint32_t> RefCount = 0;
   logger::Logger &logger;
   ur_adapter_handle_t_();
@@ -40,8 +40,8 @@ class ur_legacy_sink : public logger::Sink {
 // through UR entry points.
 // https://github.com/oneapi-src/unified-runtime/issues/1330
 ur_adapter_handle_t_::ur_adapter_handle_t_()
-    : logger(
-          logger::get_logger("hip", /*default_log_level*/ logger::Level::ERR)) {
+    : ur_object_t_(), logger(logger::get_logger(
+                          "hip", /*default_log_level*/ logger::Level::ERR)) {
 
   if (std::getenv("UR_LOG_HIP") != nullptr)
     return;

source/adapters/hip/command_buffer.cpp

@@ -48,9 +48,9 @@ commandHandleReleaseInternal(ur_exp_command_buffer_command_handle_t Command) {
 
 ur_exp_command_buffer_handle_t_::ur_exp_command_buffer_handle_t_(
     ur_context_handle_t hContext, ur_device_handle_t hDevice, bool IsUpdatable)
-    : Context(hContext), Device(hDevice), IsUpdatable(IsUpdatable),
-      HIPGraph{nullptr}, HIPGraphExec{nullptr}, RefCountInternal{1},
-      RefCountExternal{1}, NextSyncPoint{0} {
+    : ur_object_t_(), Context(hContext), Device(hDevice),
+      IsUpdatable(IsUpdatable), HIPGraph{nullptr}, HIPGraphExec{nullptr},
+      RefCountInternal{1}, RefCountExternal{1}, NextSyncPoint{0} {
   urContextRetain(hContext);
   urDeviceRetain(hDevice);
 }
@@ -80,8 +80,9 @@ ur_exp_command_buffer_command_handle_t_::
         const size_t *GlobalWorkOffsetPtr, const size_t *GlobalWorkSizePtr,
         const size_t *LocalWorkSizePtr, uint32_t NumKernelAlternatives,
         ur_kernel_handle_t *KernelAlternatives)
-    : CommandBuffer(CommandBuffer), Kernel(Kernel), Node(Node), Params(Params),
-      WorkDim(WorkDim), RefCountInternal(1), RefCountExternal(1) {
+    : ur_object_t_(), CommandBuffer(CommandBuffer), Kernel(Kernel), Node(Node),
+      Params(Params), WorkDim(WorkDim), RefCountInternal(1),
+      RefCountExternal(1) {
   CommandBuffer->incrementInternalReferenceCount();
 
   const size_t CopySize = sizeof(size_t) * WorkDim;

source/adapters/hip/command_buffer.hpp

@@ -39,7 +39,7 @@
 // Struct that stores all the information related to a kernel command in a
 // command-buffer, such that the command can be recreated. When handles can
 // be returned from other command types this struct will need refactored.
-struct ur_exp_command_buffer_command_handle_t_ {
+struct ur_exp_command_buffer_command_handle_t_ : ur_object_t_ {
   ur_exp_command_buffer_command_handle_t_(
       ur_exp_command_buffer_handle_t CommandBuffer, ur_kernel_handle_t Kernel,
       hipGraphNode_t Node, hipKernelNodeParams Params, uint32_t WorkDim,
@@ -121,7 +121,7 @@ struct ur_exp_command_buffer_command_handle_t_ {
   std::atomic_uint32_t RefCountExternal;
 };
 
-struct ur_exp_command_buffer_handle_t_ {
+struct ur_exp_command_buffer_handle_t_ : ur_object_t_ {
 
   ur_exp_command_buffer_handle_t_(ur_context_handle_t hContext,
                                   ur_device_handle_t hDevice, bool IsUpdatable);

source/adapters/hip/common.hpp

@@ -239,3 +239,8 @@ inline static unsigned getMemoryType(hipPointerAttribute_t hipPointerAttrs) {
   return hipPointerAttrs.memoryType;
 #endif
 }
+
+struct hip_ddi_getter {
+  static ur_dditable_t *value();
+};
+using ur_object_t_ = ur_handle_base_t_<hip_ddi_getter>;

source/adapters/hip/context.hpp

@@ -76,7 +76,7 @@ typedef void (*ur_context_extended_deleter_t)(void *UserData);
 ///  between native allocations for devices in the same \c ur_context_handle_t_
 ///  if necessary.
 ///
-struct ur_context_handle_t_ {
+struct ur_context_handle_t_ : ur_object_t_ {
 
   struct deleter_data {
     ur_context_extended_deleter_t Function;
@@ -90,7 +90,7 @@ struct ur_context_handle_t_ {
   std::atomic_uint32_t RefCount;
 
   ur_context_handle_t_(const ur_device_handle_t *Devs, uint32_t NumDevices)
-      : Devices{Devs, Devs + NumDevices}, RefCount{1} {
+      : ur_object_t_(), Devices{Devs, Devs + NumDevices}, RefCount{1} {
     for (auto &Dev : Devices) {
       urDeviceRetain(Dev);
     }

source/adapters/hip/device.hpp

@@ -17,7 +17,7 @@
 /// Includes an observer pointer to the platform,
 /// and implements the reference counting semantics since
 /// HIP objects are not refcounted.
-struct ur_device_handle_t_ {
+struct ur_device_handle_t_ : ur_object_t_ {
 private:
   using native_type = hipDevice_t;
 
@@ -38,8 +38,8 @@ struct ur_device_handle_t_ {
 public:
   ur_device_handle_t_(native_type HipDevice, hipEvent_t EvBase,
                       ur_platform_handle_t Platform, uint32_t DeviceIndex)
-      : HIPDevice(HipDevice), RefCount{1}, Platform(Platform), EvBase(EvBase),
-        DeviceIndex(DeviceIndex) {
+      : ur_object_t_(), HIPDevice(HipDevice), RefCount{1}, Platform(Platform),
+        EvBase(EvBase), DeviceIndex(DeviceIndex) {
 
     UR_CHECK_ERROR(hipDeviceGetAttribute(
         &MaxWorkGroupSize, hipDeviceAttributeMaxThreadsPerBlock, HIPDevice));

source/adapters/hip/event.cpp

@@ -19,7 +19,7 @@ ur_event_handle_t_::ur_event_handle_t_(ur_command_t Type,
                                        hipEvent_t EvEnd, hipEvent_t EvQueued,
                                        hipEvent_t EvStart, hipStream_t Stream,
                                        uint32_t StreamToken)
-    : CommandType{Type}, RefCount{1}, HasOwnership{true},
+    : ur_object_t_(), CommandType{Type}, RefCount{1}, HasOwnership{true},
       HasBeenWaitedOn{false}, IsRecorded{false}, IsStarted{false},
       StreamToken{StreamToken}, EventId{0}, EvEnd{EvEnd}, EvStart{EvStart},
       EvQueued{EvQueued}, Queue{Queue}, Stream{Stream}, Context{Context} {
@@ -29,11 +29,12 @@ ur_event_handle_t_::ur_event_handle_t_(ur_command_t Type,
 
 ur_event_handle_t_::ur_event_handle_t_(ur_context_handle_t Context,
                                        hipEvent_t EventNative)
-    : CommandType{UR_COMMAND_EVENTS_WAIT}, RefCount{1}, HasOwnership{false},
-      HasBeenWaitedOn{false}, IsRecorded{false}, IsStarted{false},
-      IsInterop{true}, StreamToken{std::numeric_limits<uint32_t>::max()},
-      EventId{0}, EvEnd{EventNative}, EvStart{nullptr}, EvQueued{nullptr},
-      Queue{nullptr}, Stream{nullptr}, Context{Context} {
+    : ur_object_t_(), CommandType{UR_COMMAND_EVENTS_WAIT}, RefCount{1},
+      HasOwnership{false}, HasBeenWaitedOn{false}, IsRecorded{false},
+      IsStarted{false}, IsInterop{true},
+      StreamToken{std::numeric_limits<uint32_t>::max()}, EventId{0},
+      EvEnd{EventNative}, EvStart{nullptr}, EvQueued{nullptr}, Queue{nullptr},
+      Stream{nullptr}, Context{Context} {
   urContextRetain(Context);
 }
 

source/adapters/hip/event.hpp

@@ -14,7 +14,7 @@
 
 /// UR Event mapping to hipEvent_t
 ///
-struct ur_event_handle_t_ {
+struct ur_event_handle_t_ : ur_object_t_ {
 public:
   using native_type = hipEvent_t;
 

source/adapters/hip/kernel.hpp

@@ -32,7 +32,7 @@
 /// A compiler pass converts the UR API local memory model into the
 /// HIP shared model. This object simply calculates the total of
 /// shared memory, and the initial offsets of each parameter.
-struct ur_kernel_handle_t_ {
+struct ur_kernel_handle_t_ : ur_object_t_ {
   using native_type = hipFunction_t;
 
   native_type Function;
@@ -232,8 +232,9 @@ struct ur_kernel_handle_t_ {
   ur_kernel_handle_t_(hipFunction_t Func, hipFunction_t FuncWithOffsetParam,
                       const char *Name, ur_program_handle_t Program,
                       ur_context_handle_t Ctxt)
-      : Function{Func}, FunctionWithOffsetParam{FuncWithOffsetParam},
-        Name{Name}, Context{Ctxt}, Program{Program}, RefCount{1} {
+      : ur_object_t_(), Function{Func},
+        FunctionWithOffsetParam{FuncWithOffsetParam}, Name{Name}, Context{Ctxt},
+        Program{Program}, RefCount{1} {
     assert(Program->getDevice());
     UR_CHECK_ERROR(urKernelGetGroupInfo(
         this, Program->getDevice(),

source/adapters/hip/memory.hpp

@@ -307,7 +307,7 @@ struct SurfaceMem {
 /// Migrations will occur in both cases if the most recent version of data
 /// is on a different device, marked by LastQueueWritingToMemObj->getDevice().
 ///
-struct ur_mem_handle_t_ {
+struct ur_mem_handle_t_ : ur_object_t_ {
 
   // TODO: Move as much shared data up as possible
   using ur_context = ur_context_handle_t_ *;
@@ -355,9 +355,9 @@ struct ur_mem_handle_t_ {
 
   // Subbuffer constructor
   ur_mem_handle_t_(ur_mem Parent, size_t SubBufferOffset)
-      : Context{Parent->Context}, RefCount{1}, MemFlags{Parent->MemFlags},
-        HaveMigratedToDeviceSinceLastWrite(Parent->Context->Devices.size(),
-                                           false),
+      : ur_object_t_(), Context{Parent->Context}, RefCount{1},
+        MemFlags{Parent->MemFlags}, HaveMigratedToDeviceSinceLastWrite(
+                                        Parent->Context->Devices.size(), false),
         Mem{BufferMem{std::get<BufferMem>(Parent->Mem)}} {
     auto &SubBuffer = std::get<BufferMem>(Mem);
     SubBuffer.Parent = Parent;

source/adapters/hip/physical_mem.hpp

@@ -17,10 +17,10 @@
 /// management.
 /// TODO: Implement.
 ///
-struct ur_physical_mem_handle_t_ {
+struct ur_physical_mem_handle_t_ : ur_object_t_ {
   std::atomic_uint32_t RefCount;
 
-  ur_physical_mem_handle_t_() : RefCount(1) {}
+  ur_physical_mem_handle_t_() : ur_object_t_(), RefCount(1) {}
 
   uint32_t incrementReferenceCount() noexcept { return ++RefCount; }
 

source/adapters/hip/platform.hpp

@@ -19,6 +19,6 @@
 ///  available devices since initialization is done
 ///  when devices are used.
 ///
-struct ur_platform_handle_t_ {
+struct ur_platform_handle_t_ : ur_object_t_ {
   std::vector<std::unique_ptr<ur_device_handle_t_>> Devices;
 };

source/adapters/hip/program.hpp

@@ -17,7 +17,7 @@
 #include "context.hpp"
 
 /// Implementation of UR Program on HIP Module object
-struct ur_program_handle_t_ {
+struct ur_program_handle_t_ : ur_object_t_ {
   using native_type = hipModule_t;
   native_type Module;
   const char *Binary;
@@ -47,8 +47,9 @@ struct ur_program_handle_t_ {
   ur_program_build_status_t BuildStatus = UR_PROGRAM_BUILD_STATUS_NONE;
 
   ur_program_handle_t_(ur_context_handle_t Ctxt, ur_device_handle_t Device)
-      : Module{nullptr}, Binary{}, BinarySizeInBytes{0}, RefCount{1},
-        Context{Ctxt}, Device{Device}, KernelReqdWorkGroupSizeMD{} {
+      : ur_object_t_(), Module{nullptr}, Binary{}, BinarySizeInBytes{0},
+        RefCount{1}, Context{Ctxt}, Device{Device},
+        KernelReqdWorkGroupSizeMD{} {
     urContextRetain(Context);
     urDeviceRetain(Device);
   }

source/adapters/hip/queue.hpp

@@ -18,7 +18,7 @@ using ur_stream_guard = std::unique_lock<std::mutex>;
 
 /// UR queue mapping on to hipStream_t objects.
 ///
-struct ur_queue_handle_t_ {
+struct ur_queue_handle_t_ : ur_object_t_ {
   using native_type = hipStream_t;
   static constexpr int DefaultNumComputeStreams = 64;
   static constexpr int DefaultNumTransferStreams = 16;
@@ -66,7 +66,7 @@ struct ur_queue_handle_t_ {
                      ur_context_handle_t Context, ur_device_handle_t Device,
                      unsigned int Flags, ur_queue_flags_t URFlags, int Priority,
                      bool BackendOwns = true)
-      : ComputeStreams{std::move(ComputeStreams)},
+      : ur_object_t_(), ComputeStreams{std::move(ComputeStreams)},
         TransferStreams{std::move(TransferStreams)},
         DelayCompute(this->ComputeStreams.size(), false),
         ComputeAppliedBarrier(this->ComputeStreams.size()),

source/adapters/hip/sampler.hpp

@@ -17,13 +17,13 @@
 /// Sampler property layout:
 /// | 31 30 ... 6 5 |      4 3 2      |     1      |         0        |
 /// |      N/A      | addressing mode | fiter mode | normalize coords |
-struct ur_sampler_handle_t_ {
+struct ur_sampler_handle_t_ : ur_object_t_ {
   std::atomic_uint32_t RefCount;
   uint32_t Props;
   ur_context_handle_t Context;
 
   ur_sampler_handle_t_(ur_context_handle_t Context)
-      : RefCount(1), Props(0), Context(Context) {}
+      : ur_object_t_(), RefCount(1), Props(0), Context(Context) {}
 
   uint32_t incrementReferenceCount() noexcept { return ++RefCount; }