diff --git a/source/adapters/level_zero/CMakeLists.txt b/source/adapters/level_zero/CMakeLists.txt index c4a05f6a1b..784b01793f 100644 --- a/source/adapters/level_zero/CMakeLists.txt +++ b/source/adapters/level_zero/CMakeLists.txt @@ -27,6 +27,7 @@ if(UR_BUILD_ADAPTER_L0) ${CMAKE_CURRENT_SOURCE_DIR}/program.hpp ${CMAKE_CURRENT_SOURCE_DIR}/queue.hpp ${CMAKE_CURRENT_SOURCE_DIR}/sampler.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/helpers/image_helpers.hpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.hpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.hpp ${CMAKE_CURRENT_SOURCE_DIR}/ur_level_zero.cpp @@ -46,6 +47,7 @@ if(UR_BUILD_ADAPTER_L0) ${CMAKE_CURRENT_SOURCE_DIR}/queue.cpp ${CMAKE_CURRENT_SOURCE_DIR}/sampler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/image.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/helpers/image_helpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.cpp @@ -128,6 +130,7 @@ if(UR_BUILD_ADAPTER_L0_V2) ${CMAKE_CURRENT_SOURCE_DIR}/platform.hpp ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.hpp ${CMAKE_CURRENT_SOURCE_DIR}/program.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/helpers/image_helpers.hpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.hpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.hpp ${CMAKE_CURRENT_SOURCE_DIR}/adapter.cpp @@ -137,6 +140,7 @@ if(UR_BUILD_ADAPTER_L0_V2) ${CMAKE_CURRENT_SOURCE_DIR}/platform.cpp ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/program.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/helpers/image_helpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/usm_p2p.cpp diff --git a/source/adapters/level_zero/helpers/image_helpers.cpp b/source/adapters/level_zero/helpers/image_helpers.cpp new file mode 100644 index 0000000000..9da1750e8c --- /dev/null +++ b/source/adapters/level_zero/helpers/image_helpers.cpp @@ -0,0 +1,522 @@ +//===--------- image_helpers.cpp - Level Zero Adapter --------------------===// +// +// Copyright (C) 2024 Intel Corporation +// +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "image_helpers.hpp" + +/// Construct UR image format from ZE image desc. +ur_result_t ze2urImageFormat(const ze_image_desc_t *ZeImageDesc, + ur_image_format_t *UrImageFormat) { + const ze_image_format_t &ZeImageFormat = ZeImageDesc->format; + size_t ZeImageFormatTypeSize; + switch (ZeImageFormat.layout) { + case ZE_IMAGE_FORMAT_LAYOUT_8: + case ZE_IMAGE_FORMAT_LAYOUT_8_8: + case ZE_IMAGE_FORMAT_LAYOUT_8_8_8: + case ZE_IMAGE_FORMAT_LAYOUT_8_8_8_8: + ZeImageFormatTypeSize = 8; + break; + case ZE_IMAGE_FORMAT_LAYOUT_16: + case ZE_IMAGE_FORMAT_LAYOUT_16_16: + case ZE_IMAGE_FORMAT_LAYOUT_16_16_16: + case ZE_IMAGE_FORMAT_LAYOUT_16_16_16_16: + ZeImageFormatTypeSize = 16; + break; + case ZE_IMAGE_FORMAT_LAYOUT_32: + case ZE_IMAGE_FORMAT_LAYOUT_32_32: + case ZE_IMAGE_FORMAT_LAYOUT_32_32_32: + case ZE_IMAGE_FORMAT_LAYOUT_32_32_32_32: + ZeImageFormatTypeSize = 32; + break; + default: + logger::error( + "ze2urImageFormat: unsupported image format layout: layout = {}", + ZeImageFormat.layout); + return UR_RESULT_ERROR_INVALID_VALUE; + } + + ur_image_channel_order_t ChannelOrder; + switch (ZeImageFormat.layout) { + case ZE_IMAGE_FORMAT_LAYOUT_8: + case ZE_IMAGE_FORMAT_LAYOUT_16: + case ZE_IMAGE_FORMAT_LAYOUT_32: + switch (ZeImageFormat.x) { + case ZE_IMAGE_FORMAT_SWIZZLE_R: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_R; + break; + case ZE_IMAGE_FORMAT_SWIZZLE_A: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_A; + break; + default: + logger::error( + "ze2urImageFormat: unexpected image format channel x: x = {}", + ZeImageFormat.x); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + case ZE_IMAGE_FORMAT_LAYOUT_8_8: + case ZE_IMAGE_FORMAT_LAYOUT_16_16: + case ZE_IMAGE_FORMAT_LAYOUT_32_32: + if (ZeImageFormat.x != ZE_IMAGE_FORMAT_SWIZZLE_R) { + logger::error( + "ze2urImageFormat: unexpected image format channel x: x = {}", + ZeImageFormat.x); + return UR_RESULT_ERROR_INVALID_VALUE; + } + switch (ZeImageFormat.y) { + case ZE_IMAGE_FORMAT_SWIZZLE_G: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RG; + break; + case ZE_IMAGE_FORMAT_SWIZZLE_A: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RA; + break; + case ZE_IMAGE_FORMAT_SWIZZLE_X: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RX; + break; + default: + logger::error( + "ze2urImageFormat: unexpected image format channel y: y = {}\n", + ZeImageFormat.y); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + case ZE_IMAGE_FORMAT_LAYOUT_8_8_8: + case ZE_IMAGE_FORMAT_LAYOUT_16_16_16: + case ZE_IMAGE_FORMAT_LAYOUT_32_32_32: + if (ZeImageFormat.x == ZE_IMAGE_FORMAT_SWIZZLE_R && + ZeImageFormat.y == ZE_IMAGE_FORMAT_SWIZZLE_G) { + switch (ZeImageFormat.z) { + case ZE_IMAGE_FORMAT_SWIZZLE_B: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RGB; + break; + case ZE_IMAGE_FORMAT_SWIZZLE_X: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RGX; + break; + default: + logger::error( + "ze2urImageFormat: unexpected image format channel z: z = {}\n", + ZeImageFormat.z); + return UR_RESULT_ERROR_INVALID_VALUE; + } + } else { + logger::error("ze2urImageFormat: unexpected image format channel"); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + case ZE_IMAGE_FORMAT_LAYOUT_8_8_8_8: + case ZE_IMAGE_FORMAT_LAYOUT_16_16_16_16: + case ZE_IMAGE_FORMAT_LAYOUT_32_32_32_32: + if (ZeImageFormat.x == ZE_IMAGE_FORMAT_SWIZZLE_R && + ZeImageFormat.y == ZE_IMAGE_FORMAT_SWIZZLE_G && + ZeImageFormat.z == ZE_IMAGE_FORMAT_SWIZZLE_B) { + switch (ZeImageFormat.w) { + case ZE_IMAGE_FORMAT_SWIZZLE_X: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RGBX; + break; + case ZE_IMAGE_FORMAT_SWIZZLE_A: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RGBA; + break; + default: + logger::error("ze2urImageFormat: unexpected image format channel w: w " + "= {}", + ZeImageFormat.x); + return UR_RESULT_ERROR_INVALID_VALUE; + } + } else if (ZeImageFormat.x == ZE_IMAGE_FORMAT_SWIZZLE_A && + ZeImageFormat.y == ZE_IMAGE_FORMAT_SWIZZLE_R && + ZeImageFormat.z == ZE_IMAGE_FORMAT_SWIZZLE_G && + ZeImageFormat.w == ZE_IMAGE_FORMAT_SWIZZLE_B) { + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_ARGB; + } else if (ZeImageFormat.x == ZE_IMAGE_FORMAT_SWIZZLE_B && + ZeImageFormat.y == ZE_IMAGE_FORMAT_SWIZZLE_G && + ZeImageFormat.z == ZE_IMAGE_FORMAT_SWIZZLE_R && + ZeImageFormat.w == ZE_IMAGE_FORMAT_SWIZZLE_A) { + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_BGRA; + } else { + logger::error("ze2urImageFormat: unexpected image format channel"); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + default: + logger::error( + "ze2urImageFormat: unsupported image format layout: layout = {}", + ZeImageFormat.layout); + return UR_RESULT_ERROR_INVALID_VALUE; + } + + ur_image_channel_type_t ChannelType; + switch (ZeImageFormat.type) { + case ZE_IMAGE_FORMAT_TYPE_UINT: + switch (ZeImageFormatTypeSize) { + case 8: + ChannelType = UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT8; + break; + case 16: + ChannelType = UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT16; + break; + case 32: + ChannelType = UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT32; + break; + default: + logger::error("ze2urImageFormat: unexpected image format type size: size " + "= {}", + ZeImageFormatTypeSize); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + case ZE_IMAGE_FORMAT_TYPE_SINT: + switch (ZeImageFormatTypeSize) { + case 8: + ChannelType = UR_IMAGE_CHANNEL_TYPE_SIGNED_INT8; + break; + case 16: + ChannelType = UR_IMAGE_CHANNEL_TYPE_SIGNED_INT16; + break; + case 32: + ChannelType = UR_IMAGE_CHANNEL_TYPE_SIGNED_INT32; + break; + default: + logger::error("ze2urImageFormat: unexpected image format type size: size " + "= {}", + ZeImageFormatTypeSize); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + case ZE_IMAGE_FORMAT_TYPE_UNORM: + switch (ZeImageFormatTypeSize) { + case 8: + ChannelType = UR_IMAGE_CHANNEL_TYPE_UNORM_INT8; + break; + case 16: + ChannelType = UR_IMAGE_CHANNEL_TYPE_UNORM_INT16; + break; + default: + logger::error("ze2urImageFormat: unexpected image format type size: size " + "= {}", + ZeImageFormatTypeSize); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + case ZE_IMAGE_FORMAT_TYPE_SNORM: + switch (ZeImageFormatTypeSize) { + case 8: + ChannelType = UR_IMAGE_CHANNEL_TYPE_SNORM_INT8; + break; + case 16: + ChannelType = UR_IMAGE_CHANNEL_TYPE_SNORM_INT16; + break; + default: + logger::error("ze2urImageFormat: unexpected image format type size: size " + "= {}", + ZeImageFormatTypeSize); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + case ZE_IMAGE_FORMAT_TYPE_FLOAT: + switch (ZeImageFormatTypeSize) { + case 16: + ChannelType = UR_IMAGE_CHANNEL_TYPE_HALF_FLOAT; + break; + case 32: + ChannelType = UR_IMAGE_CHANNEL_TYPE_FLOAT; + break; + default: + logger::error("ze2urImageFormat: unexpected image format type size: size " + "= {}", + ZeImageFormatTypeSize); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + default: + logger::error("ze2urImageFormat: unsupported image format type: type = {}", + ZeImageFormat.type); + return UR_RESULT_ERROR_INVALID_VALUE; + } + + UrImageFormat->channelOrder = ChannelOrder; + UrImageFormat->channelType = ChannelType; + return UR_RESULT_SUCCESS; +} + +bool Is3ChannelOrder(ur_image_channel_order_t ChannelOrder) { + switch (ChannelOrder) { + case UR_IMAGE_CHANNEL_ORDER_RGB: + case UR_IMAGE_CHANNEL_ORDER_RGX: + return true; + default: + return false; + } +} + +/// Construct ZE image desc from UR image format and desc. +ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, + const ur_image_desc_t *ImageDesc, + ZeStruct &ZeImageDesc) { + + auto [ZeImageFormatType, ZeImageFormatTypeSize] = + getImageFormatTypeAndSize(ImageFormat); + + if (ZeImageFormatTypeSize == 0) { + return UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT; + } + + // TODO: populate the layout mapping + ze_image_format_layout_t ZeImageFormatLayout; + switch (ImageFormat->channelOrder) { + case UR_IMAGE_CHANNEL_ORDER_RGBA: { + switch (ZeImageFormatTypeSize) { + case 8: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_8_8_8_8; + break; + case 16: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_16_16_16_16; + break; + case 32: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32_32_32_32; + break; + default: + logger::error("ur2zeImageDesc: unexpected data type Size\n"); + return UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT; + } + break; + } + default: + logger::error("format layout = {}", ImageFormat->channelOrder); + return UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT; + break; + } + + ze_image_format_t ZeFormatDesc = { + ZeImageFormatLayout, ZeImageFormatType, + // TODO: are swizzles deducted from image_format->image_channel_order? + ZE_IMAGE_FORMAT_SWIZZLE_R, ZE_IMAGE_FORMAT_SWIZZLE_G, + ZE_IMAGE_FORMAT_SWIZZLE_B, ZE_IMAGE_FORMAT_SWIZZLE_A}; + + ze_image_type_t ZeImageType; + switch (ImageDesc->type) { + case UR_MEM_TYPE_IMAGE1D: + ZeImageType = ZE_IMAGE_TYPE_1D; + break; + case UR_MEM_TYPE_IMAGE2D: + ZeImageType = ZE_IMAGE_TYPE_2D; + break; + case UR_MEM_TYPE_IMAGE3D: + ZeImageType = ZE_IMAGE_TYPE_3D; + break; + case UR_MEM_TYPE_IMAGE1D_ARRAY: + ZeImageType = ZE_IMAGE_TYPE_1DARRAY; + break; + case UR_MEM_TYPE_IMAGE2D_ARRAY: + ZeImageType = ZE_IMAGE_TYPE_2DARRAY; + break; + default: + logger::error("ur2zeImageDesc: unsupported image type"); + return UR_RESULT_ERROR_INVALID_IMAGE_FORMAT_DESCRIPTOR; + } + + ZeImageDesc.arraylevels = ZeImageDesc.flags = 0; + ZeImageDesc.type = ZeImageType; + ZeImageDesc.format = ZeFormatDesc; + ZeImageDesc.width = ur_cast(ImageDesc->width); + ZeImageDesc.height = ur_cast(ImageDesc->height); + ZeImageDesc.depth = ur_cast(ImageDesc->depth); + ZeImageDesc.arraylevels = ur_cast(ImageDesc->arraySize); + ZeImageDesc.miplevels = ImageDesc->numMipLevel; + + return UR_RESULT_SUCCESS; +} + +/// Return element size in bytes of a pixel. +uint32_t getPixelSizeBytes(const ur_image_format_t *Format) { + uint32_t NumChannels = 0; + switch (Format->channelOrder) { + case UR_IMAGE_CHANNEL_ORDER_A: + case UR_IMAGE_CHANNEL_ORDER_R: + case UR_IMAGE_CHANNEL_ORDER_INTENSITY: + case UR_IMAGE_CHANNEL_ORDER_LUMINANCE: + case UR_IMAGE_CHANNEL_ORDER_FORCE_UINT32: + NumChannels = 1; + break; + case UR_IMAGE_CHANNEL_ORDER_RG: + case UR_IMAGE_CHANNEL_ORDER_RA: + case UR_IMAGE_CHANNEL_ORDER_RX: + NumChannels = 2; + break; + case UR_IMAGE_CHANNEL_ORDER_RGB: + case UR_IMAGE_CHANNEL_ORDER_RGX: + NumChannels = 3; + break; + case UR_IMAGE_CHANNEL_ORDER_RGBA: + case UR_IMAGE_CHANNEL_ORDER_BGRA: + case UR_IMAGE_CHANNEL_ORDER_ARGB: + case UR_IMAGE_CHANNEL_ORDER_ABGR: + case UR_IMAGE_CHANNEL_ORDER_RGBX: + case UR_IMAGE_CHANNEL_ORDER_SRGBA: + NumChannels = 4; + break; + default: + ur::unreachable(); + } + uint32_t ChannelTypeSizeInBytes = 0; + switch (Format->channelType) { + case UR_IMAGE_CHANNEL_TYPE_SNORM_INT8: + case UR_IMAGE_CHANNEL_TYPE_UNORM_INT8: + case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT8: + case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT8: + ChannelTypeSizeInBytes = 1; + break; + case UR_IMAGE_CHANNEL_TYPE_SNORM_INT16: + case UR_IMAGE_CHANNEL_TYPE_UNORM_INT16: + case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT16: + case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT16: + case UR_IMAGE_CHANNEL_TYPE_UNORM_SHORT_565: + case UR_IMAGE_CHANNEL_TYPE_UNORM_SHORT_555: + case UR_IMAGE_CHANNEL_TYPE_HALF_FLOAT: + ChannelTypeSizeInBytes = 2; + break; + case UR_IMAGE_CHANNEL_TYPE_INT_101010: + case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT32: + case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT32: + case UR_IMAGE_CHANNEL_TYPE_FLOAT: + case UR_IMAGE_CHANNEL_TYPE_FORCE_UINT32: + ChannelTypeSizeInBytes = 4; + break; + default: + ur::unreachable(); + } + return NumChannels * ChannelTypeSizeInBytes; +} + +ur_result_t getImageRegionHelper(ze_image_desc_t ZeImageDesc, + ur_rect_offset_t *Origin, + ur_rect_region_t *Region, + ze_image_region_t &ZeRegion) { + UR_ASSERT(Origin, UR_RESULT_ERROR_INVALID_VALUE); + UR_ASSERT(Region, UR_RESULT_ERROR_INVALID_VALUE); + + if (ZeImageDesc.type == ZE_IMAGE_TYPE_1D || + ZeImageDesc.type == ZE_IMAGE_TYPE_1DARRAY) { + Region->height = 1; + Region->depth = 1; + } else if (ZeImageDesc.type == ZE_IMAGE_TYPE_2D || + ZeImageDesc.type == ZE_IMAGE_TYPE_2DARRAY) { + Region->depth = 1; + } + +#ifndef NDEBUG + UR_ASSERT((ZeImageDesc.type == ZE_IMAGE_TYPE_1D && Origin->y == 0 && + Origin->z == 0) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_1DARRAY && Origin->z == 0) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_2D && Origin->z == 0) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_2DARRAY) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_3D), + UR_RESULT_ERROR_INVALID_VALUE); + + UR_ASSERT(Region->width && Region->height && Region->depth, + UR_RESULT_ERROR_INVALID_VALUE); + UR_ASSERT( + (ZeImageDesc.type == ZE_IMAGE_TYPE_1D && Region->height == 1 && + Region->depth == 1) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_1DARRAY && Region->depth == 1) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_2D && Region->depth == 1) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_2DARRAY) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_3D), + UR_RESULT_ERROR_INVALID_VALUE); +#endif // !NDEBUG + + uint32_t OriginX = ur_cast(Origin->x); + uint32_t OriginY = ur_cast(Origin->y); + uint32_t OriginZ = ur_cast(Origin->z); + + uint32_t Width = ur_cast(Region->width); + uint32_t Height = (ZeImageDesc.type == ZE_IMAGE_TYPE_1DARRAY) + ? ZeImageDesc.arraylevels + : ur_cast(Region->height); + uint32_t Depth = (ZeImageDesc.type == ZE_IMAGE_TYPE_2DARRAY) + ? ZeImageDesc.arraylevels + : ur_cast(Region->depth); + + ZeRegion = {OriginX, OriginY, OriginZ, Width, Height, Depth}; + + return UR_RESULT_SUCCESS; +} + +std::pair +getImageFormatTypeAndSize(const ur_image_format_t *ImageFormat) { + ze_image_format_type_t ZeImageFormatType; + size_t ZeImageFormatTypeSize; + switch (ImageFormat->channelType) { + case UR_IMAGE_CHANNEL_TYPE_FLOAT: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_FLOAT; + ZeImageFormatTypeSize = 32; + break; + } + case UR_IMAGE_CHANNEL_TYPE_HALF_FLOAT: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_FLOAT; + ZeImageFormatTypeSize = 16; + break; + } + case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT32: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UINT; + ZeImageFormatTypeSize = 32; + break; + } + case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT16: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UINT; + ZeImageFormatTypeSize = 16; + break; + } + case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT8: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UINT; + ZeImageFormatTypeSize = 8; + break; + } + case UR_IMAGE_CHANNEL_TYPE_UNORM_INT16: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UNORM; + ZeImageFormatTypeSize = 16; + break; + } + case UR_IMAGE_CHANNEL_TYPE_UNORM_INT8: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UNORM; + ZeImageFormatTypeSize = 8; + break; + } + case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT32: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SINT; + ZeImageFormatTypeSize = 32; + break; + } + case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT16: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SINT; + ZeImageFormatTypeSize = 16; + break; + } + case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT8: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SINT; + ZeImageFormatTypeSize = 8; + break; + } + case UR_IMAGE_CHANNEL_TYPE_SNORM_INT16: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SNORM; + ZeImageFormatTypeSize = 16; + break; + } + case UR_IMAGE_CHANNEL_TYPE_SNORM_INT8: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SNORM; + ZeImageFormatTypeSize = 8; + break; + } + default: + logger::error("ur2zeImageDesc: unsupported image data type: data type = {}", + ImageFormat->channelType); + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_FORCE_UINT32; + ZeImageFormatTypeSize = 0; + } + return {ZeImageFormatType, ZeImageFormatTypeSize}; +} diff --git a/source/adapters/level_zero/helpers/image_helpers.hpp b/source/adapters/level_zero/helpers/image_helpers.hpp new file mode 100644 index 0000000000..f64ed6fa7e --- /dev/null +++ b/source/adapters/level_zero/helpers/image_helpers.hpp @@ -0,0 +1,38 @@ +//===--------- image_helpers.hpp - Level Zero Adapter --------------------===// +// +// Copyright (C) 2024 Intel Corporation +// +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#pragma once + +#include +#include +#include + +#include "../common.hpp" + +/// Construct UR image format from ZE image desc. +ur_result_t ze2urImageFormat(const ze_image_desc_t *ZeImageDesc, + ur_image_format_t *UrImageFormat); + +/// Construct ZE image desc from UR image format and desc. +ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, + const ur_image_desc_t *ImageDesc, + ZeStruct &ZeImageDesc); + +/// Return element size in bytes of a pixel. +uint32_t getPixelSizeBytes(const ur_image_format_t *Format); + +bool Is3ChannelOrder(ur_image_channel_order_t ChannelOrder); + +ur_result_t getImageRegionHelper(ze_image_desc_t ZeImageDesc, + ur_rect_offset_t *Origin, + ur_rect_region_t *Region, + ze_image_region_t &ZeRegion); + +std::pair +getImageFormatTypeAndSize(const ur_image_format_t *ImageFormat); \ No newline at end of file diff --git a/source/adapters/level_zero/image.cpp b/source/adapters/level_zero/image.cpp index 217a185f5b..0a8fc1112d 100644 --- a/source/adapters/level_zero/image.cpp +++ b/source/adapters/level_zero/image.cpp @@ -8,15 +8,14 @@ // //===----------------------------------------------------------------------===// -#include "image.hpp" #include "common.hpp" #include "context.hpp" #include "event.hpp" +#include "helpers/image_helpers.hpp" #include "logger/ur_logger.hpp" #include "sampler.hpp" #include "ur_interface_loader.hpp" #include "ur_level_zero.hpp" -#include "ze_api.h" typedef ze_result_t(ZE_APICALL *zeImageGetDeviceOffsetExp_pfn)( ze_image_handle_t hImage, uint64_t *pDeviceOffset); @@ -31,254 +30,11 @@ zeMemGetPitchFor2dImage_pfn zeMemGetPitchFor2dImageFunctionPtr = nullptr; zeImageGetDeviceOffsetExp_pfn zeImageGetDeviceOffsetExpFunctionPtr = nullptr; -/// Construct UR image format from ZE image desc. -ur_result_t ze2urImageFormat(const ze_image_desc_t *ZeImageDesc, - ur_image_format_t *UrImageFormat) { - const ze_image_format_t &ZeImageFormat = ZeImageDesc->format; - size_t ZeImageFormatTypeSize; - switch (ZeImageFormat.layout) { - case ZE_IMAGE_FORMAT_LAYOUT_8: - case ZE_IMAGE_FORMAT_LAYOUT_8_8: - case ZE_IMAGE_FORMAT_LAYOUT_8_8_8: - case ZE_IMAGE_FORMAT_LAYOUT_8_8_8_8: - ZeImageFormatTypeSize = 8; - break; - case ZE_IMAGE_FORMAT_LAYOUT_16: - case ZE_IMAGE_FORMAT_LAYOUT_16_16: - case ZE_IMAGE_FORMAT_LAYOUT_16_16_16: - case ZE_IMAGE_FORMAT_LAYOUT_16_16_16_16: - ZeImageFormatTypeSize = 16; - break; - case ZE_IMAGE_FORMAT_LAYOUT_32: - case ZE_IMAGE_FORMAT_LAYOUT_32_32: - case ZE_IMAGE_FORMAT_LAYOUT_32_32_32: - case ZE_IMAGE_FORMAT_LAYOUT_32_32_32_32: - ZeImageFormatTypeSize = 32; - break; - default: - logger::error( - "ze2urImageFormat: unsupported image format layout: layout = {}", - ZeImageFormat.layout); - return UR_RESULT_ERROR_INVALID_VALUE; - } - - ur_image_channel_order_t ChannelOrder; - switch (ZeImageFormat.layout) { - case ZE_IMAGE_FORMAT_LAYOUT_8: - case ZE_IMAGE_FORMAT_LAYOUT_16: - case ZE_IMAGE_FORMAT_LAYOUT_32: - switch (ZeImageFormat.x) { - case ZE_IMAGE_FORMAT_SWIZZLE_R: - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_R; - break; - case ZE_IMAGE_FORMAT_SWIZZLE_A: - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_A; - break; - default: - logger::error( - "ze2urImageFormat: unexpected image format channel x: x = {}", - ZeImageFormat.x); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - case ZE_IMAGE_FORMAT_LAYOUT_8_8: - case ZE_IMAGE_FORMAT_LAYOUT_16_16: - case ZE_IMAGE_FORMAT_LAYOUT_32_32: - if (ZeImageFormat.x != ZE_IMAGE_FORMAT_SWIZZLE_R) { - logger::error( - "ze2urImageFormat: unexpected image format channel x: x = {}", - ZeImageFormat.x); - return UR_RESULT_ERROR_INVALID_VALUE; - } - switch (ZeImageFormat.y) { - case ZE_IMAGE_FORMAT_SWIZZLE_G: - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RG; - break; - case ZE_IMAGE_FORMAT_SWIZZLE_A: - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RA; - break; - case ZE_IMAGE_FORMAT_SWIZZLE_X: - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RX; - break; - default: - logger::error( - "ze2urImageFormat: unexpected image format channel y: y = {}\n", - ZeImageFormat.y); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - case ZE_IMAGE_FORMAT_LAYOUT_8_8_8: - case ZE_IMAGE_FORMAT_LAYOUT_16_16_16: - case ZE_IMAGE_FORMAT_LAYOUT_32_32_32: - if (ZeImageFormat.x == ZE_IMAGE_FORMAT_SWIZZLE_R && - ZeImageFormat.y == ZE_IMAGE_FORMAT_SWIZZLE_G) { - switch (ZeImageFormat.z) { - case ZE_IMAGE_FORMAT_SWIZZLE_B: - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RGB; - break; - case ZE_IMAGE_FORMAT_SWIZZLE_X: - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RGX; - break; - default: - logger::error( - "ze2urImageFormat: unexpected image format channel z: z = {}\n", - ZeImageFormat.z); - return UR_RESULT_ERROR_INVALID_VALUE; - } - } else { - logger::error("ze2urImageFormat: unexpected image format channel"); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - case ZE_IMAGE_FORMAT_LAYOUT_8_8_8_8: - case ZE_IMAGE_FORMAT_LAYOUT_16_16_16_16: - case ZE_IMAGE_FORMAT_LAYOUT_32_32_32_32: - if (ZeImageFormat.x == ZE_IMAGE_FORMAT_SWIZZLE_R && - ZeImageFormat.y == ZE_IMAGE_FORMAT_SWIZZLE_G && - ZeImageFormat.z == ZE_IMAGE_FORMAT_SWIZZLE_B) { - switch (ZeImageFormat.w) { - case ZE_IMAGE_FORMAT_SWIZZLE_X: - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RGBX; - break; - case ZE_IMAGE_FORMAT_SWIZZLE_A: - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RGBA; - break; - default: - logger::error("ze2urImageFormat: unexpected image format channel w: w " - "= {}", - ZeImageFormat.x); - return UR_RESULT_ERROR_INVALID_VALUE; - } - } else if (ZeImageFormat.x == ZE_IMAGE_FORMAT_SWIZZLE_A && - ZeImageFormat.y == ZE_IMAGE_FORMAT_SWIZZLE_R && - ZeImageFormat.z == ZE_IMAGE_FORMAT_SWIZZLE_G && - ZeImageFormat.w == ZE_IMAGE_FORMAT_SWIZZLE_B) { - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_ARGB; - } else if (ZeImageFormat.x == ZE_IMAGE_FORMAT_SWIZZLE_B && - ZeImageFormat.y == ZE_IMAGE_FORMAT_SWIZZLE_G && - ZeImageFormat.z == ZE_IMAGE_FORMAT_SWIZZLE_R && - ZeImageFormat.w == ZE_IMAGE_FORMAT_SWIZZLE_A) { - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_BGRA; - } else { - logger::error("ze2urImageFormat: unexpected image format channel"); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - default: - logger::error( - "ze2urImageFormat: unsupported image format layout: layout = {}", - ZeImageFormat.layout); - return UR_RESULT_ERROR_INVALID_VALUE; - } - - ur_image_channel_type_t ChannelType; - switch (ZeImageFormat.type) { - case ZE_IMAGE_FORMAT_TYPE_UINT: - switch (ZeImageFormatTypeSize) { - case 8: - ChannelType = UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT8; - break; - case 16: - ChannelType = UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT16; - break; - case 32: - ChannelType = UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT32; - break; - default: - logger::error("ze2urImageFormat: unexpected image format type size: size " - "= {}", - ZeImageFormatTypeSize); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - case ZE_IMAGE_FORMAT_TYPE_SINT: - switch (ZeImageFormatTypeSize) { - case 8: - ChannelType = UR_IMAGE_CHANNEL_TYPE_SIGNED_INT8; - break; - case 16: - ChannelType = UR_IMAGE_CHANNEL_TYPE_SIGNED_INT16; - break; - case 32: - ChannelType = UR_IMAGE_CHANNEL_TYPE_SIGNED_INT32; - break; - default: - logger::error("ze2urImageFormat: unexpected image format type size: size " - "= {}", - ZeImageFormatTypeSize); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - case ZE_IMAGE_FORMAT_TYPE_UNORM: - switch (ZeImageFormatTypeSize) { - case 8: - ChannelType = UR_IMAGE_CHANNEL_TYPE_UNORM_INT8; - break; - case 16: - ChannelType = UR_IMAGE_CHANNEL_TYPE_UNORM_INT16; - break; - default: - logger::error("ze2urImageFormat: unexpected image format type size: size " - "= {}", - ZeImageFormatTypeSize); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - case ZE_IMAGE_FORMAT_TYPE_SNORM: - switch (ZeImageFormatTypeSize) { - case 8: - ChannelType = UR_IMAGE_CHANNEL_TYPE_SNORM_INT8; - break; - case 16: - ChannelType = UR_IMAGE_CHANNEL_TYPE_SNORM_INT16; - break; - default: - logger::error("ze2urImageFormat: unexpected image format type size: size " - "= {}", - ZeImageFormatTypeSize); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - case ZE_IMAGE_FORMAT_TYPE_FLOAT: - switch (ZeImageFormatTypeSize) { - case 16: - ChannelType = UR_IMAGE_CHANNEL_TYPE_HALF_FLOAT; - break; - case 32: - ChannelType = UR_IMAGE_CHANNEL_TYPE_FLOAT; - break; - default: - logger::error("ze2urImageFormat: unexpected image format type size: size " - "= {}", - ZeImageFormatTypeSize); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - default: - logger::error("ze2urImageFormat: unsupported image format type: type = {}", - ZeImageFormat.type); - return UR_RESULT_ERROR_INVALID_VALUE; - } - - UrImageFormat->channelOrder = ChannelOrder; - UrImageFormat->channelType = ChannelType; - return UR_RESULT_SUCCESS; -} - -static bool Is3ChannelOrder(ur_image_channel_order_t ChannelOrder) { - switch (ChannelOrder) { - case UR_IMAGE_CHANNEL_ORDER_RGB: - case UR_IMAGE_CHANNEL_ORDER_RGX: - return true; - default: - return false; - } -} - -/// Construct ZE image desc from UR image format and desc. -ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, - const ur_image_desc_t *ImageDesc, - ZeStruct &ZeImageDesc) { +// TODO: unify with image_helpers.cpp/ur2zeImageDesc +static ur_result_t +ur2zeImageDescBindless(const ur_image_format_t *ImageFormat, + const ur_image_desc_t *ImageDesc, + ZeStruct &ZeImageDesc) { auto [ZeImageFormatType, ZeImageFormatTypeSize] = getImageFormatTypeAndSize(ImageFormat); if (ZeImageFormatTypeSize == 0) { @@ -300,7 +56,7 @@ ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32; break; default: - logger::error("ur2zeImageDesc: unexpected data type size"); + logger::error("ur2zeImageDescBindless: unexpected data type size"); return UR_RESULT_ERROR_INVALID_VALUE; } break; @@ -319,7 +75,7 @@ ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32_32; break; default: - logger::error("ur2zeImageDesc: unexpected data type size"); + logger::error("ur2zeImageDescBindless: unexpected data type size"); return UR_RESULT_ERROR_INVALID_VALUE; } break; @@ -339,7 +95,7 @@ ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32_32_32_32; break; default: - logger::error("ur2zeImageDesc: unexpected data type size"); + logger::error("ur2zeImageDescBindless: unexpected data type size"); return UR_RESULT_ERROR_INVALID_VALUE; } break; @@ -357,14 +113,14 @@ ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32_32_32; break; default: - logger::error("ur2zeImageDesc: unexpected data type size"); + logger::error("ur2zeImageDescBindless: unexpected data type size"); return UR_RESULT_ERROR_INVALID_VALUE; } break; } default: logger::error("format channel order = {}", ImageFormat->channelOrder); - die("ur2zeImageDesc: unsupported image channel order\n"); + die("ur2zeImageDescBindless: unsupported image channel order\n"); break; } @@ -392,7 +148,7 @@ ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, ZeImageType = ZE_IMAGE_TYPE_2DARRAY; break; default: - logger::error("ur2zeImageDesc: unsupported image type"); + logger::error("ur2zeImageDescBindless: unsupported image type"); return UR_RESULT_ERROR_INVALID_VALUE; } @@ -411,67 +167,6 @@ ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, return UR_RESULT_SUCCESS; } -/// Return element size in bytes of a pixel. -uint32_t getPixelSizeBytes(const ur_image_format_t *Format) { - uint32_t NumChannels = 0; - switch (Format->channelOrder) { - case UR_IMAGE_CHANNEL_ORDER_A: - case UR_IMAGE_CHANNEL_ORDER_R: - case UR_IMAGE_CHANNEL_ORDER_INTENSITY: - case UR_IMAGE_CHANNEL_ORDER_LUMINANCE: - case UR_IMAGE_CHANNEL_ORDER_FORCE_UINT32: - NumChannels = 1; - break; - case UR_IMAGE_CHANNEL_ORDER_RG: - case UR_IMAGE_CHANNEL_ORDER_RA: - case UR_IMAGE_CHANNEL_ORDER_RX: - NumChannels = 2; - break; - case UR_IMAGE_CHANNEL_ORDER_RGB: - case UR_IMAGE_CHANNEL_ORDER_RGX: - NumChannels = 3; - break; - case UR_IMAGE_CHANNEL_ORDER_RGBA: - case UR_IMAGE_CHANNEL_ORDER_BGRA: - case UR_IMAGE_CHANNEL_ORDER_ARGB: - case UR_IMAGE_CHANNEL_ORDER_ABGR: - case UR_IMAGE_CHANNEL_ORDER_RGBX: - case UR_IMAGE_CHANNEL_ORDER_SRGBA: - NumChannels = 4; - break; - default: - ur::unreachable(); - } - uint32_t ChannelTypeSizeInBytes = 0; - switch (Format->channelType) { - case UR_IMAGE_CHANNEL_TYPE_SNORM_INT8: - case UR_IMAGE_CHANNEL_TYPE_UNORM_INT8: - case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT8: - case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT8: - ChannelTypeSizeInBytes = 1; - break; - case UR_IMAGE_CHANNEL_TYPE_SNORM_INT16: - case UR_IMAGE_CHANNEL_TYPE_UNORM_INT16: - case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT16: - case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT16: - case UR_IMAGE_CHANNEL_TYPE_UNORM_SHORT_565: - case UR_IMAGE_CHANNEL_TYPE_UNORM_SHORT_555: - case UR_IMAGE_CHANNEL_TYPE_HALF_FLOAT: - ChannelTypeSizeInBytes = 2; - break; - case UR_IMAGE_CHANNEL_TYPE_INT_101010: - case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT32: - case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT32: - case UR_IMAGE_CHANNEL_TYPE_FLOAT: - case UR_IMAGE_CHANNEL_TYPE_FORCE_UINT32: - ChannelTypeSizeInBytes = 4; - break; - default: - ur::unreachable(); - } - return NumChannels * ChannelTypeSizeInBytes; -} - ur_result_t bindlessImagesCreateImpl(ur_context_handle_t hContext, ur_device_handle_t hDevice, ur_exp_image_mem_native_handle_t hImageMem, @@ -487,7 +182,7 @@ ur_result_t bindlessImagesCreateImpl(ur_context_handle_t hContext, UR_RESULT_ERROR_INVALID_NULL_POINTER); ZeStruct ZeImageDesc; - UR_CALL(ur2zeImageDesc(pImageFormat, pImageDesc, ZeImageDesc)); + UR_CALL(ur2zeImageDescBindless(pImageFormat, pImageDesc, ZeImageDesc)); ZeStruct BindlessDesc; BindlessDesc.flags = ZE_IMAGE_BINDLESS_EXP_FLAG_BINDLESS; @@ -564,135 +259,6 @@ ur_result_t bindlessImagesCreateImpl(ur_context_handle_t hContext, } // namespace -ur_result_t getImageRegionHelper(ze_image_desc_t ZeImageDesc, - ur_rect_offset_t *Origin, - ur_rect_region_t *Region, - ze_image_region_t &ZeRegion) { - UR_ASSERT(Origin, UR_RESULT_ERROR_INVALID_VALUE); - UR_ASSERT(Region, UR_RESULT_ERROR_INVALID_VALUE); - - if (ZeImageDesc.type == ZE_IMAGE_TYPE_1D || - ZeImageDesc.type == ZE_IMAGE_TYPE_1DARRAY) { - Region->height = 1; - Region->depth = 1; - } else if (ZeImageDesc.type == ZE_IMAGE_TYPE_2D || - ZeImageDesc.type == ZE_IMAGE_TYPE_2DARRAY) { - Region->depth = 1; - } - -#ifndef NDEBUG - UR_ASSERT((ZeImageDesc.type == ZE_IMAGE_TYPE_1D && Origin->y == 0 && - Origin->z == 0) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_1DARRAY && Origin->z == 0) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_2D && Origin->z == 0) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_2DARRAY) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_3D), - UR_RESULT_ERROR_INVALID_VALUE); - - UR_ASSERT(Region->width && Region->height && Region->depth, - UR_RESULT_ERROR_INVALID_VALUE); - UR_ASSERT( - (ZeImageDesc.type == ZE_IMAGE_TYPE_1D && Region->height == 1 && - Region->depth == 1) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_1DARRAY && Region->depth == 1) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_2D && Region->depth == 1) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_2DARRAY) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_3D), - UR_RESULT_ERROR_INVALID_VALUE); -#endif // !NDEBUG - - uint32_t OriginX = ur_cast(Origin->x); - uint32_t OriginY = ur_cast(Origin->y); - uint32_t OriginZ = ur_cast(Origin->z); - - uint32_t Width = ur_cast(Region->width); - uint32_t Height = (ZeImageDesc.type == ZE_IMAGE_TYPE_1DARRAY) - ? ZeImageDesc.arraylevels - : ur_cast(Region->height); - uint32_t Depth = (ZeImageDesc.type == ZE_IMAGE_TYPE_2DARRAY) - ? ZeImageDesc.arraylevels - : ur_cast(Region->depth); - - ZeRegion = {OriginX, OriginY, OriginZ, Width, Height, Depth}; - - return UR_RESULT_SUCCESS; -} - -std::pair -getImageFormatTypeAndSize(const ur_image_format_t *ImageFormat) { - ze_image_format_type_t ZeImageFormatType; - size_t ZeImageFormatTypeSize; - switch (ImageFormat->channelType) { - case UR_IMAGE_CHANNEL_TYPE_FLOAT: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_FLOAT; - ZeImageFormatTypeSize = 32; - break; - } - case UR_IMAGE_CHANNEL_TYPE_HALF_FLOAT: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_FLOAT; - ZeImageFormatTypeSize = 16; - break; - } - case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT32: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UINT; - ZeImageFormatTypeSize = 32; - break; - } - case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT16: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UINT; - ZeImageFormatTypeSize = 16; - break; - } - case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT8: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UINT; - ZeImageFormatTypeSize = 8; - break; - } - case UR_IMAGE_CHANNEL_TYPE_UNORM_INT16: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UNORM; - ZeImageFormatTypeSize = 16; - break; - } - case UR_IMAGE_CHANNEL_TYPE_UNORM_INT8: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UNORM; - ZeImageFormatTypeSize = 8; - break; - } - case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT32: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SINT; - ZeImageFormatTypeSize = 32; - break; - } - case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT16: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SINT; - ZeImageFormatTypeSize = 16; - break; - } - case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT8: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SINT; - ZeImageFormatTypeSize = 8; - break; - } - case UR_IMAGE_CHANNEL_TYPE_SNORM_INT16: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SNORM; - ZeImageFormatTypeSize = 16; - break; - } - case UR_IMAGE_CHANNEL_TYPE_SNORM_INT8: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SNORM; - ZeImageFormatTypeSize = 8; - break; - } - default: - logger::error( - "urMemImageCreate: unsupported image data type: data type = {}", - ImageFormat->channelType); - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_FORCE_UINT32; - ZeImageFormatTypeSize = 0; - } - return {ZeImageFormatType, ZeImageFormatTypeSize}; -} - namespace ur::level_zero { ur_result_t urUSMPitchedAllocExp(ur_context_handle_t hContext, @@ -776,7 +342,7 @@ ur_result_t urBindlessImagesImageAllocateExp( UR_RESULT_ERROR_INVALID_NULL_POINTER); ZeStruct ZeImageDesc; - UR_CALL(ur2zeImageDesc(pImageFormat, pImageDesc, ZeImageDesc)); + UR_CALL(ur2zeImageDescBindless(pImageFormat, pImageDesc, ZeImageDesc)); ze_image_bindless_exp_desc_t ZeImageBindlessDesc; ZeImageBindlessDesc.stype = ZE_STRUCTURE_TYPE_BINDLESS_IMAGE_EXP_DESC; @@ -851,7 +417,7 @@ ur_result_t urBindlessImagesImageCopyExp( UR_RESULT_ERROR_INVALID_IMAGE_FORMAT_DESCRIPTOR); ZeStruct ZeImageDesc; - UR_CALL(ur2zeImageDesc(pSrcImageFormat, pSrcImageDesc, ZeImageDesc)); + UR_CALL(ur2zeImageDescBindless(pSrcImageFormat, pSrcImageDesc, ZeImageDesc)); bool UseCopyEngine = hQueue->useCopyEngine(/*PreferCopyEngine*/ true); // Due to the limitation of the copy engine, disable usage of Copy Engine @@ -1142,7 +708,7 @@ ur_result_t urBindlessImagesMapExternalArrayExp( ZeImageBindlessDesc.stype = ZE_STRUCTURE_TYPE_BINDLESS_IMAGE_EXP_DESC; ZeStruct ZeImageDesc; - UR_CALL(ur2zeImageDesc(pImageFormat, pImageDesc, ZeImageDesc)); + UR_CALL(ur2zeImageDescBindless(pImageFormat, pImageDesc, ZeImageDesc)); ZeImageBindlessDesc.pNext = externalMemoryData->importExtensionDesc; ZeImageBindlessDesc.flags = ZE_IMAGE_BINDLESS_EXP_FLAG_BINDLESS; diff --git a/source/adapters/level_zero/image.hpp b/source/adapters/level_zero/image.hpp deleted file mode 100644 index 43f37fa757..0000000000 --- a/source/adapters/level_zero/image.hpp +++ /dev/null @@ -1,23 +0,0 @@ -//===--------- image.hpp - Level Zero Adapter -----------------------------===// -// -// Copyright (C) 2023 Intel Corporation -// -// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM -// Exceptions. See LICENSE.TXT -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -#pragma once - -#include -#include -#include -#include - -ur_result_t getImageRegionHelper(ze_image_desc_t ZeImageDesc, - ur_rect_offset_t *Origin, - ur_rect_region_t *Region, - ze_image_region_t &ZeRegion); - -std::pair -getImageFormatTypeAndSize(const ur_image_format_t *ImageFormat); diff --git a/source/adapters/level_zero/memory.cpp b/source/adapters/level_zero/memory.cpp index e21470eee2..dd1be1c73d 100644 --- a/source/adapters/level_zero/memory.cpp +++ b/source/adapters/level_zero/memory.cpp @@ -15,8 +15,8 @@ #include "context.hpp" #include "event.hpp" +#include "helpers/image_helpers.hpp" #include "helpers/memory_helpers.hpp" -#include "image.hpp" #include "logger/ur_logger.hpp" #include "queue.hpp" #include "ur_interface_loader.hpp" @@ -1487,83 +1487,6 @@ ur_result_t urEnqueueUSMMemcpy2D( PreferCopyEngineUsage(Queue->Device, Queue->Context, Src, Dst)); } -static ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, - const ur_image_desc_t *ImageDesc, - ZeStruct &ZeImageDesc) { - - auto [ZeImageFormatType, ZeImageFormatTypeSize] = - getImageFormatTypeAndSize(ImageFormat); - - if (ZeImageFormatTypeSize == 0) { - return UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT; - } - - // TODO: populate the layout mapping - ze_image_format_layout_t ZeImageFormatLayout; - switch (ImageFormat->channelOrder) { - case UR_IMAGE_CHANNEL_ORDER_RGBA: { - switch (ZeImageFormatTypeSize) { - case 8: - ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_8_8_8_8; - break; - case 16: - ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_16_16_16_16; - break; - case 32: - ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32_32_32_32; - break; - default: - logger::error("urMemImageCreate: unexpected data type Size\n"); - return UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT; - } - break; - } - default: - logger::error("format layout = {}", ImageFormat->channelOrder); - return UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT; - break; - } - - ze_image_format_t ZeFormatDesc = { - ZeImageFormatLayout, ZeImageFormatType, - // TODO: are swizzles deducted from image_format->image_channel_order? - ZE_IMAGE_FORMAT_SWIZZLE_R, ZE_IMAGE_FORMAT_SWIZZLE_G, - ZE_IMAGE_FORMAT_SWIZZLE_B, ZE_IMAGE_FORMAT_SWIZZLE_A}; - - ze_image_type_t ZeImageType; - switch (ImageDesc->type) { - case UR_MEM_TYPE_IMAGE1D: - ZeImageType = ZE_IMAGE_TYPE_1D; - break; - case UR_MEM_TYPE_IMAGE2D: - ZeImageType = ZE_IMAGE_TYPE_2D; - break; - case UR_MEM_TYPE_IMAGE3D: - ZeImageType = ZE_IMAGE_TYPE_3D; - break; - case UR_MEM_TYPE_IMAGE1D_ARRAY: - ZeImageType = ZE_IMAGE_TYPE_1DARRAY; - break; - case UR_MEM_TYPE_IMAGE2D_ARRAY: - ZeImageType = ZE_IMAGE_TYPE_2DARRAY; - break; - default: - logger::error("urMemImageCreate: unsupported image type"); - return UR_RESULT_ERROR_INVALID_IMAGE_FORMAT_DESCRIPTOR; - } - - ZeImageDesc.arraylevels = ZeImageDesc.flags = 0; - ZeImageDesc.type = ZeImageType; - ZeImageDesc.format = ZeFormatDesc; - ZeImageDesc.width = ur_cast(ImageDesc->width); - ZeImageDesc.height = ur_cast(ImageDesc->height); - ZeImageDesc.depth = ur_cast(ImageDesc->depth); - ZeImageDesc.arraylevels = ur_cast(ImageDesc->arraySize); - ZeImageDesc.miplevels = ImageDesc->numMipLevel; - - return UR_RESULT_SUCCESS; -} - ur_result_t urMemImageCreate( /// [in] handle of the context object ur_context_handle_t Context, diff --git a/source/adapters/level_zero/ur_level_zero.hpp b/source/adapters/level_zero/ur_level_zero.hpp index 36965c5d58..221139c2b7 100644 --- a/source/adapters/level_zero/ur_level_zero.hpp +++ b/source/adapters/level_zero/ur_level_zero.hpp @@ -28,7 +28,6 @@ #include "context.hpp" #include "device.hpp" #include "event.hpp" -#include "image.hpp" #include "kernel.hpp" #include "memory.hpp" #include "physical_mem.hpp" diff --git a/source/adapters/level_zero/v2/api.cpp b/source/adapters/level_zero/v2/api.cpp index c7bd97c32c..892c86f47f 100644 --- a/source/adapters/level_zero/v2/api.cpp +++ b/source/adapters/level_zero/v2/api.cpp @@ -27,29 +27,6 @@ urContextSetExtendedDeleter(ur_context_handle_t hContext, return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t urMemImageCreate(ur_context_handle_t hContext, ur_mem_flags_t flags, - const ur_image_format_t *pImageFormat, - const ur_image_desc_t *pImageDesc, void *pHost, - ur_mem_handle_t *phMem) { - logger::error("{} function not implemented!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -ur_result_t urMemImageCreateWithNativeHandle( - ur_native_handle_t hNativeMem, ur_context_handle_t hContext, - const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, - const ur_mem_native_properties_t *pProperties, ur_mem_handle_t *phMem) { - logger::error("{} function not implemented!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -ur_result_t urMemImageGetInfo(ur_mem_handle_t hMemory, ur_image_info_t propName, - size_t propSize, void *pPropValue, - size_t *pPropSizeRet) { - logger::error("{} function not implemented!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - ur_result_t urKernelSetSpecializationConstants( ur_kernel_handle_t hKernel, uint32_t count, const ur_specialization_constant_info_t *pSpecConstants) { diff --git a/source/adapters/level_zero/v2/common.hpp b/source/adapters/level_zero/v2/common.hpp index a17263388b..0543dbd372 100644 --- a/source/adapters/level_zero/v2/common.hpp +++ b/source/adapters/level_zero/v2/common.hpp @@ -118,5 +118,8 @@ using ze_context_handle_t = HANDLE_WRAPPER_TYPE(::ze_context_handle_t, using ze_command_list_handle_t = HANDLE_WRAPPER_TYPE(::ze_command_list_handle_t, zeCommandListDestroy); +using ze_image_handle_t = + ze_handle_wrapper<::ze_image_handle_t, zeImageDestroy>; + } // namespace raii } // namespace v2 diff --git a/source/adapters/level_zero/v2/kernel.cpp b/source/adapters/level_zero/v2/kernel.cpp index 93a686ec6b..cd81e4f47e 100644 --- a/source/adapters/level_zero/v2/kernel.cpp +++ b/source/adapters/level_zero/v2/kernel.cpp @@ -291,9 +291,14 @@ ur_result_t ur_kernel_handle_t_::prepareForSubmission( for (auto &pending : pending_allocations) { void *zePtr = nullptr; if (pending.hMem) { - // NULL is a valid value - zePtr = pending.hMem->getDevicePtr(hDevice, pending.mode, 0, - pending.hMem->getSize(), migrate); + if (!pending.hMem->isImage()) { + auto hBuffer = pending.hMem->getBuffer(); + zePtr = hBuffer->getDevicePtr(hDevice, pending.mode, 0, + hBuffer->getSize(), migrate); + } else { + auto hImage = static_cast(pending.hMem->getImage()); + zePtr = reinterpret_cast(hImage->getZeImage()); + } } UR_CALL(setArgPointer(pending.argIndex, nullptr, zePtr)); } @@ -411,21 +416,21 @@ ur_result_t urKernelSetArgPointer( return exceptionToResult(std::current_exception()); } -static ur_mem_handle_t_::device_access_mode_t memAccessFromKernelProperties( +static ur_mem_buffer_t::device_access_mode_t memAccessFromKernelProperties( const ur_kernel_arg_mem_obj_properties_t *pProperties) { if (pProperties) { switch (pProperties->memoryAccess) { case UR_MEM_FLAG_READ_WRITE: - return ur_mem_handle_t_::device_access_mode_t::read_write; + return ur_mem_buffer_t::device_access_mode_t::read_write; case UR_MEM_FLAG_WRITE_ONLY: - return ur_mem_handle_t_::device_access_mode_t::write_only; + return ur_mem_buffer_t::device_access_mode_t::write_only; case UR_MEM_FLAG_READ_ONLY: - return ur_mem_handle_t_::device_access_mode_t::read_only; + return ur_mem_buffer_t::device_access_mode_t::read_only; default: - return ur_mem_handle_t_::device_access_mode_t::read_write; + return ur_mem_buffer_t::device_access_mode_t::read_write; } } - return ur_mem_handle_t_::device_access_mode_t::read_write; + return ur_mem_buffer_t::device_access_mode_t::read_write; } ur_result_t diff --git a/source/adapters/level_zero/v2/kernel.hpp b/source/adapters/level_zero/v2/kernel.hpp index 798c9f18df..1d82d8e939 100644 --- a/source/adapters/level_zero/v2/kernel.hpp +++ b/source/adapters/level_zero/v2/kernel.hpp @@ -105,7 +105,7 @@ struct ur_kernel_handle_t_ : _ur_object { struct pending_memory_allocation_t { ur_mem_handle_t hMem; - ur_mem_handle_t_::device_access_mode_t mode; + ur_mem_buffer_t::device_access_mode_t mode; uint32_t argIndex; }; diff --git a/source/adapters/level_zero/v2/memory.cpp b/source/adapters/level_zero/v2/memory.cpp index 3ec16e8352..d005566b60 100644 --- a/source/adapters/level_zero/v2/memory.cpp +++ b/source/adapters/level_zero/v2/memory.cpp @@ -11,43 +11,52 @@ #include "memory.hpp" #include "context.hpp" +#include "../helpers/image_helpers.hpp" #include "../helpers/memory_helpers.hpp" -static ur_mem_handle_t_::device_access_mode_t +template ur_result_t retain(MemHandle hMem) { + hMem->RefCount.increment(); + return UR_RESULT_SUCCESS; +} + +template ur_result_t release(MemHandle hMem) { + if (hMem->RefCount.decrementAndTest()) { + delete hMem; + } + return UR_RESULT_SUCCESS; +} + +static ur_mem_buffer_t::device_access_mode_t getDeviceAccessMode(ur_mem_flags_t memFlag) { if (memFlag & UR_MEM_FLAG_READ_WRITE) { - return ur_mem_handle_t_::device_access_mode_t::read_write; + return ur_mem_buffer_t::device_access_mode_t::read_write; } else if (memFlag & UR_MEM_FLAG_READ_ONLY) { - return ur_mem_handle_t_::device_access_mode_t::read_only; + return ur_mem_buffer_t::device_access_mode_t::read_only; } else if (memFlag & UR_MEM_FLAG_WRITE_ONLY) { - return ur_mem_handle_t_::device_access_mode_t::write_only; + return ur_mem_buffer_t::device_access_mode_t::write_only; } else { - return ur_mem_handle_t_::device_access_mode_t::read_write; + return ur_mem_buffer_t::device_access_mode_t::read_write; } } -static bool isAccessCompatible(ur_mem_handle_t_::device_access_mode_t requested, - ur_mem_handle_t_::device_access_mode_t actual) { +static bool isAccessCompatible(ur_mem_buffer_t::device_access_mode_t requested, + ur_mem_buffer_t::device_access_mode_t actual) { return requested == actual || - actual == ur_mem_handle_t_::device_access_mode_t::read_write; + actual == ur_mem_buffer_t::device_access_mode_t::read_write; } -ur_mem_handle_t_::ur_mem_handle_t_(ur_context_handle_t hContext, size_t size, - device_access_mode_t accessMode) - : accessMode(accessMode), hContext(hContext), size(size) {} - -size_t ur_mem_handle_t_::getSize() const { return size; } +ur_mem_buffer_t::ur_mem_buffer_t(ur_context_handle_t hContext, size_t size, + device_access_mode_t accessMode) + : hContext(hContext), size(size), accessMode(accessMode) {} -ur_shared_mutex &ur_mem_handle_t_::getMutex() { return Mutex; } +ur_shared_mutex &ur_mem_buffer_t::getMutex() { return Mutex; } -ur_usm_handle_t_::ur_usm_handle_t_(ur_context_handle_t hContext, size_t size, - const void *ptr) - : ur_mem_handle_t_(hContext, size, device_access_mode_t::read_write), +ur_usm_handle_t::ur_usm_handle_t(ur_context_handle_t hContext, size_t size, + const void *ptr) + : ur_mem_buffer_t(hContext, size, device_access_mode_t::read_write), ptr(const_cast(ptr)) {} -ur_usm_handle_t_::~ur_usm_handle_t_() {} - -void *ur_usm_handle_t_::getDevicePtr( +void *ur_usm_handle_t::getDevicePtr( ur_device_handle_t hDevice, device_access_mode_t access, size_t offset, size_t size, std::function migrate) { std::ignore = hDevice; @@ -58,25 +67,25 @@ void *ur_usm_handle_t_::getDevicePtr( return ptr; } -void *ur_usm_handle_t_::mapHostPtr( - ur_map_flags_t flags, size_t offset, size_t size, - std::function) { +void * +ur_usm_handle_t::mapHostPtr(ur_map_flags_t flags, size_t offset, size_t size, + std::function) { std::ignore = flags; std::ignore = offset; std::ignore = size; return ptr; } -void ur_usm_handle_t_::unmapHostPtr( +void ur_usm_handle_t::unmapHostPtr( void *pMappedPtr, std::function) { std::ignore = pMappedPtr; /* nop */ } -ur_integrated_mem_handle_t::ur_integrated_mem_handle_t( +ur_integrated_buffer_handle_t::ur_integrated_buffer_handle_t( ur_context_handle_t hContext, void *hostPtr, size_t size, host_ptr_action_t hostPtrAction, device_access_mode_t accessMode) - : ur_mem_handle_t_(hContext, size, accessMode) { + : ur_mem_buffer_t(hContext, size, accessMode) { bool hostPtrImported = false; if (hostPtrAction == host_ptr_action_t::import) { hostPtrImported = @@ -107,10 +116,10 @@ ur_integrated_mem_handle_t::ur_integrated_mem_handle_t( } } -ur_integrated_mem_handle_t::ur_integrated_mem_handle_t( +ur_integrated_buffer_handle_t::ur_integrated_buffer_handle_t( ur_context_handle_t hContext, void *hostPtr, size_t size, device_access_mode_t accessMode, bool ownHostPtr) - : ur_mem_handle_t_(hContext, size, accessMode) { + : ur_mem_buffer_t(hContext, size, accessMode) { this->ptr = usm_unique_ptr_t(hostPtr, [hContext, ownHostPtr](void *ptr) { if (!ownHostPtr) { return; @@ -119,7 +128,7 @@ ur_integrated_mem_handle_t::ur_integrated_mem_handle_t( }); } -void *ur_integrated_mem_handle_t::getDevicePtr( +void *ur_integrated_buffer_handle_t::getDevicePtr( ur_device_handle_t hDevice, device_access_mode_t access, size_t offset, size_t size, std::function migrate) { std::ignore = hDevice; @@ -130,7 +139,7 @@ void *ur_integrated_mem_handle_t::getDevicePtr( return ptr.get(); } -void *ur_integrated_mem_handle_t::mapHostPtr( +void *ur_integrated_buffer_handle_t::mapHostPtr( ur_map_flags_t flags, size_t offset, size_t size, std::function migrate) { std::ignore = flags; @@ -140,22 +149,28 @@ void *ur_integrated_mem_handle_t::mapHostPtr( return ptr.get(); } -void ur_integrated_mem_handle_t::unmapHostPtr( +void ur_integrated_buffer_handle_t::unmapHostPtr( void *pMappedPtr, std::function) { std::ignore = pMappedPtr; /* nop */ } -static ur_result_t synchronousZeCopy(ur_context_handle_t hContext, - ur_device_handle_t hDevice, void *dst, - const void *src, size_t size) { - auto commandList = hContext->commandListCache.getImmediateCommandList( +static v2::raii::command_list_unique_handle +getSyncCommandListForCopy(ur_context_handle_t hContext, + ur_device_handle_t hDevice) { + return hContext->commandListCache.getImmediateCommandList( hDevice->ZeDevice, true, hDevice ->QueueGroup[ur_device_handle_t_::queue_group_info_t::type::Compute] .ZeOrdinal, true, ZE_COMMAND_QUEUE_MODE_SYNCHRONOUS, ZE_COMMAND_QUEUE_PRIORITY_NORMAL, std::nullopt); +} + +static ur_result_t synchronousZeCopy(ur_context_handle_t hContext, + ur_device_handle_t hDevice, void *dst, + const void *src, size_t size) { + auto commandList = getSyncCommandListForCopy(hContext, hDevice); ZE2UR_CALL(zeCommandListAppendMemoryCopy, (commandList.get(), dst, src, size, nullptr, 0, nullptr)); @@ -163,8 +178,8 @@ static ur_result_t synchronousZeCopy(ur_context_handle_t hContext, return UR_RESULT_SUCCESS; } -void *ur_discrete_mem_handle_t::allocateOnDevice(ur_device_handle_t hDevice, - size_t size) { +void *ur_discrete_buffer_handle_t::allocateOnDevice(ur_device_handle_t hDevice, + size_t size) { assert(hDevice); auto id = hDevice->Id.value(); @@ -188,9 +203,9 @@ void *ur_discrete_mem_handle_t::allocateOnDevice(ur_device_handle_t hDevice, } ur_result_t -ur_discrete_mem_handle_t::migrateBufferTo(ur_device_handle_t hDevice, void *src, - size_t size) { - TRACK_SCOPE_LATENCY("ur_discrete_mem_handle_t::migrateBufferTo"); +ur_discrete_buffer_handle_t::migrateBufferTo(ur_device_handle_t hDevice, + void *src, size_t size) { + TRACK_SCOPE_LATENCY("ur_discrete_buffer_handle_t::migrateBufferTo"); auto Id = hDevice->Id.value(); void *dst = deviceAllocations[Id].get() ? deviceAllocations[Id].get() @@ -201,10 +216,10 @@ ur_discrete_mem_handle_t::migrateBufferTo(ur_device_handle_t hDevice, void *src, return UR_RESULT_SUCCESS; } -ur_discrete_mem_handle_t::ur_discrete_mem_handle_t( +ur_discrete_buffer_handle_t::ur_discrete_buffer_handle_t( ur_context_handle_t hContext, void *hostPtr, size_t size, device_access_mode_t accessMode) - : ur_mem_handle_t_(hContext, size, accessMode), + : ur_mem_buffer_t(hContext, size, accessMode), deviceAllocations(hContext->getPlatform()->getNumDevices()), activeAllocationDevice(nullptr), mapToPtr(hostPtr), hostAllocations() { if (hostPtr) { @@ -213,11 +228,11 @@ ur_discrete_mem_handle_t::ur_discrete_mem_handle_t( } } -ur_discrete_mem_handle_t::ur_discrete_mem_handle_t( +ur_discrete_buffer_handle_t::ur_discrete_buffer_handle_t( ur_context_handle_t hContext, ur_device_handle_t hDevice, void *devicePtr, size_t size, device_access_mode_t accessMode, void *writeBackMemory, bool ownZePtr) - : ur_mem_handle_t_(hContext, size, accessMode), + : ur_mem_buffer_t(hContext, size, accessMode), deviceAllocations(hContext->getPlatform()->getNumDevices()), activeAllocationDevice(hDevice), writeBackPtr(writeBackMemory), hostAllocations() { @@ -236,7 +251,7 @@ ur_discrete_mem_handle_t::ur_discrete_mem_handle_t( } } -ur_discrete_mem_handle_t::~ur_discrete_mem_handle_t() { +ur_discrete_buffer_handle_t::~ur_discrete_buffer_handle_t() { if (!activeAllocationDevice || !writeBackPtr) return; @@ -245,17 +260,17 @@ ur_discrete_mem_handle_t::~ur_discrete_mem_handle_t() { getSize()); } -void *ur_discrete_mem_handle_t::getActiveDeviceAlloc(size_t offset) { +void *ur_discrete_buffer_handle_t::getActiveDeviceAlloc(size_t offset) { assert(activeAllocationDevice); return ur_cast( deviceAllocations[activeAllocationDevice->Id.value()].get()) + offset; } -void *ur_discrete_mem_handle_t::getDevicePtr( +void *ur_discrete_buffer_handle_t::getDevicePtr( ur_device_handle_t hDevice, device_access_mode_t access, size_t offset, size_t size, std::function migrate) { - TRACK_SCOPE_LATENCY("ur_discrete_mem_handle_t::getDevicePtr"); + TRACK_SCOPE_LATENCY("ur_discrete_buffer_handle_t::getDevicePtr"); std::ignore = access; std::ignore = size; @@ -289,10 +304,10 @@ void *ur_discrete_mem_handle_t::getDevicePtr( return getActiveDeviceAlloc(offset); } -void *ur_discrete_mem_handle_t::mapHostPtr( +void *ur_discrete_buffer_handle_t::mapHostPtr( ur_map_flags_t flags, size_t offset, size_t size, std::function migrate) { - TRACK_SCOPE_LATENCY("ur_discrete_mem_handle_t::mapHostPtr"); + TRACK_SCOPE_LATENCY("ur_discrete_buffer_handle_t::mapHostPtr"); // TODO: use async alloc? void *ptr = mapToPtr; @@ -321,10 +336,10 @@ void *ur_discrete_mem_handle_t::mapHostPtr( return hostAllocations.back().ptr.get(); } -void ur_discrete_mem_handle_t::unmapHostPtr( +void ur_discrete_buffer_handle_t::unmapHostPtr( void *pMappedPtr, std::function migrate) { - TRACK_SCOPE_LATENCY("ur_discrete_mem_handle_t::unmapHostPtr"); + TRACK_SCOPE_LATENCY("ur_discrete_buffer_handle_t::unmapHostPtr"); auto hostAlloc = std::find_if(hostAllocations.begin(), hostAllocations.end(), @@ -365,22 +380,15 @@ static bool useHostBuffer(ur_context_handle_t hContext) { ZE_DEVICE_PROPERTY_FLAG_INTEGRATED; } -namespace ur::level_zero { -ur_result_t urMemRetain(ur_mem_handle_t hMem); -ur_result_t urMemRelease(ur_mem_handle_t hMem); -} // namespace ur::level_zero - -ur_mem_sub_buffer_t::ur_mem_sub_buffer_t(ur_mem_handle_t hParent, size_t offset, - size_t size, +ur_mem_sub_buffer_t::ur_mem_sub_buffer_t(ur_mem_buffer_t *hParent, + size_t offset, size_t size, device_access_mode_t accessMode) - : ur_mem_handle_t_(hParent->getContext(), size, accessMode), - hParent(hParent), offset(offset), size(size) { - ur::level_zero::urMemRetain(hParent); + : ur_mem_buffer_t(hParent->getContext(), size, accessMode), + hParent(hParent), offset(offset) { + retain(hParent); } -ur_mem_sub_buffer_t::~ur_mem_sub_buffer_t() { - ur::level_zero::urMemRelease(hParent); -} +ur_mem_sub_buffer_t::~ur_mem_sub_buffer_t() { release(hParent); } void *ur_mem_sub_buffer_t::getDevicePtr( ur_device_handle_t hDevice, device_access_mode_t access, size_t offset, @@ -401,10 +409,85 @@ void ur_mem_sub_buffer_t::unmapHostPtr( return hParent->unmapHostPtr(pMappedPtr, migrate); } -size_t ur_mem_sub_buffer_t::getSize() const { return size; } - ur_shared_mutex &ur_mem_sub_buffer_t::getMutex() { return hParent->getMutex(); } +ur_mem_image_t::ur_mem_image_t(ur_context_handle_t hContext, + ur_mem_flags_t flags, + const ur_image_format_t *pImageFormat, + const ur_image_desc_t *pImageDesc, void *pHost) + : hContext(hContext) { + UR_CALL_THROWS(ur2zeImageDesc(pImageFormat, pImageDesc, zeImageDesc)); + + // Currently we have the "0" device in context with mutliple root devices to + // own the image. + // TODO: Implement explicit copying for acessing the image from other devices + // in the context. + ur_device_handle_t hDevice = hContext->getDevices()[0]; + ZE2UR_CALL_THROWS(zeImageCreate, (hContext->getZeHandle(), hDevice->ZeDevice, + &zeImageDesc, zeImage.ptr())); + + if ((flags & UR_MEM_FLAG_USE_HOST_POINTER) != 0 || + (flags & UR_MEM_FLAG_ALLOC_COPY_HOST_POINTER) != 0) { + // Initialize image synchronously with immediate offload. + + auto commandList = getSyncCommandListForCopy(hContext, hDevice); + ZE2UR_CALL_THROWS(zeCommandListAppendImageCopyFromMemory, + (commandList.get(), zeImage.get(), pHost, nullptr, + nullptr, 0, nullptr)); + } +} + +ur_mem_image_t::ur_mem_image_t(ur_context_handle_t hContext, + const ur_image_format_t *pImageFormat, + const ur_image_desc_t *pImageDesc, + ze_image_handle_t zeImage, bool ownZeImage) + : hContext(hContext), zeImage(zeImage, ownZeImage) { + UR_CALL_THROWS(ur2zeImageDesc(pImageFormat, pImageDesc, zeImageDesc)); +} + +static void verifyImageRegion(ze_image_desc_t &zeImageDesc, + ze_image_region_t &zeRegion, size_t rowPitch, + size_t slicePitch) { +#ifndef NDEBUG + if (!(rowPitch == 0 || + // special case RGBA image pitch equal to region's width + (zeImageDesc.format.layout == ZE_IMAGE_FORMAT_LAYOUT_32_32_32_32 && + rowPitch == 4 * 4 * zeRegion.width) || + (zeImageDesc.format.layout == ZE_IMAGE_FORMAT_LAYOUT_16_16_16_16 && + rowPitch == 4 * 2 * zeRegion.width) || + (zeImageDesc.format.layout == ZE_IMAGE_FORMAT_LAYOUT_8_8_8_8 && + rowPitch == 4 * zeRegion.width))) + throw UR_RESULT_ERROR_INVALID_IMAGE_SIZE; +#endif + if (!(slicePitch == 0 || slicePitch == rowPitch * zeRegion.height)) + throw UR_RESULT_ERROR_INVALID_IMAGE_SIZE; +} + +std::pair +ur_mem_image_t::getRWRegion(ur_rect_offset_t &origin, ur_rect_region_t ®ion, + size_t rowPitch, size_t slicePitch) { + ze_image_region_t zeRegion; + UR_CALL_THROWS(getImageRegionHelper(zeImageDesc, &origin, ®ion, zeRegion)); + + verifyImageRegion(zeImageDesc, zeRegion, rowPitch, slicePitch); + + return {zeImage.get(), zeRegion}; +} + +ur_mem_image_t::copy_desc_t ur_mem_image_t::getCopyRegions( + ur_mem_image_t &src, ur_mem_image_t &dst, ur_rect_offset_t &srcOrigin, + ur_rect_offset_t &dstOrigin, ur_rect_region_t ®ion) { + ze_image_region_t zeSrcRegion; + UR_CALL_THROWS( + getImageRegionHelper(src.zeImageDesc, &srcOrigin, ®ion, zeSrcRegion)); + + ze_image_region_t zeDstRegion; + UR_CALL_THROWS( + getImageRegionHelper(dst.zeImageDesc, &dstOrigin, ®ion, zeDstRegion)); + + return {{src.zeImage.get(), zeSrcRegion}, {src.zeImage.get(), zeDstRegion}}; +} + namespace ur::level_zero { ur_result_t urMemBufferCreate(ur_context_handle_t hContext, ur_mem_flags_t flags, size_t size, @@ -425,13 +508,13 @@ ur_result_t urMemBufferCreate(ur_context_handle_t hContext, if (useHostBuffer(hContext)) { auto hostPtrAction = flags & UR_MEM_FLAG_USE_HOST_POINTER - ? ur_integrated_mem_handle_t::host_ptr_action_t::import - : ur_integrated_mem_handle_t::host_ptr_action_t::copy; - *phBuffer = new ur_integrated_mem_handle_t(hContext, hostPtr, size, - hostPtrAction, accessMode); + ? ur_integrated_buffer_handle_t::host_ptr_action_t::import + : ur_integrated_buffer_handle_t::host_ptr_action_t::copy; + *phBuffer = ur_mem_handle_t_::create( + hContext, hostPtr, size, hostPtrAction, accessMode); } else { - *phBuffer = - new ur_discrete_mem_handle_t(hContext, hostPtr, size, accessMode); + *phBuffer = ur_mem_handle_t_::create( + hContext, hostPtr, size, accessMode); } return UR_RESULT_SUCCESS; @@ -439,10 +522,12 @@ ur_result_t urMemBufferCreate(ur_context_handle_t hContext, return exceptionToResult(std::current_exception()); } -ur_result_t urMemBufferPartition(ur_mem_handle_t hBuffer, ur_mem_flags_t flags, +ur_result_t urMemBufferPartition(ur_mem_handle_t hMem, ur_mem_flags_t flags, ur_buffer_create_type_t bufferCreateType, const ur_buffer_region_t *pRegion, ur_mem_handle_t *phMem) try { + auto hBuffer = hMem->getBuffer(); + UR_ASSERT(bufferCreateType == UR_BUFFER_CREATE_TYPE_REGION, UR_RESULT_ERROR_INVALID_ENUMERATION); UR_ASSERT((pRegion->origin < hBuffer->getSize() && @@ -454,8 +539,8 @@ ur_result_t urMemBufferPartition(ur_mem_handle_t hBuffer, ur_mem_flags_t flags, UR_ASSERT(isAccessCompatible(accessMode, hBuffer->getDeviceAccessMode()), UR_RESULT_ERROR_INVALID_VALUE); - *phMem = new ur_mem_sub_buffer_t(hBuffer, pRegion->origin, pRegion->size, - accessMode); + *phMem = ur_mem_handle_t_::create( + hBuffer, pRegion->origin, pRegion->size, accessMode); return UR_RESULT_SUCCESS; } catch (...) { @@ -492,22 +577,22 @@ ur_result_t urMemBufferCreateWithNativeHandle( } // assume read-write - auto accessMode = ur_mem_handle_t_::device_access_mode_t::read_write; + auto accessMode = ur_mem_buffer_t::device_access_mode_t::read_write; if (useHostBuffer(hContext) && memoryAttrs.type == ZE_MEMORY_TYPE_HOST) { - *phMem = new ur_integrated_mem_handle_t(hContext, ptr, size, accessMode, - ownNativeHandle); + *phMem = ur_mem_handle_t_::create( + hContext, ptr, size, accessMode, ownNativeHandle); // if useHostBuffer(hContext) is true but the allocation is on device, we'll // treat it as discrete memory } else { if (memoryAttrs.type == ZE_MEMORY_TYPE_HOST) { // For host allocation, we need to copy the data to a device buffer // and then copy it back on release - *phMem = new ur_discrete_mem_handle_t(hContext, hDevice, nullptr, size, - accessMode, ptr, ownNativeHandle); + *phMem = ur_mem_handle_t_::create( + hContext, hDevice, nullptr, size, accessMode, ptr, ownNativeHandle); } else { // For device/shared allocation, we can use it directly - *phMem = new ur_discrete_mem_handle_t( + *phMem = ur_mem_handle_t_::create( hContext, hDevice, ptr, size, accessMode, nullptr, ownNativeHandle); } } @@ -517,7 +602,7 @@ ur_result_t urMemBufferCreateWithNativeHandle( return exceptionToResult(std::current_exception()); } -ur_result_t urMemGetInfo(ur_mem_handle_t hMemory, ur_mem_info_t propName, +ur_result_t urMemGetInfo(ur_mem_handle_t hMem, ur_mem_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet) try { // No locking needed here, we only read const members @@ -526,14 +611,23 @@ ur_result_t urMemGetInfo(ur_mem_handle_t hMemory, ur_mem_info_t propName, switch (propName) { case UR_MEM_INFO_CONTEXT: { - return returnValue(hMemory->getContext()); + if (hMem->isImage()) { + return returnValue(hMem->getImage()->getContext()); + } else { + return returnValue(hMem->getBuffer()->getContext()); + } } case UR_MEM_INFO_SIZE: { + if (hMem->isImage()) { + // TODO: implement size calculation + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + // Get size of the allocation - return returnValue(size_t{hMemory->getSize()}); + return returnValue(size_t{hMem->getBuffer()->getSize()}); } case UR_MEM_INFO_REFERENCE_COUNT: { - return returnValue(hMemory->getRefCount().load()); + return returnValue(hMem->getObject()->RefCount.load()); } default: { return UR_RESULT_ERROR_INVALID_ENUMERATION; @@ -546,17 +640,13 @@ ur_result_t urMemGetInfo(ur_mem_handle_t hMemory, ur_mem_info_t propName, } ur_result_t urMemRetain(ur_mem_handle_t hMem) try { - hMem->getRefCount().increment(); - return UR_RESULT_SUCCESS; + return retain(hMem->getObject()); } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urMemRelease(ur_mem_handle_t hMem) try { - if (hMem->getRefCount().decrementAndTest()) { - delete hMem; - } - return UR_RESULT_SUCCESS; + return release(hMem->getObject()); } catch (...) { return exceptionToResult(std::current_exception()); } @@ -564,14 +654,68 @@ ur_result_t urMemRelease(ur_mem_handle_t hMem) try { ur_result_t urMemGetNativeHandle(ur_mem_handle_t hMem, ur_device_handle_t hDevice, ur_native_handle_t *phNativeMem) try { - std::scoped_lock lock(hMem->getMutex()); + if (hMem->isImage()) { + auto hImage = hMem->getImage(); + *phNativeMem = reinterpret_cast(hImage->getZeImage()); + return UR_RESULT_SUCCESS; + } + + auto hBuffer = hMem->getBuffer(); - auto ptr = hMem->getDevicePtr( - hDevice, ur_mem_handle_t_::device_access_mode_t::read_write, 0, - hMem->getSize(), nullptr); + std::scoped_lock lock(hBuffer->getMutex()); + + auto ptr = hBuffer->getDevicePtr( + hDevice, ur_mem_buffer_t::device_access_mode_t::read_write, 0, + hBuffer->getSize(), nullptr); *phNativeMem = reinterpret_cast(ptr); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); } + +ur_result_t urMemImageCreate(ur_context_handle_t hContext, ur_mem_flags_t flags, + const ur_image_format_t *pImageFormat, + const ur_image_desc_t *pImageDesc, void *pHost, + ur_mem_handle_t *phMem) try { + // TODO: implement read-only, write-only + if ((flags & UR_MEM_FLAG_READ_WRITE) == 0) { + die("urMemImageCreate: Level-Zero implements only read-write buffer," + "no read-only or write-only yet."); + } + + *phMem = ur_mem_handle_t_::create( + hContext, flags, pImageFormat, pImageDesc, pHost); + return UR_RESULT_SUCCESS; +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +ur_result_t urMemImageCreateWithNativeHandle( + ur_native_handle_t hNativeMem, ur_context_handle_t hContext, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + const ur_mem_native_properties_t *pProperties, ur_mem_handle_t *phMem) try { + auto zeImage = reinterpret_cast(hNativeMem); + bool ownNativeHandle = pProperties ? pProperties->isNativeHandleOwned : false; + + *phMem = ur_mem_handle_t_::create( + hContext, pImageFormat, pImageDesc, zeImage, ownNativeHandle); + return UR_RESULT_SUCCESS; +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +ur_result_t urMemImageGetInfo(ur_mem_handle_t hMemory, ur_image_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet) { + logger::error("{} function not implemented!", __FUNCTION__); + + std::ignore = hMemory; + std::ignore = propName; + std::ignore = propSize; + std::ignore = pPropValue; + std::ignore = pPropSizeRet; + + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + } // namespace ur::level_zero diff --git a/source/adapters/level_zero/v2/memory.hpp b/source/adapters/level_zero/v2/memory.hpp index 81e8495239..d58b8b7062 100644 --- a/source/adapters/level_zero/v2/memory.hpp +++ b/source/adapters/level_zero/v2/memory.hpp @@ -10,6 +10,8 @@ #pragma once +#include + #include #include "../device.hpp" @@ -17,12 +19,14 @@ using usm_unique_ptr_t = std::unique_ptr>; -struct ur_mem_handle_t_ : private _ur_object { +struct ur_mem_buffer_t : public _ur_object { enum class device_access_mode_t { read_write, read_only, write_only }; - ur_mem_handle_t_(ur_context_handle_t hContext, size_t size, - device_access_mode_t accesMode); - virtual ~ur_mem_handle_t_() = default; + ur_mem_buffer_t(ur_context_handle_t hContext, size_t size, + device_access_mode_t accesMode); + virtual ~ur_mem_buffer_t() = default; + + virtual ur_shared_mutex &getMutex(); // Following functions should always be called under the lock. @@ -39,23 +43,19 @@ struct ur_mem_handle_t_ : private _ur_object { unmapHostPtr(void *pMappedPtr, std::function memcpy) = 0; - inline device_access_mode_t getDeviceAccessMode() const { return accessMode; } - inline ur_context_handle_t getContext() const { return hContext; } - inline ReferenceCounter &getRefCount() { return RefCount; } - - virtual size_t getSize() const; - virtual ur_shared_mutex &getMutex(); + device_access_mode_t getDeviceAccessMode() const { return accessMode; } + ur_context_handle_t getContext() const { return hContext; } + size_t getSize() const { return size; } protected: - const device_access_mode_t accessMode; const ur_context_handle_t hContext; const size_t size; + const device_access_mode_t accessMode; }; // non-owning buffer wrapper around USM pointer -struct ur_usm_handle_t_ : ur_mem_handle_t_ { - ur_usm_handle_t_(ur_context_handle_t hContext, size_t size, const void *ptr); - ~ur_usm_handle_t_(); +struct ur_usm_handle_t : ur_mem_buffer_t { + ur_usm_handle_t(ur_context_handle_t hContext, size_t size, const void *ptr); void * getDevicePtr(ur_device_handle_t, device_access_mode_t, size_t offset, @@ -73,16 +73,16 @@ struct ur_usm_handle_t_ : ur_mem_handle_t_ { // Manages memory buffer for integrated GPU. // For integrated devices the buffer has been allocated in host memory // and can be accessed by the device without copying. -struct ur_integrated_mem_handle_t : public ur_mem_handle_t_ { +struct ur_integrated_buffer_handle_t : public ur_mem_buffer_t { enum class host_ptr_action_t { import, copy }; - ur_integrated_mem_handle_t(ur_context_handle_t hContext, void *hostPtr, - size_t size, host_ptr_action_t useHostPtr, - device_access_mode_t accesMode); + ur_integrated_buffer_handle_t(ur_context_handle_t hContext, void *hostPtr, + size_t size, host_ptr_action_t useHostPtr, + device_access_mode_t accesMode); - ur_integrated_mem_handle_t(ur_context_handle_t hContext, void *hostPtr, - size_t size, device_access_mode_t accesMode, - bool ownHostPtr); + ur_integrated_buffer_handle_t(ur_context_handle_t hContext, void *hostPtr, + size_t size, device_access_mode_t accesMode, + bool ownHostPtr); void * getDevicePtr(ur_device_handle_t, device_access_mode_t, size_t offset, @@ -110,19 +110,19 @@ struct host_allocation_desc_t { // Manages memory buffer for discrete GPU. // Memory is allocated on the device and migrated/copies if necessary. -struct ur_discrete_mem_handle_t : public ur_mem_handle_t_ { +struct ur_discrete_buffer_handle_t : public ur_mem_buffer_t { // If hostPtr is not null, the buffer is allocated immediately on the // first device in the context. Otherwise, the buffer is allocated on // firt getDevicePtr call. - ur_discrete_mem_handle_t(ur_context_handle_t hContext, void *hostPtr, - size_t size, device_access_mode_t accesMode); - ~ur_discrete_mem_handle_t(); + ur_discrete_buffer_handle_t(ur_context_handle_t hContext, void *hostPtr, + size_t size, device_access_mode_t accesMode); + ~ur_discrete_buffer_handle_t(); // Create buffer on top of existing device memory. - ur_discrete_mem_handle_t(ur_context_handle_t hContext, - ur_device_handle_t hDevice, void *devicePtr, - size_t size, device_access_mode_t accesMode, - void *writeBackMemory, bool ownDevicePtr); + ur_discrete_buffer_handle_t(ur_context_handle_t hContext, + ur_device_handle_t hDevice, void *devicePtr, + size_t size, device_access_mode_t accesMode, + void *writeBackMemory, bool ownDevicePtr); void * getDevicePtr(ur_device_handle_t, device_access_mode_t, size_t offset, @@ -157,8 +157,8 @@ struct ur_discrete_mem_handle_t : public ur_mem_handle_t_ { size_t size); }; -struct ur_mem_sub_buffer_t : public ur_mem_handle_t_ { - ur_mem_sub_buffer_t(ur_mem_handle_t hParent, size_t offset, size_t size, +struct ur_mem_sub_buffer_t : public ur_mem_buffer_t { + ur_mem_sub_buffer_t(ur_mem_buffer_t *hParent, size_t offset, size_t size, device_access_mode_t accesMode); ~ur_mem_sub_buffer_t(); @@ -171,11 +171,96 @@ struct ur_mem_sub_buffer_t : public ur_mem_handle_t_ { void unmapHostPtr(void *pMappedPtr, std::function) override; - size_t getSize() const override; ur_shared_mutex &getMutex() override; private: - ur_mem_handle_t hParent; + ur_mem_buffer_t *hParent; size_t offset; - size_t size; +}; + +struct ur_mem_image_t : _ur_object { + ur_mem_image_t(ur_context_handle_t hContext, ur_mem_flags_t flags, + const ur_image_format_t *pImageFormat, + const ur_image_desc_t *pImageDesc, void *pHost); + ur_mem_image_t(ur_context_handle_t, const ur_image_format_t *pImageFormat, + const ur_image_desc_t *pImageDesc, ze_image_handle_t zeImage, + bool ownZeImage); + + ze_image_handle_t getZeImage() const { return zeImage.get(); } + + std::pair + getRWRegion(ur_rect_offset_t &origin, ur_rect_region_t ®ion, + size_t rowPitch, size_t slicePitch); + + struct copy_desc_t { + std::pair src; + std::pair dst; + }; + + static copy_desc_t getCopyRegions(ur_mem_image_t &src, ur_mem_image_t &dst, + ur_rect_offset_t &srcOrigin, + ur_rect_offset_t &dstOrigin, + ur_rect_region_t ®ion); + + ur_context_handle_t getContext() const { return hContext; } + +private: + const ur_context_handle_t hContext; + v2::raii::ze_image_handle_t zeImage; + ZeStruct zeImageDesc; +}; + +struct ur_mem_handle_t_ : private _ur_object { + template static T cast(U p) { + static bool constexpr is_convertible = std::is_convertible_v; + std::ignore = is_convertible; + // check at runtime, not compile time, to make it work in visit + assert(is_convertible); + return reinterpret_cast(p); + } + + template + static ur_mem_handle_t_ *create(Args &&...args) { + return new ur_mem_handle_t_(std::in_place_type, + std::forward(args)...); + } + + ur_mem_buffer_t *getBuffer() { + if (std::holds_alternative(mem)) { + throw UR_RESULT_ERROR_INVALID_MEM_OBJECT; + } + return std::visit( + [](auto &&arg) -> ur_mem_buffer_t * { + return cast(&arg); + }, + mem); + } + + ur_mem_image_t *getImage() { + if (!std::holds_alternative(mem)) { + throw UR_RESULT_ERROR_INVALID_MEM_OBJECT; + } + return std::visit( + [](auto &&arg) -> ur_mem_image_t * { + return cast(&arg); + }, + mem); + } + + _ur_object *getObject() { + return std::visit( + [](auto &&arg) -> _ur_object * { return cast<_ur_object *>(&arg); }, + mem); + } + + bool isImage() const { return std::holds_alternative(mem); } + +private: + template + ur_mem_handle_t_(std::in_place_type_t, Args &&...args) + : mem(std::in_place_type, std::forward(args)...) {} + + std::variant + mem; }; diff --git a/source/adapters/level_zero/v2/queue_immediate_in_order.cpp b/source/adapters/level_zero/v2/queue_immediate_in_order.cpp index 8da52fe6b6..f23b7154cd 100644 --- a/source/adapters/level_zero/v2/queue_immediate_in_order.cpp +++ b/source/adapters/level_zero/v2/queue_immediate_in_order.cpp @@ -292,17 +292,16 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueEventsWaitWithBarrierExt( } ur_result_t ur_queue_immediate_in_order_t::enqueueGenericCopyUnlocked( - ur_mem_handle_t src, ur_mem_handle_t dst, bool blocking, size_t srcOffset, + ur_mem_buffer_t *src, ur_mem_buffer_t *dst, bool blocking, size_t srcOffset, size_t dstOffset, size_t size, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent, ur_command_t commandType) { auto zeSignalEvent = getSignalEvent(phEvent, commandType); - auto waitList = getWaitListView(phEventWaitList, numEventsInWaitList); bool memoryMigrated = false; auto pSrc = ur_cast(src->getDevicePtr( - hDevice, ur_mem_handle_t_::device_access_mode_t::read_only, srcOffset, + hDevice, ur_mem_buffer_t::device_access_mode_t::read_only, srcOffset, size, [&](void *src, void *dst, size_t size) { ZE2UR_CALL_THROWS(zeCommandListAppendMemoryCopy, (commandListManager.getZeCommandList(), dst, src, @@ -311,7 +310,7 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueGenericCopyUnlocked( })); auto pDst = ur_cast(dst->getDevicePtr( - hDevice, ur_mem_handle_t_::device_access_mode_t::write_only, dstOffset, + hDevice, ur_mem_buffer_t::device_access_mode_t::write_only, dstOffset, size, [&](void *src, void *dst, size_t size) { ZE2UR_CALL_THROWS(zeCommandListAppendMemoryCopy, (commandListManager.getZeCommandList(), dst, src, @@ -339,14 +338,16 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueGenericCopyUnlocked( } ur_result_t ur_queue_immediate_in_order_t::enqueueMemBufferRead( - ur_mem_handle_t hBuffer, bool blockingRead, size_t offset, size_t size, + ur_mem_handle_t hMem, bool blockingRead, size_t offset, size_t size, void *pDst, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { TRACK_SCOPE_LATENCY("ur_queue_immediate_in_order_t::enqueueMemBufferRead"); + auto hBuffer = hMem->getBuffer(); + UR_ASSERT(offset + size <= hBuffer->getSize(), UR_RESULT_ERROR_INVALID_SIZE); - ur_usm_handle_t_ dstHandle(hContext, size, pDst); + ur_usm_handle_t dstHandle(hContext, size, pDst); std::scoped_lock lock(this->Mutex, hBuffer->getMutex()); @@ -357,14 +358,16 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueMemBufferRead( } ur_result_t ur_queue_immediate_in_order_t::enqueueMemBufferWrite( - ur_mem_handle_t hBuffer, bool blockingWrite, size_t offset, size_t size, + ur_mem_handle_t hMem, bool blockingWrite, size_t offset, size_t size, const void *pSrc, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { TRACK_SCOPE_LATENCY("ur_queue_immediate_in_order_t::enqueueMemBufferWrite"); + auto hBuffer = hMem->getBuffer(); + UR_ASSERT(offset + size <= hBuffer->getSize(), UR_RESULT_ERROR_INVALID_SIZE); - ur_usm_handle_t_ srcHandle(hContext, size, pSrc); + ur_usm_handle_t srcHandle(hContext, size, pSrc); std::scoped_lock lock(this->Mutex, hBuffer->getMutex()); @@ -375,7 +378,7 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueMemBufferWrite( } ur_result_t ur_queue_immediate_in_order_t::enqueueRegionCopyUnlocked( - ur_mem_handle_t src, ur_mem_handle_t dst, bool blocking, + ur_mem_buffer_t *src, ur_mem_buffer_t *dst, bool blocking, ur_rect_offset_t srcOrigin, ur_rect_offset_t dstOrigin, ur_rect_region_t region, size_t srcRowPitch, size_t srcSlicePitch, size_t dstRowPitch, size_t dstSlicePitch, uint32_t numEventsInWaitList, @@ -390,7 +393,7 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueRegionCopyUnlocked( bool memoryMigrated = false; auto pSrc = ur_cast(src->getDevicePtr( - hDevice, ur_mem_handle_t_::device_access_mode_t::read_only, 0, + hDevice, ur_mem_buffer_t::device_access_mode_t::read_only, 0, src->getSize(), [&](void *src, void *dst, size_t size) { ZE2UR_CALL_THROWS(zeCommandListAppendMemoryCopy, (commandListManager.getZeCommandList(), dst, src, @@ -398,7 +401,7 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueRegionCopyUnlocked( memoryMigrated = true; })); auto pDst = ur_cast(dst->getDevicePtr( - hDevice, ur_mem_handle_t_::device_access_mode_t::write_only, 0, + hDevice, ur_mem_buffer_t::device_access_mode_t::write_only, 0, dst->getSize(), [&](void *src, void *dst, size_t size) { ZE2UR_CALL_THROWS(zeCommandListAppendMemoryCopy, (commandListManager.getZeCommandList(), dst, src, @@ -428,7 +431,7 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueRegionCopyUnlocked( } ur_result_t ur_queue_immediate_in_order_t::enqueueMemBufferReadRect( - ur_mem_handle_t hBuffer, bool blockingRead, ur_rect_offset_t bufferOrigin, + ur_mem_handle_t hMem, bool blockingRead, ur_rect_offset_t bufferOrigin, ur_rect_offset_t hostOrigin, ur_rect_region_t region, size_t bufferRowPitch, size_t bufferSlicePitch, size_t hostRowPitch, size_t hostSlicePitch, void *pDst, uint32_t numEventsInWaitList, @@ -436,7 +439,9 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueMemBufferReadRect( TRACK_SCOPE_LATENCY( "ur_queue_immediate_in_order_t::enqueueMemBufferReadRect"); - ur_usm_handle_t_ dstHandle(hContext, 0, pDst); + auto hBuffer = hMem->getBuffer(); + + ur_usm_handle_t dstHandle(hContext, 0, pDst); std::scoped_lock lock(this->Mutex, hBuffer->getMutex()); @@ -449,7 +454,7 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueMemBufferReadRect( } ur_result_t ur_queue_immediate_in_order_t::enqueueMemBufferWriteRect( - ur_mem_handle_t hBuffer, bool blockingWrite, ur_rect_offset_t bufferOrigin, + ur_mem_handle_t hMem, bool blockingWrite, ur_rect_offset_t bufferOrigin, ur_rect_offset_t hostOrigin, ur_rect_region_t region, size_t bufferRowPitch, size_t bufferSlicePitch, size_t hostRowPitch, size_t hostSlicePitch, void *pSrc, uint32_t numEventsInWaitList, @@ -457,7 +462,9 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueMemBufferWriteRect( TRACK_SCOPE_LATENCY( "ur_queue_immediate_in_order_t::enqueueMemBufferWriteRect"); - ur_usm_handle_t_ srcHandle(hContext, 0, pSrc); + auto hBuffer = hMem->getBuffer(); + + ur_usm_handle_t srcHandle(hContext, 0, pSrc); std::scoped_lock lock(this->Mutex, hBuffer->getMutex()); @@ -470,11 +477,14 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueMemBufferWriteRect( } ur_result_t ur_queue_immediate_in_order_t::enqueueMemBufferCopy( - ur_mem_handle_t hBufferSrc, ur_mem_handle_t hBufferDst, size_t srcOffset, + ur_mem_handle_t hSrc, ur_mem_handle_t hDst, size_t srcOffset, size_t dstOffset, size_t size, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { TRACK_SCOPE_LATENCY("ur_queue_immediate_in_order_t::enqueueMemBufferCopy"); + auto hBufferSrc = hSrc->getBuffer(); + auto hBufferDst = hDst->getBuffer(); + UR_ASSERT(srcOffset + size <= hBufferSrc->getSize(), UR_RESULT_ERROR_INVALID_SIZE); UR_ASSERT(dstOffset + size <= hBufferDst->getSize(), @@ -490,14 +500,17 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueMemBufferCopy( } ur_result_t ur_queue_immediate_in_order_t::enqueueMemBufferCopyRect( - ur_mem_handle_t hBufferSrc, ur_mem_handle_t hBufferDst, - ur_rect_offset_t srcOrigin, ur_rect_offset_t dstOrigin, - ur_rect_region_t region, size_t srcRowPitch, size_t srcSlicePitch, - size_t dstRowPitch, size_t dstSlicePitch, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { + ur_mem_handle_t hSrc, ur_mem_handle_t hDst, ur_rect_offset_t srcOrigin, + ur_rect_offset_t dstOrigin, ur_rect_region_t region, size_t srcRowPitch, + size_t srcSlicePitch, size_t dstRowPitch, size_t dstSlicePitch, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent) { TRACK_SCOPE_LATENCY( "ur_queue_immediate_in_order_t::enqueueMemBufferCopyRect"); + auto hBufferSrc = hSrc->getBuffer(); + auto hBufferDst = hDst->getBuffer(); + std::scoped_lock lock( this->Mutex, hBufferSrc->getMutex(), hBufferDst->getMutex()); @@ -508,11 +521,13 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueMemBufferCopyRect( } ur_result_t ur_queue_immediate_in_order_t::enqueueMemBufferFill( - ur_mem_handle_t hBuffer, const void *pPattern, size_t patternSize, + ur_mem_handle_t hMem, const void *pPattern, size_t patternSize, size_t offset, size_t size, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { TRACK_SCOPE_LATENCY("ur_queue_immediate_in_order_t::enqueueMemBufferFill"); + auto hBuffer = hMem->getBuffer(); + UR_ASSERT(offset + size <= hBuffer->getSize(), UR_RESULT_ERROR_INVALID_SIZE); std::scoped_lock lock(this->Mutex, @@ -524,64 +539,101 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueMemBufferFill( } ur_result_t ur_queue_immediate_in_order_t::enqueueMemImageRead( - ur_mem_handle_t hImage, bool blockingRead, ur_rect_offset_t origin, + ur_mem_handle_t hMem, bool blockingRead, ur_rect_offset_t origin, ur_rect_region_t region, size_t rowPitch, size_t slicePitch, void *pDst, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { - std::ignore = hImage; - std::ignore = blockingRead; - std::ignore = origin; - std::ignore = region; - std::ignore = rowPitch; - std::ignore = slicePitch; - std::ignore = pDst; - std::ignore = numEventsInWaitList; - std::ignore = phEventWaitList; - std::ignore = phEvent; - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + TRACK_SCOPE_LATENCY("ur_queue_immediate_in_order_t::enqueueMemImageRead"); + + auto hImage = hMem->getImage(); + + std::scoped_lock lock(this->Mutex); + + auto zeSignalEvent = getSignalEvent(phEvent, UR_COMMAND_MEM_IMAGE_READ); + auto waitList = getWaitListView(phEventWaitList, numEventsInWaitList); + + auto [zeImage, zeRegion] = + hImage->getRWRegion(origin, region, rowPitch, slicePitch); + + ZE2UR_CALL(zeCommandListAppendImageCopyToMemory, + (commandListManager.getZeCommandList(), pDst, zeImage, &zeRegion, + zeSignalEvent, waitList.second, waitList.first)); + + if (blockingRead) { + ZE2UR_CALL(zeCommandListHostSynchronize, + (commandListManager.getZeCommandList(), UINT64_MAX)); + } + + return UR_RESULT_SUCCESS; } ur_result_t ur_queue_immediate_in_order_t::enqueueMemImageWrite( - ur_mem_handle_t hImage, bool blockingWrite, ur_rect_offset_t origin, + ur_mem_handle_t hMem, bool blockingWrite, ur_rect_offset_t origin, ur_rect_region_t region, size_t rowPitch, size_t slicePitch, void *pSrc, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { - std::ignore = hImage; - std::ignore = blockingWrite; - std::ignore = origin; - std::ignore = region; - std::ignore = rowPitch; - std::ignore = slicePitch; - std::ignore = pSrc; - std::ignore = numEventsInWaitList; - std::ignore = phEventWaitList; - std::ignore = phEvent; - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + TRACK_SCOPE_LATENCY("ur_queue_immediate_in_order_t::enqueueMemImageWrite"); + + auto hImage = hMem->getImage(); + + std::scoped_lock lock(this->Mutex); + + auto zeSignalEvent = getSignalEvent(phEvent, UR_COMMAND_MEM_IMAGE_WRITE); + auto waitList = getWaitListView(phEventWaitList, numEventsInWaitList); + + auto [zeImage, zeRegion] = + hImage->getRWRegion(origin, region, rowPitch, slicePitch); + + ZE2UR_CALL(zeCommandListAppendImageCopyFromMemory, + (commandListManager.getZeCommandList(), zeImage, pSrc, &zeRegion, + zeSignalEvent, waitList.second, waitList.first)); + + if (blockingWrite) { + ZE2UR_CALL(zeCommandListHostSynchronize, + (commandListManager.getZeCommandList(), UINT64_MAX)); + } + + return UR_RESULT_SUCCESS; } ur_result_t ur_queue_immediate_in_order_t::enqueueMemImageCopy( - ur_mem_handle_t hImageSrc, ur_mem_handle_t hImageDst, - ur_rect_offset_t srcOrigin, ur_rect_offset_t dstOrigin, - ur_rect_region_t region, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { - std::ignore = hImageSrc; - std::ignore = hImageDst; - std::ignore = srcOrigin; - std::ignore = dstOrigin; - std::ignore = region; - std::ignore = numEventsInWaitList; - std::ignore = phEventWaitList; - std::ignore = phEvent; - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + ur_mem_handle_t hSrc, ur_mem_handle_t hDst, ur_rect_offset_t srcOrigin, + ur_rect_offset_t dstOrigin, ur_rect_region_t region, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent) { + TRACK_SCOPE_LATENCY("ur_queue_immediate_in_order_t::enqueueMemImageWrite"); + + auto hImageSrc = hSrc->getImage(); + auto hImageDst = hDst->getImage(); + + std::scoped_lock lock(this->Mutex); + + auto zeSignalEvent = getSignalEvent(phEvent, UR_COMMAND_MEM_IMAGE_COPY); + auto waitList = getWaitListView(phEventWaitList, numEventsInWaitList); + + auto desc = ur_mem_image_t::getCopyRegions(*hImageSrc, *hImageDst, srcOrigin, + dstOrigin, region); + + auto [zeImageSrc, zeRegionSrc] = desc.src; + auto [zeImageDst, zeRegionDst] = desc.dst; + + ZE2UR_CALL(zeCommandListAppendImageCopyRegion, + (commandListManager.getZeCommandList(), zeImageDst, zeImageSrc, + &zeRegionDst, &zeRegionSrc, zeSignalEvent, waitList.second, + waitList.first)); + + return UR_RESULT_SUCCESS; } ur_result_t ur_queue_immediate_in_order_t::enqueueMemBufferMap( - ur_mem_handle_t hBuffer, bool blockingMap, ur_map_flags_t mapFlags, + ur_mem_handle_t hMem, bool blockingMap, ur_map_flags_t mapFlags, size_t offset, size_t size, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent, void **ppRetMap) { TRACK_SCOPE_LATENCY("ur_queue_immediate_in_order_t::enqueueMemBufferMap"); + auto hBuffer = hMem->getBuffer(); + std::scoped_lock lock(this->Mutex, hBuffer->getMutex()); @@ -624,6 +676,8 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueMemUnmap( const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { TRACK_SCOPE_LATENCY("ur_queue_immediate_in_order_t::enqueueMemUnmap"); + auto hBuffer = hMem->getBuffer(); + std::scoped_lock lock(this->Mutex); auto zeSignalEvent = getSignalEvent(phEvent, UR_COMMAND_MEM_UNMAP); @@ -638,7 +692,7 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueMemUnmap( (commandListManager.getZeCommandList(), waitList.second, waitList.first)); bool memoryMigrated = false; - hMem->unmapHostPtr(pMappedPtr, [&](void *src, void *dst, size_t size) { + hBuffer->unmapHostPtr(pMappedPtr, [&](void *src, void *dst, size_t size) { ZE2UR_CALL_THROWS(zeCommandListAppendMemoryCopy, (commandListManager.getZeCommandList(), dst, src, size, nullptr, waitList.second, waitList.first)); @@ -652,18 +706,17 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueMemUnmap( } ur_result_t ur_queue_immediate_in_order_t::enqueueGenericFillUnlocked( - ur_mem_handle_t dst, size_t offset, size_t patternSize, + ur_mem_buffer_t *hBuffer, size_t offset, size_t patternSize, const void *pPattern, size_t size, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent, ur_command_t commandType) { - auto zeSignalEvent = getSignalEvent(phEvent, commandType); auto waitList = getWaitListView(phEventWaitList, numEventsInWaitList); bool memoryMigrated = false; - auto pDst = ur_cast(dst->getDevicePtr( - hDevice, ur_mem_handle_t_::device_access_mode_t::read_only, offset, size, + auto pDst = ur_cast(hBuffer->getDevicePtr( + hDevice, ur_mem_buffer_t::device_access_mode_t::read_only, offset, size, [&](void *src, void *dst, size_t size) { ZE2UR_CALL_THROWS(zeCommandListAppendMemoryCopy, (commandListManager.getZeCommandList(), dst, src, @@ -698,7 +751,7 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueUSMFill( std::scoped_lock lock(this->Mutex); - ur_usm_handle_t_ dstHandle(hContext, size, pMem); + ur_usm_handle_t dstHandle(hContext, size, pMem); return enqueueGenericFillUnlocked(&dstHandle, 0, patternSize, pPattern, size, numEventsInWaitList, phEventWaitList, phEvent, UR_COMMAND_USM_FILL); @@ -822,8 +875,8 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueUSMMemcpy2D( std::scoped_lock lock(this->Mutex); - ur_usm_handle_t_ srcHandle(hContext, 0, pSrc); - ur_usm_handle_t_ dstHandle(hContext, 0, pDst); + ur_usm_handle_t srcHandle(hContext, 0, pSrc); + ur_usm_handle_t dstHandle(hContext, 0, pDst); return enqueueRegionCopyUnlocked(&srcHandle, &dstHandle, blocking, zeroOffset, zeroOffset, region, srcPitch, 0, dstPitch, 0, diff --git a/source/adapters/level_zero/v2/queue_immediate_in_order.hpp b/source/adapters/level_zero/v2/queue_immediate_in_order.hpp index 16ec86e22f..a0c7c3dc97 100644 --- a/source/adapters/level_zero/v2/queue_immediate_in_order.hpp +++ b/source/adapters/level_zero/v2/queue_immediate_in_order.hpp @@ -15,6 +15,7 @@ #include "context.hpp" #include "event.hpp" #include "event_pool_cache.hpp" +#include "memory.hpp" #include "queue_api.hpp" #include "ur/ur.hpp" @@ -45,7 +46,7 @@ struct ur_queue_immediate_in_order_t : _ur_object, public ur_queue_t_ { void deferEventFree(ur_event_handle_t hEvent) override; ur_result_t enqueueRegionCopyUnlocked( - ur_mem_handle_t src, ur_mem_handle_t dst, bool blocking, + ur_mem_buffer_t *src, ur_mem_buffer_t *dst, bool blocking, ur_rect_offset_t srcOrigin, ur_rect_offset_t dstOrigin, ur_rect_region_t region, size_t srcRowPitch, size_t srcSlicePitch, size_t dstRowPitch, size_t dstSlicePitch, uint32_t numEventsInWaitList, @@ -53,13 +54,13 @@ struct ur_queue_immediate_in_order_t : _ur_object, public ur_queue_t_ { ur_command_t commandType); ur_result_t enqueueGenericCopyUnlocked( - ur_mem_handle_t src, ur_mem_handle_t dst, bool blocking, size_t srcOffset, - size_t dstOffset, size_t size, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent, - ur_command_t commandType); + ur_mem_buffer_t *src, ur_mem_buffer_t *dst, bool blocking, + size_t srcOffset, size_t dstOffset, size_t size, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent, ur_command_t commandType); ur_result_t enqueueGenericFillUnlocked( - ur_mem_handle_t hBuffer, size_t offset, size_t patternSize, + ur_mem_buffer_t *hBuffer, size_t offset, size_t patternSize, const void *pPattern, size_t size, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent, ur_command_t commandType); diff --git a/test/conformance/device/urDeviceGetInfo.cpp b/test/conformance/device/urDeviceGetInfo.cpp index 6f53f95907..3c834eb618 100644 --- a/test/conformance/device/urDeviceGetInfo.cpp +++ b/test/conformance/device/urDeviceGetInfo.cpp @@ -302,7 +302,7 @@ TEST_P(urDeviceGetInfoTest, Success) { ur_device_info_t info_type = getParam(); if (info_type == UR_DEVICE_INFO_GLOBAL_MEM_FREE) { - UUR_KNOWN_FAILURE_ON(uur::LevelZeroV2{}, uur::NativeCPU{}); + UUR_KNOWN_FAILURE_ON(uur::NativeCPU{}); } if (info_type == UR_DEVICE_INFO_MAX_READ_WRITE_IMAGE_ARGS) { diff --git a/test/conformance/enqueue/urEnqueueKernelLaunch.cpp b/test/conformance/enqueue/urEnqueueKernelLaunch.cpp index 20f78c2d8c..1a34c302ef 100644 --- a/test/conformance/enqueue/urEnqueueKernelLaunch.cpp +++ b/test/conformance/enqueue/urEnqueueKernelLaunch.cpp @@ -441,8 +441,6 @@ struct urEnqueueKernelLaunchWithVirtualMemory : uur::urKernelExecutionTest { UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urEnqueueKernelLaunchWithVirtualMemory); TEST_P(urEnqueueKernelLaunchWithVirtualMemory, Success) { - UUR_KNOWN_FAILURE_ON(uur::LevelZeroV2{}); - size_t work_dim = 1; size_t global_offset = 0; size_t global_size = alloc_size / sizeof(uint32_t); diff --git a/test/conformance/enqueue/urEnqueueMemImageCopy.cpp b/test/conformance/enqueue/urEnqueueMemImageCopy.cpp index 1ec2adb9df..88cb11b19d 100644 --- a/test/conformance/enqueue/urEnqueueMemImageCopy.cpp +++ b/test/conformance/enqueue/urEnqueueMemImageCopy.cpp @@ -13,7 +13,7 @@ struct urEnqueueMemImageCopyTest uint32_t data[4]; }; void SetUp() override { - UUR_KNOWN_FAILURE_ON(uur::LevelZeroV2{}, uur::NativeCPU{}); + UUR_KNOWN_FAILURE_ON(uur::NativeCPU{}); UUR_RETURN_ON_FATAL_FAILURE(urQueueTestWithParam::SetUp()); @@ -129,7 +129,7 @@ UUR_DEVICE_TEST_SUITE_P(urEnqueueMemImageCopyTest, printImageCopyTestString); TEST_P(urEnqueueMemImageCopyTest, Success) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); ASSERT_SUCCESS(urEnqueueMemImageCopy(queue, srcImage, dstImage, {0, 0, 0}, {0, 0, 0}, size, 0, nullptr, nullptr)); @@ -140,7 +140,7 @@ TEST_P(urEnqueueMemImageCopyTest, Success) { } TEST_P(urEnqueueMemImageCopyTest, SuccessPartialCopy) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); ASSERT_SUCCESS(urEnqueueMemImageCopy(queue, srcImage, dstImage, {0, 0, 0}, {0, 0, 0}, partialRegion, 0, nullptr, @@ -164,7 +164,7 @@ TEST_P(urEnqueueMemImageCopyTest, SuccessPartialCopy) { } TEST_P(urEnqueueMemImageCopyTest, SuccessPartialCopyWithSrcOffset) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); ASSERT_SUCCESS(urEnqueueMemImageCopy(queue, srcImage, dstImage, partialRegionOffset, {0, 0, 0}, @@ -188,7 +188,7 @@ TEST_P(urEnqueueMemImageCopyTest, SuccessPartialCopyWithSrcOffset) { } TEST_P(urEnqueueMemImageCopyTest, SuccessPartialCopyWithDstOffset) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); ASSERT_SUCCESS(urEnqueueMemImageCopy(queue, srcImage, dstImage, {0, 0, 0}, partialRegionOffset, partialRegion, 0, @@ -254,7 +254,7 @@ TEST_P(urEnqueueMemImageCopyTest, InvalidNullPtrEventWaitList) { } TEST_P(urEnqueueMemImageCopyTest, InvalidSize) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); ASSERT_EQ_RESULT(UR_RESULT_ERROR_INVALID_SIZE, urEnqueueMemImageCopy(queue, srcImage, dstImage, {1, 0, 0}, diff --git a/test/conformance/enqueue/urEnqueueMemImageRead.cpp b/test/conformance/enqueue/urEnqueueMemImageRead.cpp index 3b37199409..4dc9c93e18 100644 --- a/test/conformance/enqueue/urEnqueueMemImageRead.cpp +++ b/test/conformance/enqueue/urEnqueueMemImageRead.cpp @@ -8,7 +8,6 @@ struct urEnqueueMemImageReadTest : uur::urMemImageQueueTest { void SetUp() override { - UUR_KNOWN_FAILURE_ON(uur::LevelZeroV2{}); UUR_RETURN_ON_FATAL_FAILURE(uur::urMemImageQueueTest::SetUp()); } }; @@ -86,7 +85,7 @@ TEST_P(urEnqueueMemImageReadTest, InvalidNullPtrEventWaitList) { } TEST_P(urEnqueueMemImageReadTest, InvalidOrigin1D) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); std::vector output(width * 4, 42); ur_rect_offset_t bad_origin{1, 0, 0}; @@ -97,7 +96,7 @@ TEST_P(urEnqueueMemImageReadTest, InvalidOrigin1D) { } TEST_P(urEnqueueMemImageReadTest, InvalidOrigin2D) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); std::vector output(width * height * 4, 42); ur_rect_offset_t bad_origin{0, 1, 0}; @@ -108,7 +107,7 @@ TEST_P(urEnqueueMemImageReadTest, InvalidOrigin2D) { } TEST_P(urEnqueueMemImageReadTest, InvalidOrigin3D) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); std::vector output(width * height * depth * 4, 42); ur_rect_offset_t bad_origin{0, 0, 1}; @@ -119,7 +118,7 @@ TEST_P(urEnqueueMemImageReadTest, InvalidOrigin3D) { } TEST_P(urEnqueueMemImageReadTest, InvalidRegion1D) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); std::vector output(width * 4, 42); ur_rect_region_t bad_region{width + 1, 1, 1}; @@ -130,7 +129,7 @@ TEST_P(urEnqueueMemImageReadTest, InvalidRegion1D) { } TEST_P(urEnqueueMemImageReadTest, InvalidRegion2D) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); std::vector output(width * height * 4, 42); ur_rect_region_t bad_region{width, height + 1, 1}; @@ -141,7 +140,7 @@ TEST_P(urEnqueueMemImageReadTest, InvalidRegion2D) { } TEST_P(urEnqueueMemImageReadTest, InvalidRegion3D) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); std::vector output(width * height * depth * 4, 42); ur_rect_region_t bad_region{width, height, depth + 1}; diff --git a/test/conformance/enqueue/urEnqueueMemImageWrite.cpp b/test/conformance/enqueue/urEnqueueMemImageWrite.cpp index f7329fd725..8b1b475327 100644 --- a/test/conformance/enqueue/urEnqueueMemImageWrite.cpp +++ b/test/conformance/enqueue/urEnqueueMemImageWrite.cpp @@ -8,7 +8,6 @@ struct urEnqueueMemImageWriteTest : uur::urMemImageQueueTest { void SetUp() override { - UUR_KNOWN_FAILURE_ON(uur::LevelZeroV2{}); UUR_RETURN_ON_FATAL_FAILURE(uur::urMemImageQueueTest::SetUp()); } }; @@ -84,7 +83,7 @@ TEST_P(urEnqueueMemImageWriteTest, InvalidNullPtrEventWaitList) { } TEST_P(urEnqueueMemImageWriteTest, InvalidOrigin1D) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); std::vector input(width * 4, 42); ur_rect_offset_t bad_origin{1, 0, 0}; @@ -95,7 +94,7 @@ TEST_P(urEnqueueMemImageWriteTest, InvalidOrigin1D) { } TEST_P(urEnqueueMemImageWriteTest, InvalidOrigin2D) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); std::vector input(width * height * 4, 42); ur_rect_offset_t bad_origin{0, 1, 0}; @@ -106,7 +105,7 @@ TEST_P(urEnqueueMemImageWriteTest, InvalidOrigin2D) { } TEST_P(urEnqueueMemImageWriteTest, InvalidOrigin3D) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); std::vector input(width * height * depth * 4, 42); ur_rect_offset_t bad_origin{0, 0, 1}; @@ -117,7 +116,7 @@ TEST_P(urEnqueueMemImageWriteTest, InvalidOrigin3D) { } TEST_P(urEnqueueMemImageWriteTest, InvalidRegion1D) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); std::vector input(width * 4, 42); ur_rect_region_t bad_region{width + 1, 1, 1}; @@ -128,7 +127,7 @@ TEST_P(urEnqueueMemImageWriteTest, InvalidRegion1D) { } TEST_P(urEnqueueMemImageWriteTest, InvalidRegion2D) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); std::vector input(width * height * 4, 42); ur_rect_region_t bad_region{width, height + 1, 1}; @@ -139,7 +138,7 @@ TEST_P(urEnqueueMemImageWriteTest, InvalidRegion2D) { } TEST_P(urEnqueueMemImageWriteTest, InvalidRegion3D) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); std::vector input(width * height * depth * 4, 42); ur_rect_region_t bad_region{width, height, depth + 1}; diff --git a/test/conformance/memory/urMemBufferCreateWithNativeHandle.cpp b/test/conformance/memory/urMemBufferCreateWithNativeHandle.cpp index c0bd46dc62..86c7c5f19f 100644 --- a/test/conformance/memory/urMemBufferCreateWithNativeHandle.cpp +++ b/test/conformance/memory/urMemBufferCreateWithNativeHandle.cpp @@ -116,7 +116,7 @@ using urMemBufferMultiQueueMemBufferTest = uur::urMultiDeviceMemBufferQueueTest; UUR_INSTANTIATE_PLATFORM_TEST_SUITE_P(urMemBufferMultiQueueMemBufferTest); TEST_P(urMemBufferMultiQueueMemBufferTest, WriteBack) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); void *ptr; ASSERT_SUCCESS(urUSMHostAlloc(context, nullptr, nullptr, size, &ptr)); diff --git a/test/conformance/memory/urMemBufferPartition.cpp b/test/conformance/memory/urMemBufferPartition.cpp index d48fc45736..4071d57cc7 100644 --- a/test/conformance/memory/urMemBufferPartition.cpp +++ b/test/conformance/memory/urMemBufferPartition.cpp @@ -17,14 +17,14 @@ UUR_DEVICE_TEST_SUITE_P(urMemBufferPartitionWithFlagsTest, uur::deviceTestWithParamPrinter); TEST_P(urMemBufferPartitionWithFlagsTest, Success) { - UUR_KNOWN_FAILURE_ON(uur::LevelZeroV2{}); - if (getParam() == UR_MEM_FLAG_WRITE_ONLY) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::NativeCPU{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}, + uur::NativeCPU{}); } if (getParam() == UR_MEM_FLAG_READ_ONLY) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::NativeCPU{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}, + uur::NativeCPU{}); } uur::raii::Mem buffer = nullptr; diff --git a/test/conformance/memory/urMemGetInfo.cpp b/test/conformance/memory/urMemGetInfo.cpp index d307a34fc0..4ae1a9d211 100644 --- a/test/conformance/memory/urMemGetInfo.cpp +++ b/test/conformance/memory/urMemGetInfo.cpp @@ -108,15 +108,13 @@ TEST_P(urMemGetInfoTest, InvalidNullPointerPropSizeRet) { } struct urMemGetInfoImageTest : uur::urMemImageTest { - void SetUp() override { - UUR_KNOWN_FAILURE_ON(uur::LevelZeroV2{}); - uur::urMemImageTest::SetUp(); - } + void SetUp() override { uur::urMemImageTest::SetUp(); } }; UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urMemGetInfoImageTest); TEST_P(urMemGetInfoImageTest, SuccessSize) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::OpenCL{"UHD Graphics"}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}, + uur::OpenCL{"UHD Graphics"}); ur_mem_info_t property_name = UR_MEM_INFO_SIZE; size_t property_size = 0; diff --git a/test/conformance/memory/urMemImageCreate.cpp b/test/conformance/memory/urMemImageCreate.cpp index 94be2f3f5a..aa36ebd89f 100644 --- a/test/conformance/memory/urMemImageCreate.cpp +++ b/test/conformance/memory/urMemImageCreate.cpp @@ -203,7 +203,7 @@ TEST_P(urMemImageCreateTest, InvalidSize) { uur::OpenCL{"Intel(R) UHD Graphics 770"}); // This fail is specific to the "Multi device testing" ci job. - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); uur::raii::Mem image_handle = nullptr; @@ -234,7 +234,7 @@ TEST_P(urMemImageCreateTest, InvalidSize) { TEST_P(urMemImageCreateTest, InvalidImageDescStype) { // This fail is specific to the "Multi device testing" ci job. - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); uur::raii::Mem image_handle = nullptr; ur_image_desc_t invalid_image_desc = image_desc; @@ -335,7 +335,7 @@ TEST_P(urMemImageCreateWithHostPtrFlagsTest, Success) { UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) FPGA"}); // This fail is specific to the "Multi device testing" ci job. - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); uur::raii::Mem host_ptr_buffer = nullptr; ASSERT_SUCCESS(urMemImageCreate(context, UR_MEM_FLAG_ALLOC_HOST_POINTER, diff --git a/test/conformance/memory/urMemImageCreateWithImageFormatParam.cpp b/test/conformance/memory/urMemImageCreateWithImageFormatParam.cpp index 1764e59793..d3e4cca4c7 100644 --- a/test/conformance/memory/urMemImageCreateWithImageFormatParam.cpp +++ b/test/conformance/memory/urMemImageCreateWithImageFormatParam.cpp @@ -93,14 +93,14 @@ UUR_DEVICE_TEST_SUITE_P( uur::deviceTestWithParamPrinter); TEST_P(urMemImageCreateTestWithImageFormatParam, Success) { - UUR_KNOWN_FAILURE_ON(uur::LevelZeroV2{}, uur::NativeCPU{}, + UUR_KNOWN_FAILURE_ON(uur::NativeCPU{}, uur::OpenCL{"Intel(R) UHD Graphics 770"}); ur_image_channel_order_t channel_order = std::get<1>(GetParam()).channelOrder; ur_image_channel_type_t channel_type = std::get<1>(GetParam()).channelType; if (channel_order == UR_IMAGE_CHANNEL_ORDER_RGBA) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); } ur_image_format_t image_format{channel_order, channel_type}; diff --git a/test/conformance/memory/urMemImageCreateWithNativeHandle.cpp b/test/conformance/memory/urMemImageCreateWithNativeHandle.cpp index 03452ae83c..702610d589 100644 --- a/test/conformance/memory/urMemImageCreateWithNativeHandle.cpp +++ b/test/conformance/memory/urMemImageCreateWithNativeHandle.cpp @@ -8,10 +8,7 @@ #include struct urMemImageCreateWithNativeHandleTest : uur::urMemImageTest { - void SetUp() { - UUR_KNOWN_FAILURE_ON(uur::LevelZeroV2{}); - UUR_RETURN_ON_FATAL_FAILURE(urMemImageTest::SetUp()); - } + void SetUp() { UUR_RETURN_ON_FATAL_FAILURE(urMemImageTest::SetUp()); } }; UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urMemImageCreateWithNativeHandleTest); diff --git a/test/conformance/memory/urMemImageGetInfo.cpp b/test/conformance/memory/urMemImageGetInfo.cpp index ec56b73ea4..30f70f6ce3 100644 --- a/test/conformance/memory/urMemImageGetInfo.cpp +++ b/test/conformance/memory/urMemImageGetInfo.cpp @@ -16,7 +16,7 @@ bool operator==(ur_image_format_t lhs, ur_image_format_t rhs) { } TEST_P(urMemImageGetInfoTest, SuccessFormat) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); size_t property_size = 0; const ur_image_info_t property_name = UR_IMAGE_INFO_FORMAT; @@ -221,7 +221,7 @@ TEST_P(urMemImageGetInfoTest, InvalidSizeZero) { TEST_P(urMemImageGetInfoTest, InvalidSizeSmall) { // This fail is specific to the "Multi device testing" ci job. - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); int property_size = 0; ASSERT_EQ_RESULT(urMemImageGetInfo(image, UR_IMAGE_INFO_FORMAT, diff --git a/test/conformance/testing/include/uur/fixtures.h b/test/conformance/testing/include/uur/fixtures.h index 4f6aa816ea..95a75c27ea 100644 --- a/test/conformance/testing/include/uur/fixtures.h +++ b/test/conformance/testing/include/uur/fixtures.h @@ -936,8 +936,6 @@ template struct urUSMPoolTestWithParam : urContextTestWithParam { struct urVirtualMemGranularityTest : urContextTest { void SetUp() override { - UUR_KNOWN_FAILURE_ON(uur::LevelZeroV2{}); - UUR_RETURN_ON_FATAL_FAILURE(urContextTest::SetUp()); ur_bool_t virtual_memory_support = false; @@ -958,8 +956,6 @@ struct urVirtualMemGranularityTest : urContextTest { template struct urVirtualMemGranularityTestWithParam : urContextTestWithParam { void SetUp() override { - UUR_KNOWN_FAILURE_ON(uur::LevelZeroV2{}); - UUR_RETURN_ON_FATAL_FAILURE(urContextTestWithParam::SetUp()); ur_bool_t virtual_memory_support = false; diff --git a/test/conformance/virtual_memory/urPhysicalMemCreate.cpp b/test/conformance/virtual_memory/urPhysicalMemCreate.cpp index 644f42b5d2..9e54d63ca4 100644 --- a/test/conformance/virtual_memory/urPhysicalMemCreate.cpp +++ b/test/conformance/virtual_memory/urPhysicalMemCreate.cpp @@ -33,7 +33,7 @@ UUR_DEVICE_TEST_SUITE_P(urPhysicalMemCreateWithSizeParamTest, uur::deviceTestWithParamPrinter); TEST_P(urPhysicalMemCreateWithSizeParamTest, Success) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); ASSERT_SUCCESS( urPhysicalMemCreate(context, device, size, nullptr, &physical_mem)); @@ -41,7 +41,7 @@ TEST_P(urPhysicalMemCreateWithSizeParamTest, Success) { } TEST_P(urPhysicalMemCreateWithSizeParamTest, InvalidSize) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); if (granularity == 1) { GTEST_SKIP() << "A granularity of 1 means that any size will be accepted."; diff --git a/test/conformance/virtual_memory/urPhysicalMemGetInfo.cpp b/test/conformance/virtual_memory/urPhysicalMemGetInfo.cpp index ce22006c18..a2c14caacc 100644 --- a/test/conformance/virtual_memory/urPhysicalMemGetInfo.cpp +++ b/test/conformance/virtual_memory/urPhysicalMemGetInfo.cpp @@ -10,7 +10,7 @@ using urPhysicalMemGetInfoTest = uur::urPhysicalMemTest; UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urPhysicalMemGetInfoTest); TEST_P(urPhysicalMemGetInfoTest, SuccessContext) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); ur_physical_mem_info_t property_name = UR_PHYSICAL_MEM_INFO_CONTEXT; size_t property_size = 0; @@ -29,7 +29,7 @@ TEST_P(urPhysicalMemGetInfoTest, SuccessContext) { } TEST_P(urPhysicalMemGetInfoTest, SuccessDevice) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); ur_physical_mem_info_t property_name = UR_PHYSICAL_MEM_INFO_DEVICE; size_t property_size = 0; @@ -48,7 +48,7 @@ TEST_P(urPhysicalMemGetInfoTest, SuccessDevice) { } TEST_P(urPhysicalMemGetInfoTest, SuccessSize) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); ur_physical_mem_info_t property_name = UR_PHYSICAL_MEM_INFO_SIZE; size_t property_size = 0; @@ -67,7 +67,7 @@ TEST_P(urPhysicalMemGetInfoTest, SuccessSize) { } TEST_P(urPhysicalMemGetInfoTest, SuccessProperties) { - UUR_KNOWN_FAILURE_ON(uur::LevelZero{}); + UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{}); ur_physical_mem_info_t property_name = UR_PHYSICAL_MEM_INFO_PROPERTIES; size_t property_size = 0; diff --git a/test/conformance/virtual_memory/urVirtualMemGranularityGetInfo.cpp b/test/conformance/virtual_memory/urVirtualMemGranularityGetInfo.cpp index 0668bfd9f8..ff9392e091 100644 --- a/test/conformance/virtual_memory/urVirtualMemGranularityGetInfo.cpp +++ b/test/conformance/virtual_memory/urVirtualMemGranularityGetInfo.cpp @@ -7,8 +7,6 @@ struct urVirtualMemGranularityGetInfoTest : uur::urContextTest { void SetUp() override { - UUR_KNOWN_FAILURE_ON(uur::LevelZeroV2{}); - UUR_RETURN_ON_FATAL_FAILURE(urContextTest::SetUp()); ur_bool_t virtual_memory_support = false; ASSERT_SUCCESS(