Single precision and architecture files for SYCL variants

arch/cambridge/dawn/intel/2024.0.0/env.sh

@@ -0,0 +1,52 @@
+# (C) Copyright 1988- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+# Source me to get the correct configure/build/run environment
+
+# Store tracing and disable (module is *way* too verbose)
+{ tracing_=${-//[^x]/}; set +x; } 2>/dev/null
+
+module_load() {
+  echo "+ module load $1"
+  module load $1
+}
+module_unload() {
+  echo "+ module unload $1"
+  module unload $1
+}
+
+# Unload all modules to be certain
+module purge
+
+# Load modules
+module load rhel8/slurm
+module load rhel8/global
+module load dot
+module load dawn-env/2023-12-22
+module load intel-oneapi-compilers/2024.0.0
+module load intel-oneapi-mpi/2021.11.0
+module_load boost/1.83.0
+module_load cmake/3.27.9
+module_load hdf5/1.14.3
+
+set -x
+
+# below should be the default these days
+export EnableImplicitScaling=0
+# card 0, tile 0
+export ZE_AFFINITY_MASK=0.0
+export ONEAPI_DEVICE_SELECTOR=level_zero:gpu
+# 256 registers per thread, fewer threads
+export SYCL_PROGRAM_COMPILE_OPTIONS="-ze-opt-large-register-file"
+# this option affects the overhead of SYCL offload calls, in this case 0 seems to help 
+export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=0
+
+# Restore tracing to stored setting
+{ if [[ -n "$tracing_" ]]; then set -x; else set +x; fi } 2>/dev/null
+
+export ECBUILD_TOOLCHAIN="./toolchain.cmake"

arch/cambridge/dawn/intel/2024.0.0/toolchain.cmake

@@ -0,0 +1,23 @@
+# (C) Copyright 1988- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+####################################################################
+# COMPILER FLAGS
+####################################################################
+
+set( OpenMP_Fortran_FLAGS "-fopenmp" CACHE STRING "" )
+set( OpenMP_C_FLAGS "-fopenmp" CACHE STRING "" )
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3")
+
+####################################################################
+# LINK FLAGS
+####################################################################
+
+set( ECBUILD_SHARED_LINKER_FLAGS "-Wl,--eh-frame-hdr " )
+set( ECBUILD_MODULE_LINKER_FLAGS "-Wl,--eh-frame-hdr -Wl,-Map,loadmap" )
+set( ECBUILD_EXE_LINKER_FLAGS    "-Wl,--eh-frame-hdr -Wl,-Map,loadmap -Wl,--as-needed" )

arch/cambridge/dawn/intel/2024.1.0/env.sh

@@ -0,0 +1,52 @@
+# (C) Copyright 1988- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+# Source me to get the correct configure/build/run environment
+
+# Store tracing and disable (module is *way* too verbose)
+{ tracing_=${-//[^x]/}; set +x; } 2>/dev/null
+
+module_load() {
+  echo "+ module load $1"
+  module load $1
+}
+module_unload() {
+  echo "+ module unload $1"
+  module unload $1
+}
+
+# Unload all modules to be certain
+module purge
+
+# Load modules
+module load rhel8/slurm
+module load rhel8/global
+module load dot
+module load dawn-env/2024-04-15
+module load intel-oneapi-compilers/2024.1.0
+module load intel-oneapi-mpi/2021.12.0
+module_load boost/1.84.0
+module_load cmake/3.27.9
+module_load hdf5/1.14.3
+
+set -x
+
+# below should be the default these days
+export EnableImplicitScaling=0
+# card 0, tile 0
+export ZE_AFFINITY_MASK=0.0
+export ONEAPI_DEVICE_SELECTOR=level_zero:gpu
+# 256 registers per thread, fewer threads
+export SYCL_PROGRAM_COMPILE_OPTIONS="-ze-opt-large-register-file"
+# this option affects the overhead of SYCL offload calls, in this case 0 seems to help 
+export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=0
+
+# Restore tracing to stored setting
+{ if [[ -n "$tracing_" ]]; then set -x; else set +x; fi } 2>/dev/null
+
+export ECBUILD_TOOLCHAIN="./toolchain.cmake"

arch/cambridge/dawn/intel/2024.1.0/toolchain.cmake

@@ -0,0 +1,23 @@
+# (C) Copyright 1988- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+####################################################################
+# COMPILER FLAGS
+####################################################################
+
+set( OpenMP_Fortran_FLAGS "-fopenmp" CACHE STRING "" )
+set( OpenMP_C_FLAGS "-fopenmp" CACHE STRING "" )
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3")
+
+####################################################################
+# LINK FLAGS
+####################################################################
+
+set( ECBUILD_SHARED_LINKER_FLAGS "-Wl,--eh-frame-hdr " )
+set( ECBUILD_MODULE_LINKER_FLAGS "-Wl,--eh-frame-hdr -Wl,-Map,loadmap" )
+set( ECBUILD_EXE_LINKER_FLAGS    "-Wl,--eh-frame-hdr -Wl,-Map,loadmap -Wl,--as-needed" )

bundle.yml

@@ -5,6 +5,7 @@ name    : cloudsc-bundle
 version : 1.0.0-develop
 cmake : >
         CMAKE_LINK_DEPENDS_NO_SHARED=ON
+        CMAKE_EXPORT_COMPILE_COMMANDS=ON
         BUILD_serialbox=OFF
         BUILD_field_api=OFF
         BUILD_eckit=OFF
@@ -123,6 +124,14 @@ options :
         cmake: >
             ENABLE_SYCL=ON
 
+    - sycl-sub-group-size :
+         help  : SYCL subgroup size
+         cmake : SYCL_SUB_GROUP_SIZE={{value}}
+
+    - cloudsc-cpp-math :
+         help  : [C|STD|SYCL]
+         cmake : CLOUDSC_CPP_MATH={{value}}
+
     - with-mpi :
         help  : Enable MPI-parallel kernel
         cmake : ENABLE_MPI=ON

src/cloudsc_sycl/CMakeLists.txt

@@ -14,12 +14,38 @@ ecbuild_add_option( FEATURE CLOUDSC_SYCL
 
 if( HAVE_CLOUDSC_SYCL )
 
+    unset(SYCL_COMPILE_OPTIONS)
+    if( CMAKE_Fortran_COMPILER_ID MATCHES "IntelLLVM" )
+        set(SYCL_COMPILE_OPTIONS -gline-tables-only -fsycl -fsycl-targets=spir64 -fsycl-early-optimizations)
+    endif()
+
+    if (CLOUDSC_CPP_MATH STREQUAL "C")
+        set(GPU_MATH "1")
+        ecbuild_info("Building SYCL variant using C math interface")
+    elseif (CLOUDSC_CPP_MATH STREQUAL "STD")
+        set(GPU_MATH "2")
+        ecbuild_info("Building SYCL variant using C++ STL math interface")
+    elseif(CLOUDSC_CPP_MATH STREQUAL "SYCL")
+        set(GPU_MATH "3")
+        ecbuild_info("Building SYCL variant using SYCL math interface")
+    else ()
+        set(GPU_MATH "1")
+        ecbuild_info("Building SYCL variant using C math interface")
+    endif()
+
+    if (SYCL_SUB_GROUP_SIZE)
+        set(SUB_GROUP_SIZE "${SYCL_SUB_GROUP_SIZE}")
+    else ()
+        set(SUB_GROUP_SIZE "16")
+    endif()
+
     ecbuild_add_library(
         TARGET dwarf-cloudsc-scc-sycl-lib
         INSTALL_HEADERS LISTED
         SOURCES
+	    cloudsc/dtype.h
             cloudsc/yoecldp_c.h
-	    cloudsc/load_state.h
+            cloudsc/load_state.h
             cloudsc/load_state.cpp
             cloudsc/cloudsc_c.kernel
             cloudsc/cloudsc_driver.h
@@ -28,7 +54,7 @@ if( HAVE_CLOUDSC_SYCL )
             cloudsc/cloudsc_validate.cpp
             cloudsc/mycpu.h
             cloudsc/mycpu.cpp
-	PUBLIC_INCLUDES
+        PUBLIC_INCLUDES
             $<INSTALL_INTERFACE:include>
             $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cloudsc>
         PUBLIC_LIBS
@@ -39,11 +65,21 @@ if( HAVE_CLOUDSC_SYCL )
             ${CLOUDSC_DEFINITIONS}
     )
 
+    target_compile_definitions(dwarf-cloudsc-scc-sycl-lib PUBLIC C_MATH=1)
+    target_compile_definitions(dwarf-cloudsc-scc-sycl-lib PUBLIC STD_MATH=2)
+    target_compile_definitions(dwarf-cloudsc-scc-sycl-lib PUBLIC SYCL_MATH=3)
+    target_compile_definitions(dwarf-cloudsc-scc-sycl-lib PUBLIC GPU_MATH=${GPU_MATH})
+    target_compile_definitions(dwarf-cloudsc-scc-sycl-lib PUBLIC SUB_GROUP_SIZE=${SUB_GROUP_SIZE})
+
+    if( DEFINED SYCL_COMPILE_OPTIONS )
+        target_compile_options(dwarf-cloudsc-scc-sycl-lib PUBLIC ${SYCL_COMPILE_OPTIONS})
+    endif()
+
     add_sycl_to_target(
-	    TARGET
-	    	dwarf-cloudsc-scc-sycl-lib
-	    SOURCES
-	    	cloudsc/cloudsc_c.kernel
+            TARGET
+                dwarf-cloudsc-scc-sycl-lib
+            SOURCES
+                cloudsc/cloudsc_c.kernel
     )
 
     ecbuild_add_executable(
@@ -52,10 +88,14 @@ if( HAVE_CLOUDSC_SYCL )
         LIBS dwarf-cloudsc-scc-sycl-lib
     )
 
+    if( DEFINED SYCL_COMPILE_OPTIONS )
+        target_compile_options(dwarf-cloudsc-scc-sycl PUBLIC ${SYCL_COMPILE_OPTIONS})
+    endif()
+
     ecbuild_add_test(
         TARGET dwarf-cloudsc-sycl-serial
         COMMAND bin/dwarf-cloudsc-scc-sycl
-        ARGS 1 1000 128
+        ARGS 1 1024 128
         WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../..
         OMP 1
     )
@@ -66,6 +106,7 @@ if( HAVE_CLOUDSC_SYCL )
         TARGET dwarf-cloudsc-scc-hoist-sycl-lib
         INSTALL_HEADERS LISTED
         SOURCES
+	    cloudsc/dtype.h
             cloudsc/yoecldp_c.h
             cloudsc/load_state.h
             cloudsc/load_state.cpp
@@ -87,23 +128,36 @@ if( HAVE_CLOUDSC_SYCL )
             ${CLOUDSC_DEFINITIONS}
     )
 
+    target_compile_definitions(dwarf-cloudsc-scc-hoist-sycl-lib PUBLIC C_MATH=1)
+    target_compile_definitions(dwarf-cloudsc-scc-hoist-sycl-lib PUBLIC STD_MATH=2)
+    target_compile_definitions(dwarf-cloudsc-scc-hoist-sycl-lib PUBLIC SYCL_MATH=3)
+    target_compile_definitions(dwarf-cloudsc-scc-hoist-sycl-lib PUBLIC GPU_MATH=${GPU_MATH})
+    target_compile_definitions(dwarf-cloudsc-scc-hoist-sycl-lib PUBLIC SUB_GROUP_SIZE=${SUB_GROUP_SIZE})
+
+    if( DEFINED SYCL_COMPILE_OPTIONS )
+        target_compile_options(dwarf-cloudsc-scc-hoist-sycl-lib PUBLIC ${SYCL_COMPILE_OPTIONS})
+    endif()
+
     add_sycl_to_target(
-	    TARGET
-	    	dwarf-cloudsc-scc-hoist-sycl-lib
-	    SOURCES
-	    	cloudsc/cloudsc_c_hoist.kernel
+            TARGET
+                dwarf-cloudsc-scc-hoist-sycl-lib
+            SOURCES
+                cloudsc/cloudsc_c_hoist.kernel
     )
 
     ecbuild_add_executable(
         TARGET  dwarf-cloudsc-scc-hoist-sycl
         SOURCES dwarf_cloudsc.cpp
         LIBS dwarf-cloudsc-scc-hoist-sycl-lib
     )
+    if( DEFINED SYCL_COMPILE_OPTIONS )
+        target_compile_options(dwarf-cloudsc-scc-hoist-sycl PUBLIC ${SYCL_COMPILE_OPTIONS})
+    endif()
 
     ecbuild_add_test(
         TARGET dwarf-cloudsc-sycl-hoist-serial
         COMMAND bin/dwarf-cloudsc-scc-hoist-sycl
-        ARGS 1 1000 128
+        ARGS 1 1024 128
         WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../..
         OMP 1
     )
@@ -114,6 +168,7 @@ if( HAVE_CLOUDSC_SYCL )
         TARGET dwarf-cloudsc-scc-k-caching-sycl-lib
         INSTALL_HEADERS LISTED
         SOURCES
+	    cloudsc/dtype.h
             cloudsc/yoecldp_c.h
             cloudsc/load_state.h
             cloudsc/load_state.cpp
@@ -135,23 +190,36 @@ if( HAVE_CLOUDSC_SYCL )
             ${CLOUDSC_DEFINITIONS}
     )
 
+    target_compile_definitions(dwarf-cloudsc-scc-k-caching-sycl-lib PUBLIC C_MATH=1)
+    target_compile_definitions(dwarf-cloudsc-scc-k-caching-sycl-lib PUBLIC STD_MATH=2)
+    target_compile_definitions(dwarf-cloudsc-scc-k-caching-sycl-lib PUBLIC SYCL_MATH=3)
+    target_compile_definitions(dwarf-cloudsc-scc-k-caching-sycl-lib PUBLIC GPU_MATH=${GPU_MATH})
+    target_compile_definitions(dwarf-cloudsc-scc-k-caching-sycl-lib PUBLIC SUB_GROUP_SIZE=${SUB_GROUP_SIZE})
+
+    if( DEFINED SYCL_COMPILE_OPTIONS )
+        target_compile_options(dwarf-cloudsc-scc-k-caching-sycl-lib PUBLIC ${SYCL_COMPILE_OPTIONS})
+    endif()
+
     add_sycl_to_target(
-	    TARGET 
-	    	dwarf-cloudsc-scc-k-caching-sycl-lib 
-	    SOURCES 
-	    	cloudsc/cloudsc_c_k_caching.kernel
+            TARGET
+                dwarf-cloudsc-scc-k-caching-sycl-lib
+            SOURCES
+                cloudsc/cloudsc_c_k_caching.kernel
     )
 
     ecbuild_add_executable(
         TARGET  dwarf-cloudsc-scc-k-caching-sycl
         SOURCES dwarf_cloudsc.cpp
         LIBS dwarf-cloudsc-scc-k-caching-sycl-lib
     )
+    if( DEFINED SYCL_COMPILE_OPTIONS )
+        target_compile_options(dwarf-cloudsc-scc-k-caching-sycl PUBLIC ${SYCL_COMPILE_OPTIONS})
+    endif()
 
     ecbuild_add_test(
         TARGET dwarf-cloudsc-sycl-k-caching-serial
         COMMAND bin/dwarf-cloudsc-scc-k-caching-sycl
-        ARGS 1 1000 128
+        ARGS 1 1024 128
         WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../..
         OMP 1
     )
@@ -169,6 +237,4 @@ if( HAVE_CLOUDSC_SYCL )
        ${CMAKE_CURRENT_SOURCE_DIR}/../../config-files/reference.h5 ${CMAKE_CURRENT_BINARY_DIR}/../../../reference.h5 )
  endif()
 
-else()
-	ecbuild_info( "Serialbox not found, disabling SYCL version" )
 endif()