From ce15c54cb9e5756b1d157ad5e5e10bf98a4d90f0 Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Fri, 1 Mar 2024 08:15:47 -0800 Subject: [PATCH 1/4] Upgraded to CMSSW 14_1_0_pre0 --- README.md | 24 ++++++++++++------------ SDL/Constants.h | 6 ++++-- SDL/Event.cc | 2 +- setup.sh | 4 ++-- setup_hpg.sh | 4 ++-- 5 files changed, 21 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 9d1e0f6c..95f7b741 100644 --- a/README.md +++ b/README.md @@ -129,14 +129,14 @@ cd .. ```bash mkdir workingFolder # Create the folder you will be working in cd workingFolder -cmsrel CMSSW_13_3_0_pre3 -cd CMSSW_13_3_0_pre3/src +cmsrel CMSSW_14_1_0_pre0 +cd CMSSW_14_1_0_pre0/src cmsenv git cms-init git remote add SegLink git@github.com:SegmentLinking/cmssw.git -git fetch SegLink CMSSW_13_3_0_pre3_LST_X +git fetch SegLink CMSSW_14_1_0_pre0_LST_X git cms-addpkg RecoTracker Configuration -git checkout CMSSW_13_3_0_pre3_LST_X +git checkout CMSSW_14_1_0_pre0_LST_X #To include both the CPU library and GPU library into CMSSW, create 3 xml files (headers file has no library). #Before writing the following xml file, check that libsdl_cpu.so and libsdl_gpu.so can be found under the ../../../TrackLooper/SDL/ folder. cat <lst_headers.xml @@ -247,7 +247,7 @@ The setup scripts included in this repository assume that the [CernVM File Syste The necessary dependencies are CUDA, ROOT, the Boost libraries, Alpaka, and some CMSSW headers. CUDA, ROOT, and Boost, are fairly standard libraries and are available from multiple package managers. For the remaining necessary headers you will need to clone the [Alpaka](https://github.com/alpaka-group/alpaka) and [CMSSW](https://github.com/cms-sw/cmssw) repositories. The Alpaka repository is reasonably sized, but the CMSSW one extremely large, especially considering that we only need a tiny fraction of its files to build LST. We can get only the Alpaka interface headers from CMSSW by running the following commands. ``` bash -git clone --filter=blob:none --no-checkout --depth 1 --sparse --branch CMSSW_13_3_X https://github.com/cms-sw/cmssw.git +git clone --filter=blob:none --no-checkout --depth 1 --sparse --branch CMSSW_14_1_X https://github.com/cms-sw/cmssw.git cd cmssw git sparse-checkout add HeterogeneousCore/AlpakaInterface git checkout @@ -257,16 +257,16 @@ Then all that is left to do is set some environment variables. We give an exampl ```bash # These two lines are only needed to set the right version of gcc and nvcc. They are not needed for standard installations. -export PATH=/cvmfs/cms.cern.ch/el8_amd64_gcc11/external/gcc/11.4.1-30ebdc301ebd200f2ae0e3d880258e65/bin:/cvmfs/cms.cern.ch/el8_amd64_gcc11/cms/cmssw/CMSSW_13_3_0_pre3/external/el8_amd64_gcc11/bin:$PATH -export LD_LIBRARY_PATH=/cvmfs/cms.cern.ch/el8_amd64_gcc11/cms/cmssw/CMSSW_13_3_0_pre3/biglib/el8_amd64_gcc11:/cvmfs/cms.cern.ch/el8_amd64_gcc11/cms/cmssw/CMSSW_13_3_0_pre3/lib/el8_amd64_gcc11:/cvmfs/cms.cern.ch/el8_amd64_gcc11/cms/cmssw/CMSSW_13_3_0_pre3/external/el8_amd64_gcc11/lib:/cvmfs/cms.cern.ch/el8_amd64_gcc11/external/gcc/11.4.1-30ebdc301ebd200f2ae0e3d880258e65/lib64:/cvmfs/cms.cern.ch/el8_amd64_gcc11/external/gcc/11.4.1-30ebdc301ebd200f2ae0e3d880258e65/lib:$LD_LIBRARY_PATH +export PATH=/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/gcc/12.3.1-40d504be6370b5a30e3947a6e575ca28/bin:/cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_0_pre0/external/el8_amd64_gcc12/bin:$PATH +export LD_LIBRARY_PATH=/cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_0_pre0/biglib/el8_amd64_gcc12:/cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_0_pre0/lib/el8_amd64_gcc12:/cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_0_pre0/external/el8_amd64_gcc12/lib:/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/gcc/12.3.1-40d504be6370b5a30e3947a6e575ca28/lib64:/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/gcc/12.3.1-40d504be6370b5a30e3947a6e575ca28/lib:$LD_LIBRARY_PATH # These are the lines that you need to manually change for a CVMFS-less setup. # In this example we use cvmfs paths since that is where the dependencies are in lnx7188/cgpu1, but they can point to local directories. -export BOOST_ROOT=/cvmfs/cms.cern.ch/el8_amd64_gcc11/external/boost/1.80.0-536665e33076f709097addc8619ee0a0 -export ALPAKA_ROOT=/cvmfs/cms.cern.ch/el8_amd64_gcc11/external/alpaka/develop-20230621-5659bacc780954326761375d025a3208 -export CUDA_HOME=/cvmfs/cms.cern.ch/el8_amd64_gcc11/external/cuda/11.8.0-9f0af0f4206be7b705fe550319c49a11 -export ROOT_ROOT=/cvmfs/cms.cern.ch/el8_amd64_gcc11/lcg/root/6.26.11-700833a012ccdeb623021803b605d653 -export CMSSW_BASE=/cvmfs/cms.cern.ch/el8_amd64_gcc11/cms/cmssw/CMSSW_13_3_0_pre3 +export BOOST_ROOT=/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/boost/1.80.0-9d4e0dc8cf1f8fc8a67cf53ae917955e +export ALPAKA_ROOT=/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/alpaka/1.1.0-0e0b978d445f7af747cf00064c146356 +export CUDA_HOME=/cvmfs/cms.cern.ch/el8_amd64_gcc12/external/cuda/12.2.1-bdf3fff69eaec65abe18a7569592cab6 +export ROOT_ROOT=/cvmfs/cms.cern.ch/el8_amd64_gcc12/lcg/root/6.30.05-d0417d1d3c55ded4cc9aa75628166a77 +export CMSSW_BASE=/cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_0_pre0 # These lines are needed to account for some extra environment variables that are exported in the setup script. export LD_LIBRARY_PATH=$PWD/SDL/cuda:$PWD/SDL/cpu:$PWD:$LD_LIBRARY_PATH diff --git a/SDL/Constants.h b/SDL/Constants.h index 60829b1b..0a10ad2e 100644 --- a/SDL/Constants.h +++ b/SDL/Constants.h @@ -62,10 +62,12 @@ namespace SDL { }; #endif - auto const devHost = alpaka::getDevByIdx(0u); + auto const platformAcc = alpaka::Platform{}; + auto const platformHost = alpaka::Platform{}; + auto const devHost = alpaka::getDevByIdx(platformHost, 0u); #if defined ALPAKA_ACC_GPU_CUDA_ENABLED || defined ALPAKA_ACC_CPU_B_SEQ_T_THREADS_ENABLED || \ defined ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED - auto const devAcc = alpaka::getDevByIdx(0u); + auto const devAcc = alpaka::getDevByIdx(platformAcc, 0u); using QueueAcc = alpaka::Queue; #endif diff --git a/SDL/Event.cc b/SDL/Event.cc index a43fc383..90ca0ac9 100644 --- a/SDL/Event.cc +++ b/SDL/Event.cc @@ -51,7 +51,7 @@ void SDL::Event::init(bool verbose) { } // Standalone constructor that has each event object create its own queue. -SDL::Event::Event(bool verbose) : queue(alpaka::getDevByIdx(0u)) { init(verbose); } +SDL::Event::Event(bool verbose) : queue(alpaka::getDevByIdx(platformAcc, 0u)) { init(verbose); } void SDL::Event::resetEvent() { //reset the arrays diff --git a/setup.sh b/setup.sh index 77d0e1f3..6bc292ca 100644 --- a/setup.sh +++ b/setup.sh @@ -6,8 +6,8 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source $DIR/code/rooutil/thisrooutil.sh -export SCRAM_ARCH=el8_amd64_gcc11 -export CMSSW_VERSION=CMSSW_13_3_0_pre3 +export SCRAM_ARCH=el8_amd64_gcc12 +export CMSSW_VERSION=CMSSW_14_1_0_pre0 source /cvmfs/cms.cern.ch/cmsset_default.sh cd /cvmfs/cms.cern.ch/$SCRAM_ARCH/cms/cmssw/$CMSSW_VERSION/src diff --git a/setup_hpg.sh b/setup_hpg.sh index d92b3802..2ac19737 100644 --- a/setup_hpg.sh +++ b/setup_hpg.sh @@ -11,8 +11,8 @@ module load cuda/11.4.3 git DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source $DIR/code/rooutil/thisrooutil.sh -export SCRAM_ARCH=el8_amd64_gcc11 -export CMSSW_VERSION=CMSSW_13_3_0_pre3 +export SCRAM_ARCH=el8_amd64_gcc12 +export CMSSW_VERSION=CMSSW_14_1_0_pre0 export CUDA_HOME=${HPC_CUDA_DIR} source /cvmfs/cms.cern.ch/cmsset_default.sh From 2cddd19b8c98b7446522ab8c96d1ab7bf6751acf Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Fri, 1 Mar 2024 09:18:11 -0800 Subject: [PATCH 2/4] Fixed compilation errors when warnings are enabled --- SDL/Event.cc | 8 ++++---- SDL/PixelTriplet.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/SDL/Event.cc b/SDL/Event.cc index 90ca0ac9..8731fe28 100644 --- a/SDL/Event.cc +++ b/SDL/Event.cc @@ -908,9 +908,9 @@ void SDL::Event::createPixelTriplets() { alpaka::wait(queue); #ifdef Warnings - auto nPixelTriplets_buf = allocBufWrapper(devHost, 1, queue); + auto nPixelTriplets_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nPixelTriplets_buf, pixelTripletsBuffers->nPixelTriplets_buf, 1); + alpaka::memcpy(queue, nPixelTriplets_buf, pixelTripletsBuffers->nPixelTriplets_buf, 1u); alpaka::wait(queue); std::cout << "number of pixel triplets = " << *alpaka::getPtrNative(nPixelTriplets_buf) << std::endl; @@ -1168,9 +1168,9 @@ void SDL::Event::createPixelQuintuplets() { alpaka::wait(queue); #ifdef Warnings - auto nPixelQuintuplets_buf = allocBufWrapper(devHost, 1, queue); + auto nPixelQuintuplets_buf = allocBufWrapper(devHost, 1, queue); - alpaka::memcpy(queue, nPixelQuintuplets_buf, pixelQuintupletsBuffers->nPixelQuintuplets_buf, 1); + alpaka::memcpy(queue, nPixelQuintuplets_buf, pixelQuintupletsBuffers->nPixelQuintuplets_buf, 1u); alpaka::wait(queue); std::cout << "number of pixel quintuplets = " << *alpaka::getPtrNative(nPixelQuintuplets_buf) << std::endl; diff --git a/SDL/PixelTriplet.h b/SDL/PixelTriplet.h index 57c2799d..cba81807 100644 --- a/SDL/PixelTriplet.h +++ b/SDL/PixelTriplet.h @@ -1078,7 +1078,7 @@ namespace SDL { if (tripletLowerModuleIndex >= *modulesInGPU.nLowerModules) { printf("tripletLowerModuleIndex %d >= modulesInGPU.nLowerModules %d \n", tripletLowerModuleIndex, - modulesInGPU.nLowerModules); + *modulesInGPU.nLowerModules); continue; //sanity check } #endif From da10b50ba12f52601e9614ea65b3a42bb55c2d81 Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Fri, 8 Mar 2024 11:19:15 -0800 Subject: [PATCH 3/4] Generate PTX code for compute_70 --- SDL/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SDL/Makefile b/SDL/Makefile index 9df51a4d..c751b905 100644 --- a/SDL/Makefile +++ b/SDL/Makefile @@ -27,7 +27,7 @@ LIBS=$(LIB_CUDA) $(LIB_CPU) # # Different architectures to optimize for -GENCODE_SM70 := -gencode arch=compute_70,code=sm_70 +GENCODE_SM70 := -gencode arch=compute_70,code=[sm_70,compute_70] GENCODE_SM89 := -gencode arch=compute_89,code=sm_89 CXX = g++ From fb5b7636389c105a8bd93ceff4094ec5aa2d1068 Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Fri, 8 Mar 2024 14:52:21 -0500 Subject: [PATCH 4/4] Generate PTX code for compute_89 too Co-authored-by: Slava Krutelyov --- SDL/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SDL/Makefile b/SDL/Makefile index c751b905..369ef7af 100644 --- a/SDL/Makefile +++ b/SDL/Makefile @@ -28,7 +28,7 @@ LIBS=$(LIB_CUDA) $(LIB_CPU) # Different architectures to optimize for GENCODE_SM70 := -gencode arch=compute_70,code=[sm_70,compute_70] -GENCODE_SM89 := -gencode arch=compute_89,code=sm_89 +GENCODE_SM89 := -gencode arch=compute_89,code=[sm_89,compute_89] CXX = g++ CXXFLAGS_CPU = -march=native -mtune=native -Ofast -fno-reciprocal-math -fopenmp-simd -g -Wall -Wshadow -Woverloaded-virtual -fPIC -fopenmp -I..