From 605c4a696fcaf56921598d1331001aaeb12614a5 Mon Sep 17 00:00:00 2001
From: Brian Harrison <Brian.Harrison@amd.com>
Date: Thu, 15 Aug 2024 14:46:00 +0000
Subject: [PATCH] Disable MLIR when building debug to workaround sanitizer
 issue

---
 Jenkinsfile                                     | 13 ++++++++++---
 test/nightlies/JenkinsfileNightlyAux            |  3 ++-
 test/nightlies/JenkinsfileNightlyConv2D         |  3 ++-
 test/nightlies/JenkinsfileNightlyConv2Daux      |  3 ++-
 test/nightlies/JenkinsfileNightlyFastFullConv2D |  3 ++-
 test/nightlies/JenkinsfileNightlyFusions        |  3 ++-
 test/nightlies/JenkinsfileNightlyRNN            |  3 ++-
 test/nightlies/JenkinsfileNightlyReduce         |  3 ++-
 test/nightlies/JenkinsfileNightlyTensorOps      |  3 ++-
 9 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 424c1d8a2f..a54735e8c6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -33,14 +33,21 @@ def cmake_build(Map conf=[:]){
     def debug_flags = "-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined -Wno-option-ignored " + conf.get("extradebugflags", "")
     def build_envs = "CTEST_PARALLEL_LEVEL=4 " + conf.get("build_env","")
     def prefixpath = conf.get("prefixpath","/opt/rocm")
+    def build_type_debug = (conf.get("build_type",'release') == 'debug')
+    def code_conv_enabled = conf.get("codecov", false)
+
     def mlir_args = " -DMIOPEN_USE_MLIR=" + conf.get("mlir_build", "ON")
+    // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
+    if (build_type_debug || code_conv_enabled)
+    {
+        mlir_args = " -DMIOPEN_USE_MLIR=OFF"
+    }
+
     def setup_args = mlir_args + " -DMIOPEN_GPU_SYNC=Off " + conf.get("setup_flags","")
     def build_fin = conf.get("build_fin", "OFF")
 
     setup_args = setup_args + " -DCMAKE_PREFIX_PATH=${prefixpath} "
 
-    def build_type_debug = (conf.get("build_type",'release') == 'debug')
-
     //cmake_env can overwrite default CXX variables.
     def cmake_envs = "CXX=${compiler} CXXFLAGS='-Werror' " + conf.get("cmake_ex_env","")
 
@@ -72,7 +79,7 @@ def cmake_build(Map conf=[:]){
         test_flags = " --disable-verification-cache " + test_flags
     }
 
-    if(conf.get("codecov", false)){ //Need
+    if(code_conv_enabled){ //Need
         setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags} -fprofile-arcs -ftest-coverage' -DCODECOV_TEST=On " + setup_args
     }else if(build_type_debug){
         setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}'" + setup_args
diff --git a/test/nightlies/JenkinsfileNightlyAux b/test/nightlies/JenkinsfileNightlyAux
index d1b7e70951..feb6ab5dc7 100644
--- a/test/nightlies/JenkinsfileNightlyAux
+++ b/test/nightlies/JenkinsfileNightlyAux
@@ -46,7 +46,8 @@ def buildJob(compiler, flags, image, test, testargs){
             timeout(time: 8, unit: 'HOURS')
             {
                 sh "echo \$HSA_ENABLE_SDMA"
-                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
+                // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
+                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
                 sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
                 sh "MIOPEN_LOG_LEVEL=5 ./build/bin/${test} ${testargs}"
 
diff --git a/test/nightlies/JenkinsfileNightlyConv2D b/test/nightlies/JenkinsfileNightlyConv2D
index 5b67507376..809cfff95c 100644
--- a/test/nightlies/JenkinsfileNightlyConv2D
+++ b/test/nightlies/JenkinsfileNightlyConv2D
@@ -47,7 +47,8 @@ def buildJob(compiler, flags, image, test, testargs){
             timeout(time: 8, unit: 'HOURS')
             {
                 sh "echo \$HSA_ENABLE_SDMA"
-                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
+                // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
+                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
                 sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
                 sh "MIOPEN_LOG_LEVEL=6 ./build/bin/${test} ${testargs}"
 
diff --git a/test/nightlies/JenkinsfileNightlyConv2Daux b/test/nightlies/JenkinsfileNightlyConv2Daux
index e6439313f7..f967352b21 100644
--- a/test/nightlies/JenkinsfileNightlyConv2Daux
+++ b/test/nightlies/JenkinsfileNightlyConv2Daux
@@ -48,7 +48,8 @@ def buildJob(compiler, flags, image, test, testargs){
             timeout(time: 8, unit: 'HOURS')
             {
                 sh "echo \$HSA_ENABLE_SDMA"
-                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
+                // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
+                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
                 sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
                 sh "MIOPEN_LOG_LEVEL=6 ./build/bin/${test} ${testargs}"
 
diff --git a/test/nightlies/JenkinsfileNightlyFastFullConv2D b/test/nightlies/JenkinsfileNightlyFastFullConv2D
index eaa703b3b1..a94992002b 100644
--- a/test/nightlies/JenkinsfileNightlyFastFullConv2D
+++ b/test/nightlies/JenkinsfileNightlyFastFullConv2D
@@ -43,7 +43,8 @@ def buildJob(compiler, flags, image, test, testargs){
             timeout(time: 12, unit: 'HOURS')
             {
                 sh "echo \$HSA_ENABLE_SDMA"
-                sh "rm -rf build; mkdir build; cd build; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
+                // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
+                sh "rm -rf build; mkdir build; cd build; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
                 sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
                 sh "MIOPEN_ENABLE_LOGGING_CMD=1 MIOPEN_LOG_LEVEL=6 ./build/bin/${test} ${testargs}"
 
diff --git a/test/nightlies/JenkinsfileNightlyFusions b/test/nightlies/JenkinsfileNightlyFusions
index 84d8329c6a..269480baf7 100644
--- a/test/nightlies/JenkinsfileNightlyFusions
+++ b/test/nightlies/JenkinsfileNightlyFusions
@@ -46,7 +46,8 @@ def buildJob(compiler, flags, image, test, testargs){
             timeout(time: 8, unit: 'HOURS')
             {
                 sh "echo \$HSA_ENABLE_SDMA"
-                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
+                // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
+                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
                 sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
                 sh "MIOPEN_LOG_LEVEL=5 ./build/bin/${test} ${testargs}"
 
diff --git a/test/nightlies/JenkinsfileNightlyRNN b/test/nightlies/JenkinsfileNightlyRNN
index b299f832cc..33ac860053 100644
--- a/test/nightlies/JenkinsfileNightlyRNN
+++ b/test/nightlies/JenkinsfileNightlyRNN
@@ -47,7 +47,8 @@ def buildJob(compiler, flags, image, test, testargs){
             timeout(time: 8, unit: 'HOURS')
             {
                 sh "echo \$HSA_ENABLE_SDMA"
-                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
+                // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
+                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=Off -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
                 sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
                 sh "MIOPEN_LOG_LEVEL=5 ./build/bin/${test} ${testargs}"
 
diff --git a/test/nightlies/JenkinsfileNightlyReduce b/test/nightlies/JenkinsfileNightlyReduce
index a7d6c6becb..c2b14e8f79 100644
--- a/test/nightlies/JenkinsfileNightlyReduce
+++ b/test/nightlies/JenkinsfileNightlyReduce
@@ -47,7 +47,8 @@ def buildJob(compiler, flags, image, test, testargs){
             timeout(time: 6, unit: 'HOURS')
             {
                 sh "echo \$HSA_ENABLE_SDMA"
-                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
+                // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
+                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=OFF -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
                 sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
                 sh "MIOPEN_LOG_LEVEL=5 ./build/bin/${test} ${testargs}"
 
diff --git a/test/nightlies/JenkinsfileNightlyTensorOps b/test/nightlies/JenkinsfileNightlyTensorOps
index e19a0a9f18..d326c64f9f 100644
--- a/test/nightlies/JenkinsfileNightlyTensorOps
+++ b/test/nightlies/JenkinsfileNightlyTensorOps
@@ -48,7 +48,8 @@ def buildJob(compiler, flags, image, test, testargs){
             timeout(time: 6, unit: 'HOURS')
             {
                 sh "echo \$HSA_ENABLE_SDMA"
-                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
+                // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors.
+                sh "rm -rf build; mkdir build; cd build; export PATH=/opt/rocm/bin:$PATH; CXX=${compiler} CXXFLAGS='-Werror' cmake -DMIOPEN_GPU_SYNC=On -DMIOPEN_USE_MLIR=Off -DMIOPEN_TEST_FLAGS='--disable-verification-cache' -DCMAKE_CXX_FLAGS_DEBUG='-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined' ${flags} .."
                 sh "cd build; CTEST_PARALLEL_LEVEL=4 MIOPEN_VERIFY_CACHE_PATH=/var/jenkins/.cache/miopen/vcache dumb-init make -j\$(nproc) ${test}"
                 sh "MIOPEN_LOG_LEVEL=5 ./build/bin/${test} ${testargs}"