ci: work around recent boot hangs with "LLVM trunk + kernel mainline"

Let's track it in xlab-uiuc#15.
Before this is solved (or confirmed to be stability issues in LLVM
upstream), let's use a known working commit of LLVM (on 2024-08-21).

Also included are a few other measures for monitoring this issue:

* Prefix the boot step (QEMU script) with `timeout` to fail explicitly
  and sooner once there's a hang
* Try boot the kernel without any patch for comparison

ci/linux-mainline/2_pull_source.sh

@@ -2,6 +2,7 @@
 
 repo=${1:-"xlab-uiuc/linux-mcdc"}
 branch=${2:-"llvm-trunk-next"}
+llvm_ref=${3:-"6b98a723653214a6cde05ae3cb5233af328ff101"}
 
 echo $repo
 echo $branch
@@ -26,9 +27,14 @@ if [[ "$kernel_latest_tag" != "$current_base" ]]; then
 fi
 
 # LLVM if we want to build it from source (optional)
-git clone https://github.com/llvm/llvm-project.git --depth 5
+git clone https://github.com/llvm/llvm-project.git
 # Linux kernel
 git clone https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git --branch $kernel_latest_tag --depth 5
+cp -r linux linux2
+
+# Use a specific ref of LLVM
+cd $MCDC_HOME/llvm-project
+git checkout $llvm_ref
 
 # Apply kernel patches
 cd $MCDC_HOME/linux

ci/linux-mainline/4_build_kernel.sh

@@ -1,5 +1,36 @@
 #!/bin/bash
 
+#### Let's see if vanilla kernel without the patch boots {
+
+cd $MCDC_HOME/linux2
+make LLVM=1 defconfig
+
+./scripts/config -e CONFIG_9P_FS_POSIX_ACL
+./scripts/config -e CONFIG_9P_FS
+./scripts/config -e CONFIG_NET_9P_VIRTIO
+./scripts/config -e CONFIG_NET_9P
+./scripts/config -e CONFIG_PCI
+./scripts/config -e CONFIG_VIRTIO_PCI
+./scripts/config -e CONFIG_OVERLAY_FS
+./scripts/config -e CONFIG_DEBUG_FS
+./scripts/config -e CONFIG_CONFIGFS_FS
+./scripts/config -e CONFIG_MAGIC_SYSRQ
+make LLVM=1 olddefconfig
+
+./scripts/config -e CONFIG_KUNIT
+./scripts/config -e CONFIG_KUNIT_ALL_TESTS
+# See https://github.com/xlab-uiuc/linux-mcdc/pull/10#issuecomment-2291603705
+./scripts/config -d CONFIG_KUNIT_FAULT_TEST
+make LLVM=1 olddefconfig
+
+make LLVM=1 -j$(nproc) >& /dev/null
+
+timeout 60 $MCDC_HOME/linux-mcdc/scripts/q -c "uname -a"
+
+make LLVM=1 mrproper
+
+##### } // Let's see if vanilla kernel without the patch boots
+
 cd $MCDC_HOME/linux
 make LLVM=1 defconfig
 

ci/linux-v5.15.153/2_pull_source.sh

@@ -2,6 +2,7 @@
 
 repo=${1:-"xlab-uiuc/linux-mcdc"}
 branch=${2:-"llvm-trunk-next"}
+llvm_ref=${3:-"6b98a723653214a6cde05ae3cb5233af328ff101"}
 
 echo $repo
 echo $branch
@@ -11,10 +12,14 @@ cd $MCDC_HOME
 # This meta repository
 git clone https://github.com/$repo.git --branch $branch
 # LLVM if we want to build it from source (optional)
-git clone https://github.com/llvm/llvm-project.git --depth 5
+git clone https://github.com/llvm/llvm-project.git
 # Linux kernel
 git clone https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git --branch v5.15.153 --depth 5
 
+# Use a specific ref of LLVM
+cd $MCDC_HOME/llvm-project
+git checkout $llvm_ref
+
 # Apply kernel patches
 cd $MCDC_HOME/linux
 git apply $MCDC_HOME/linux-mcdc/patches/v0.6/0001-llvm-cov-add-Clang-s-Source-based-Code-Coverage-supp.patch

ci/linux-v5.15.153/5_boot_kernel_and_collect_coverage.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+guest_timeout=600
+
 #
 # Test the functionality of reset
 #
@@ -11,7 +13,11 @@ GUEST_COMMANDS="$GUEST_COMMANDS; echo 1 > /sys/kernel/debug/llvm-cov/reset"
 GUEST_COMMANDS="$GUEST_COMMANDS; cp /sys/kernel/debug/llvm-cov/profraw ."
 
 cd $MCDC_HOME/linux
-$MCDC_HOME/linux-mcdc/scripts/q -c "$GUEST_COMMANDS"
+timeout $guest_timeout $MCDC_HOME/linux-mcdc/scripts/q -c "$GUEST_COMMANDS"
+ret=$?
+if [[ $ret -eq 124 ]]; then
+    exit 1
+fi
 
 file profraw |& tee /tmp/file.log
 if ! grep "LLVM raw profile data, version 10" /tmp/file.log > /dev/null; then
@@ -45,7 +51,11 @@ GUEST_COMMANDS="$GUEST_COMMANDS; ls /sys/kernel/debug/llvm-cov"
 GUEST_COMMANDS="$GUEST_COMMANDS; cp /sys/kernel/debug/llvm-cov/profraw ."
 
 cd $MCDC_HOME/linux
-$MCDC_HOME/linux-mcdc/scripts/q -c "$GUEST_COMMANDS"
+timeout $guest_timeout $MCDC_HOME/linux-mcdc/scripts/q -c "$GUEST_COMMANDS"
+ret=$?
+if [[ $ret -eq 124 ]]; then
+    exit 1
+fi
 
 file profraw |& tee /tmp/file.log
 if ! grep "LLVM raw profile data, version 10" /tmp/file.log > /dev/null; then