feat: add vfio modules to image (#198)

* feat: make vfio implementation bootc compliant

* chore: update comment

* chore: tell people to tag @HikariKnight instead of "me"

just/aurora-system.just

@@ -35,11 +35,11 @@ toggle-devmode:
     source /usr/lib/ujust/ujust.sh
     CURRENT_IMAGE=$(rpm-ostree status -b --json | jq -r '.deployments[0]."container-image-reference"')
     if grep -q "/var/ublue-os/image" <<< $CURRENT_IMAGE ; then
-        cat <<EOF
+        bash -c "cat <<EOF
     Before we can switch to the Aurora Developer Experience
     the current system needs an update. Please run 'ujust update'
     and reboot your system when the update is finished
-    EOF
+    EOF"
         exit
     fi
     if /bin/grep -q "\-dx" <<< $CURRENT_IMAGE ; then
@@ -132,22 +132,22 @@ configure-vfio ACTION="":
     fi
     OPTION={{ ACTION }}
     if [ "$OPTION" == "help" ]; then
-        cat <<EOF
+        bash -c "cat <<EOF
     Usage: ujust configure-vfio <option>
       <option>: Specify the quick option to skip the prompt
       Use 'vfio-on' to select Enable VFIO drivers
       Use 'vfio-off' to select Disable VFIO drivers
-      Use 'kvmfr' to select Autocreate Looking-Glass shm
-    EOF
+      Use 'kvmfr' to enable kvmfr module for looking-glass
+    EOF"
         exit 0
     elif [ "$OPTION" == "" ]; then
-        cat <<EOF
+        bash -c "cat <<EOF
     ${bold}VFIO and kvmfr Configuration${normal}
     This is only used for GPU passthrough of a secondary dGPU.
-    It will enable vfio and configure kvmfr for use with $(Urllink "https://looking-glass.io" "Looking Glass")
+    It will enable vfio and configure kvmfr for use with $(Urllink "https://looking-glass.io" "Looking Glass").
     If you do not plan to use any of this then press ESC.
     ${bold}NOTE:${normal} Since this is a niche use case, support will be ${b}very limited${n}
-    EOF
+    EOF"
         OPTION=$(Choose \
           "Enable VFIO drivers" \
           "Disable VFIO drivers" \
@@ -158,8 +158,6 @@ configure-vfio ACTION="":
         echo "Enabling VFIO..."
         CPU_VENDOR=$(grep "vendor_id" "/proc/cpuinfo" | uniq | awk -F": " '{ print $2 }')
         VENDOR_KARG="unset"
-        echo 'add_drivers+=" vfio vfio_iommu_type1 vfio-pci "' | sudo tee /etc/dracut.conf.d/vfio.conf
-        rpm-ostree initramfs --enable
         if [[ ${CPU_VENDOR} == "AuthenticAMD" ]]; then
             VENDOR_KARG="amd_iommu=on"
         elif [[ ${CPU_VENDOR} == "GenuineIntel" ]]; then
@@ -176,11 +174,23 @@ configure-vfio ACTION="":
                 --append-if-missing="vfio_pci.disable_vga=1" \
                 --append-if-missing="kvm.ignore_msrs=1" \
                 --append-if-missing="kvm.report_ignored_msrs=0"
+            if sudo test ! -f "/etc/libvirt/hooks/qemu"; then
+                echo "Adding libvirt qemu hooks"
+                sudo wget 'https://raw.githubusercontent.com/PassthroughPOST/VFIO-Tools/master/libvirt_hooks/qemu' -O /etc/libvirt/hooks/qemu
+                sudo chmod +x /etc/libvirt/hooks/qemu
+                sudo grep -A1 -B1 "# Add" /etc/libvirt/hooks/qemu | sed 's/^# //g'
+                echo ""
+                if sudo test ! -d "/etc/libvirt/hooks/qemu.d"; then
+                    sudo mkdir /etc/libvirt/hooks/qemu.d
+                fi
+            fi
             echo "VFIO will be enabled on next boot, make sure you enable IOMMU, VT-d or AMD-v in your BIOS!"
             echo "Please understand that since this is such a niche use case, support will be very limited!"
             echo "To add your unused/second GPU device ids to the vfio driver by running"
             echo 'rpm-ostree kargs --append-if-missing="vfio-pci.ids=xxxx:yyyy,xxxx:yyzz"'
-            echo "NOTE: Your second GPU will not be usable by the host after you do this!"
+            echo 'you can find these using "ls-iommu -gr" or "ls-iommu -grr"'
+            echo 'for a dedicated gpu there will be at least 1 vga device and 1 audio device.'
+            echo "NOTE: The GPU you bound to vfio-pci will not be usable by the host after you do this!"
         fi
     elif [[ "${OPTION,,}" =~ (^disable[[:space:]]vfio|vfio-off) ]]; then
         echo ""
@@ -191,13 +201,10 @@ configure-vfio ACTION="":
             VFIO_IDS="$(rpm-ostree kargs | sed -E 's/.+(vfio_pci.ids=.+\s)/\1/' | awk '{ print $1 }' | grep vfio_pci.ids)"
             VFIO_IDS_KARG=""
             if [ -n "$VFIO_IDS" ]; then
-              echo "Found VFIO ids in kargs, adding the below line to removal list"
-              echo "$VFIO_IDS"
-              VFIO_IDS_KARG="--delete-if-present=\"$VFIO_IDS\""
+                echo "Found VFIO ids in kargs, adding the below line to removal list"
+                echo "$VFIO_IDS"
+                VFIO_IDS_KARG="--delete-if-present=\"$VFIO_IDS\""
             fi
-            echo "Removing dracut modules"
-            sudo rm /etc/dracut.conf.d/vfio.conf
-            rpm-ostree initramfs --enable
             rpm-ostree kargs \
                 --delete-if-present="iommu=pt" \
                 --delete-if-present="iommu=on" \

system_files/dx/usr/lib/dracut/dracut.conf.d/80-vfio.conf

@@ -0,0 +1 @@
+force_drivers+=" vfio vfio_iommu_type1 vfio-pci "

system_files/dx/usr/libexec/aurora-dx-kvmfr-setup

@@ -1,31 +1,38 @@
 #!/usr/bin/env bash
 source /usr/lib/ujust/ujust.sh
+# Get the image name based on the filename
+IMAGE_NAME=$(basename "$0" | sed -E 's/(-dx|)-kvmfr-setup//')
 
 # Required disclaimer and where to report issues first
 echo "$(Urllink "https://looking-glass.io/docs/rc/ivshmem_kvmfr/#libvirt" "This module") along with $(Urllink "https://looking-glass.io" "Looking Glass") is very experimental and not recommended for production use!"
 echo "The ublue team packages the kvmfr module only because it has to be supplied with the system image while using an atomic desktop."
-echo "If you do plan to use Looking Glass, please $(Urllink "https://universal-blue.discourse.group/docs?topic=956" "follow the guide here") on how to compile it for your system."
+echo "If you do plan to use Looking Glass, please $(Urllink "https://docs.bazzite.gg/Advanced/looking-glass/#compiling-looking-glass-client" "follow the guide here") on how to compile it for your system."
 echo "To use the kvmfr module after enabling it, just add and edit the xml for libvirt from the documentation in the first link."
-echo "Since we package the kvmfr module please open kvmfr related issues you have on Aurora and tag me"
-echo "in the $(Urllink "https://discord.gg/WEu6BdFEtp" "Universal Blue Discord") or the $(Urllink "https://github.com/ublue-os/aurora/issues" "Aurora Github issue tracker")."
+echo "Since we package the kvmfr module please open kvmfr related issues you have on ${IMAGE_NAME^} and tag @HikariKnight"
+echo "in the $(Urllink "https://discord.gg/WEu6BdFEtp" "Universal Blue Discord") or the $(Urllink "https://github.com/ublue-os/${IMAGE_NAME,,}/issues" "${IMAGE_NAME^} Github issue tracker")."
 echo "~ @HikariKnight"
 
 CONFIRM=$(Choose Ok Cancel)
 if [ "$CONFIRM" == "Cancel" ]; then
     exit 0
 fi
 
-# Add kvmfr to dracut so that it's modprobe file can be used
-echo ""
+# Add kvmfr modprobe file following upstream documentation
 echo "Setting up kvmfr module so it loads next boot"
-sudo bash -c 'cat << KVMFR_DRACUT > /etc/dracut.conf.d/kvmfr.conf
-install_items+=" /etc/modprobe.d/kvmfr.conf "
-KVMFR_DRACUT'
+if [ -f "/etc/modprobe.d/kvmfr.conf" ]; then
+    echo "Re-creating dummy kvmfr modprobe file"
+    sudo rm /etc/modprobe.d/kvmfr.conf
+fi
+sudo bash -c 'cat << KVMFR_MODPROBE > /etc/modprobe.d/kvmfr.conf
+# This is a dummy file and changing it does nothing
+# If you want to change the kvmfr static_size_mb
+# Run "rpm-ostree kargs --replace=kvmfr.static_size_mb=oldvalue=newvalue"
+# Default value set by us is 128 which is enough for 4k SDR
+# Find the current value by running "rpm-ostree kargs"
+KVMFR_MODPROBE'
 
-# Add kvmfr modprobe file following upstream documentation
-sudo bash -c "cat << KVMFR_MODPROBE > /etc/modprobe.d/kvmfr.conf
-options kvmfr static_size_mb=128
-KVMFR_MODPROBE"
+# Add kvmfr static size karg
+rpm-ostree kargs --append-if-missing="kvmfr.static_size_mb=128"
 
 # Add upstream udev rule for kvmfr, adjusted for fedora systems
 echo "Adding udev rule for /dev/kvmfr0"
@@ -64,6 +71,10 @@ fi
 if [ ! -d "$HOME/.config/selinux_te/pp" ]; then
     mkdir -p "$HOME/.config/selinux_te/pp"
 fi
+if [ -f "$HOME/.config/selinux_te/kvmfr.te" ]; then
+    echo "Re-creating kvmfr selinux type enforcement rules"
+    rm $HOME/.config/selinux_te/kvmfr.te
+fi
 bash -c "cat << KVMFR_SELINUX > $HOME/.config/selinux_te/kvmfr.te
 module kvmfr 1.0;
 require {
@@ -80,14 +91,16 @@ echo "This is the type enforcement we wrote for SELinux and you can find it in $
 echo "#======= start of kvmfr.te ======="
 cat "$HOME/.config/selinux_te/kvmfr.te"
 echo "#======== end of kvmfr.te ========"
+CONFIRM=$(Choose OK)
+
+# Convert .te a .pp file for semodule
 checkmodule -M -m -o "$HOME/.config/selinux_te/mod/kvmfr.mod" "$HOME/.config/selinux_te/kvmfr.te"
 semodule_package -o "$HOME/.config/selinux_te/pp/kvmfr.pp" -m "$HOME/.config/selinux_te/mod/kvmfr.mod"
 sudo semodule -i "$HOME/.config/selinux_te/pp/kvmfr.pp"
 
 # Load kvmfr module into currently booted system
 echo "Loading kvmfr module so you do not have to reboot to use it the first time"
 sudo modprobe kvmfr static_size_mb=128
-sudo chown $USER:qemu /dev/kvmfr0
 
 # Final message and regenerate initramfs so kvmfr loads next boot
 echo ""
@@ -100,9 +113,11 @@ echo "If you need to change it to a different value"
 echo "you can do that in /etc/modprobe.d/kvmfr.conf"
 echo "$(Urllink "https://looking-glass.io/docs/rc/ivshmem_kvmfr/#libvirt" "Please read official documentation for kvmfr for how to use it")"
 echo ""
-echo "Press OK to start the process of regenerating your initramfs, this will take a long time"
-echo "and there is no good way to track progress for it, if anything is wrong it will error out."
-echo "${b}NOTE: You can start using kvmfr right now without rebooting, but you will need to regenerate initramfs for it to auto load next boot.${n}"
+if [ -e "/dev/kvmfr0" ]; then
+    sudo chown $USER:qemu /dev/kvmfr0
+    echo "${b}NOTE: You can start using kvmfr right now without rebooting${n}"
+else
+    echo "Please reboot in order to fully load the kvmfr module."
+fi
 
 CONFIRM=$(Choose OK)
-rpm-ostree initramfs --enable