diff --git a/alioth/src/mem/mem.rs b/alioth/src/mem/mem.rs index 7295242..96f800d 100644 --- a/alioth/src/mem/mem.rs +++ b/alioth/src/mem/mem.rs @@ -67,8 +67,6 @@ pub enum Error { LockPoisoned, #[error("cannot allocate")] CanotAllocate, - #[error("cannot register MMIO notifier: {0}")] - Notifier(#[source] Box), #[error("{0}")] Hv(#[from] hv::Error), #[error("cannot handle action: {0:x?}")] diff --git a/alioth/src/virtio/dev/dev.rs b/alioth/src/virtio/dev/dev.rs index 4786a49..1df334e 100644 --- a/alioth/src/virtio/dev/dev.rs +++ b/alioth/src/virtio/dev/dev.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::fmt::Debug; +use std::os::fd::AsRawFd; use std::sync::atomic::{AtomicU16, AtomicU64, AtomicU8}; use std::sync::mpsc::{self, Receiver, Sender}; use std::sync::Arc; @@ -20,8 +21,10 @@ use std::thread::JoinHandle; use bitfield::bitfield; use mio::event::Event; -use mio::{Events, Poll, Registry, Token, Waker}; +use mio::unix::SourceFd; +use mio::{Events, Interest, Poll, Registry, Token, Waker}; +use crate::hv::{IoeventFd, IoeventFdRegistry}; use crate::mem::emulated::Mmio; use crate::mem::mapped::RamBus; use crate::mem::MemRegion; @@ -114,25 +117,28 @@ where } #[derive(Debug)] -pub struct VirtioDevice +pub struct VirtioDevice where D: Virtio, S: IrqSender, + E: IoeventFd, { pub name: Arc, pub device_config: Arc, pub reg: Arc, pub queue_regs: Arc>, + pub ioeventfds: Arc>, pub shared_mem_regions: Option>, pub waker: Arc, pub event_tx: Sender>, worker_handle: Option>, } -impl VirtioDevice +impl VirtioDevice where D: Virtio, S: IrqSender, + E: IoeventFd, { fn shutdown(&mut self) -> Result<(), Box> { let Some(handle) = self.worker_handle.take() else { @@ -146,7 +152,10 @@ where Ok(()) } - pub fn new(name: Arc, dev: D, memory: Arc) -> Result { + pub fn new(name: Arc, dev: D, memory: Arc, registry: &R) -> Result + where + R: IoeventFdRegistry, + { let poll = Poll::new()?; let device_config = dev.config(); let reg = Arc::new(Register { @@ -159,6 +168,18 @@ where ..Default::default() }); let queue_regs = Arc::new(queue_regs.collect::>()); + let ioeventfds = Arc::new( + (0..num_queues) + .map(|_| registry.create()) + .collect::, _>>()?, + ); + for (index, fd) in ioeventfds.iter().enumerate() { + poll.registry().register( + &mut SourceFd(&fd.as_fd().as_raw_fd()), + Token(TOKEN_IS_QUEUE as usize | index), + Interest::READABLE, + )?; + } let token = TOKEN_IS_QUEUE | TOKEN_WORKER_EVENT; let waker = Waker::new(poll.registry(), Token(token as usize))?; let shared_mem_regions = dev.shared_mem_regions(); @@ -186,6 +207,7 @@ where name, reg, queue_regs, + ioeventfds, worker_handle: Some(handle), event_tx, waker: Arc::new(waker), @@ -196,10 +218,11 @@ where } } -impl Drop for VirtioDevice +impl Drop for VirtioDevice where D: Virtio, S: IrqSender, + E: IoeventFd, { fn drop(&mut self) { if let Err(e) = self.shutdown() { diff --git a/alioth/src/virtio/pci.rs b/alioth/src/virtio/pci.rs index 4ba53c9..a7a26f6 100644 --- a/alioth/src/virtio/pci.rs +++ b/alioth/src/virtio/pci.rs @@ -23,9 +23,9 @@ use mio::Waker; use parking_lot::{Mutex, RwLock}; use zerocopy::{AsBytes, FromBytes, FromZeroes}; -use crate::hv::MsiSender; +use crate::hv::{IoeventFd, IoeventFdRegistry, MsiSender}; use crate::mem::emulated::Mmio; -use crate::mem::{MemRange, MemRegion, MemRegionEntry}; +use crate::mem::{MemRange, MemRegion, MemRegionCallback, MemRegionEntry}; use crate::pci::cap::{ MsixCap, MsixCapMmio, MsixCapOffset, MsixMsgCtrl, MsixTableEntry, MsixTableMmio, PciCap, PciCapHdr, PciCapId, PciCapList, @@ -422,10 +422,15 @@ where VirtioCommonCfg::LAYOUT_QUEUE_RESET => { todo!() } - (VirtioPciRegister::OFFSET_QUEUE_NOTIFY, _) => { - let event = WakeEvent::Notify { - q_index: val as u16, - }; + (offset, _) + if offset >= VirtioPciRegister::OFFSET_QUEUE_NOTIFY + && offset + < VirtioPciRegister::OFFSET_QUEUE_NOTIFY + + size_of::() * self.queues.len() => + { + let q_index = (offset - VirtioPciRegister::OFFSET_QUEUE_NOTIFY) as u16 / 4; + log::warn!("{}: notifying queue-{q_index} by vm exit!", self.name); + let event = WakeEvent::Notify { q_index }; self.wake_up_dev(event) } _ => { @@ -440,6 +445,38 @@ where } } +#[derive(Debug)] +struct IoeventFdCallback +where + R: IoeventFdRegistry, +{ + registry: R, + ioeventfds: Arc>, +} + +impl MemRegionCallback for IoeventFdCallback +where + R: IoeventFdRegistry, +{ + fn mapped(&self, addr: usize) -> mem::Result<()> { + for (q_index, fd) in self.ioeventfds.iter().enumerate() { + let base_addr = addr + (12 << 10) + VirtioPciRegister::OFFSET_QUEUE_NOTIFY; + let notify_addr = base_addr + q_index * size_of::(); + self.registry.register(fd, notify_addr, 0, None)?; + log::info!("q-{q_index} ioeventfd registered at {notify_addr:x}",) + } + Ok(()) + } + + fn unmapped(&self) -> mem::Result<()> { + for fd in self.ioeventfds.iter() { + self.registry.deregister(fd)?; + log::info!("ioeventfd {fd:?} de-registered") + } + Ok(()) + } +} + const VIRTIO_VENDOR_ID: u16 = 0x1af4; const VIRTIO_DEVICE_ID_BASE: u16 = 0x1040; @@ -514,22 +551,31 @@ impl PciCap for VirtioPciNotifyCap { } #[derive(Debug)] -pub struct VirtioPciDevice +pub struct VirtioPciDevice where D: Virtio, M: MsiSender, + E: IoeventFd, { - pub dev: VirtioDevice>, + pub dev: VirtioDevice, E>, pub config: Arc, pub registers: Arc>, } -impl VirtioPciDevice +impl VirtioPciDevice where M: MsiSender, D: Virtio, + E: IoeventFd, { - pub fn new(dev: VirtioDevice>, msi_sender: M) -> Result { + pub fn new( + dev: VirtioDevice, E>, + msi_sender: M, + ioeventfd_reg: R, + ) -> Result + where + R: IoeventFdRegistry, + { let (class, subclass) = get_class(D::device_id()); let mut header = DeviceHeader { common: CommonHeader { @@ -608,7 +654,7 @@ where length: (size_of::() * num_queues) as u32, ..Default::default() }, - multiplier: 0, // TODO use 4 for KVM_IOEVENTFD + multiplier: size_of::() as u32, }; let cap_device_config = VirtioPciCap { header: PciCapHdr { @@ -701,6 +747,10 @@ where bar0.ranges .push(MemRange::Span((12 << 10) - msix_table_size)); bar0.ranges.push(MemRange::Emulated(registers.clone())); + bar0.callbacks.lock().push(Box::new(IoeventFdCallback { + registry: ioeventfd_reg, + ioeventfds: dev.ioeventfds.clone(), + })); if device_config.size() > 0 { bar0.ranges.push(MemRange::Emulated(device_config)) } @@ -734,10 +784,11 @@ where } } -impl Pci for VirtioPciDevice +impl Pci for VirtioPciDevice where M: MsiSender, D: Virtio, + E: IoeventFd, { fn config(&self) -> Arc { self.config.clone() diff --git a/alioth/src/virtio/virtio.rs b/alioth/src/virtio/virtio.rs index 6d4720a..26afd1a 100644 --- a/alioth/src/virtio/virtio.rs +++ b/alioth/src/virtio/virtio.rs @@ -17,7 +17,7 @@ use std::fmt::Debug; use bitflags::bitflags; use thiserror::Error; -use crate::mem; +use crate::{hv, mem}; #[path = "dev/dev.rs"] pub mod dev; @@ -28,7 +28,7 @@ pub mod queue; #[derive(Debug, Error)] pub enum Error { #[error("hypervisor: {0}")] - Hv(#[source] Box), + Hv(#[from] hv::Error), #[error("IO: {0}")] Io(#[from] std::io::Error), diff --git a/alioth/src/vm.rs b/alioth/src/vm.rs index 62e21ef..79e9e3f 100644 --- a/alioth/src/vm.rs +++ b/alioth/src/vm.rs @@ -25,7 +25,7 @@ use crate::board::{self, ArchBoard, Board, BoardConfig, STATE_CREATED, STATE_RUN use crate::device::fw_cfg::{FwCfg, FwCfgItemParam, PORT_SELECTOR}; use crate::device::pvpanic::PvPanic; use crate::device::serial::Serial; -use crate::hv::{self, Hypervisor, Vm, VmConfig}; +use crate::hv::{self, Hypervisor, IoeventFdRegistry, Vm, VmConfig}; use crate::loader::{self, Payload}; use crate::mem::Memory; use crate::pci::bus::PciBus; @@ -152,16 +152,27 @@ where &mut self, name: String, param: P, - ) -> Result::Vm as Vm>::MsiSender>>, Error> + ) -> Result< + Arc< + VirtioPciDevice< + D, + <::Vm as Vm>::MsiSender, + <<::Vm as Vm>::IoeventFdRegistry as IoeventFdRegistry>::IoeventFd, + >, + >, + Error, + > where P: DevParam, D: Virtio, { let name = Arc::new(name); let dev = param.build(name.clone())?; - let virtio_dev = VirtioDevice::new(name.clone(), dev, self.board.memory.ram_bus().clone())?; + let registry = self.board.vm.create_ioeventfd_registry()?; + let virtio_dev = + VirtioDevice::new(name.clone(), dev, self.board.memory.ram_bus(), ®istry)?; let msi_sender = self.board.vm.create_msi_sender()?; - let dev = VirtioPciDevice::new(virtio_dev, msi_sender)?; + let dev = VirtioPciDevice::new(virtio_dev, msi_sender, registry)?; let dev = Arc::new(dev); let pci_dev = PciDevice::new(name.clone(), dev.clone()); self.add_pci_dev(pci_dev)?;