From 86771b1266dac22eae0300b8f2737196fa4e8f57 Mon Sep 17 00:00:00 2001 From: Alexandru-Cezar Sardan Date: Wed, 5 Jan 2022 16:08:11 +0000 Subject: [PATCH 1/3] interrupt: Add traits for interrupt abstractions. Add a series of Traits and structs for defining the interface for interrupts. The goal is to allow individual development of device crates that can work with different types of interrupt mechanisms. The interrupt mechanism can be implemented in the VMM based on the platform or the available hardware (e.g. CPU, Interrupt controller). These set of traits allow devices to use a simple interface through which they can request, release, configure interrupts and also signal events to the VMM. Signed-off-by: Alexandru-Cezar Sardan --- src/interrupt/legacy.rs | 39 ++++++ src/interrupt/mod.rs | 289 ++++++++++++++++++++++++++++++++++++++++ src/interrupt/msi.rs | 32 +++++ src/lib.rs | 1 + 4 files changed, 361 insertions(+) create mode 100644 src/interrupt/legacy.rs create mode 100644 src/interrupt/mod.rs create mode 100644 src/interrupt/msi.rs diff --git a/src/interrupt/legacy.rs b/src/interrupt/legacy.rs new file mode 100644 index 0000000..62df572 --- /dev/null +++ b/src/interrupt/legacy.rs @@ -0,0 +1,39 @@ +// Copyright (C) 2021 Amazon.com, Inc. or its affiliates. +// All Rights Reserved. + +// SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause + +//! Traits and Structs to manage legacy interrupt sources for devices. +//! +//! Legacy interrupt sources typically include pin based interrupt lines. + +use crate::interrupt::ConfigurableInterrupt; + +/// Definition for PCI INTx pins. +#[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd)] +pub enum IntXPin { + /// INTA + IntA = 0x1, + /// INTB + IntB = 0x2, + /// INTC + IntC = 0x3, + /// INTD + IntD = 0x4, +} + +/// Standard configuration for Legacy interrupts. +#[derive(Copy, Clone, Debug, Default, Eq, PartialEq)] +pub struct LegacyIrqConfig { + /// Input of the system interrupt controllers the device's interrupt pin is connected to. + /// Implemented by any device that makes use of an interrupt pin. + pub interrupt_line: Option, + /// Specifies which interrupt pin the device uses. + pub interrupt_pin: Option, +} + +/// Trait for defining properties of Legacy interrupts. +pub trait LegacyInterrupt: ConfigurableInterrupt {} + +/// Blanket implementation for Interrupts that use a LegacyIrqConfig. +impl LegacyInterrupt for T where T: ConfigurableInterrupt {} diff --git a/src/interrupt/mod.rs b/src/interrupt/mod.rs new file mode 100644 index 0000000..5f98a9a --- /dev/null +++ b/src/interrupt/mod.rs @@ -0,0 +1,289 @@ +// Copyright (C) 2019-2020 Alibaba Cloud, Red Hat, Inc and Amazon.com, Inc. or its affiliates. +// All Rights Reserved. + +// SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause + +//! Traits and Structs to manage interrupt sources for devices. +//! +//! In system programming, an interrupt is a signal to the processor emitted by hardware or +//! software indicating an event that needs immediate attention. An interrupt alerts the processor +//! to a high-priority condition requiring the interruption of the current code the processor is +//! executing. The processor responds by suspending its current activities, saving its state, and +//! executing a function called an interrupt handler (or an interrupt service routine, ISR) to deal +//! with the event. This interruption is temporary, and, after the interrupt handler finishes, +//! unless handling the interrupt has emitted a fatal error, the processor resumes normal +//! activities. +//! +//! Hardware interrupts are used by devices to communicate that they require attention from the +//! operating system, or a bare-metal program running on the CPU if there are no OSes. The act of +//! initiating a hardware interrupt is referred to as an interrupt request (IRQ). Different devices +//! are usually associated with different interrupts using a unique value associated with each +//! interrupt. This makes it possible to know which hardware device caused which interrupts. +//! These interrupt values are often called IRQ lines, or just interrupt lines. +//! +//! Nowadays, IRQ lines is not the only mechanism to deliver device interrupts to processors. +//! MSI [(Message Signaled Interrupt)](https://en.wikipedia.org/wiki/Message_Signaled_Interrupts) +//! is another commonly used alternative in-band method of signaling an interrupt, using special +//! in-band messages to replace traditional out-of-band assertion of dedicated interrupt lines. +//! While more complex to implement in a device, message signaled interrupts have some significant +//! advantages over pin-based out-of-band interrupt signaling. Message signaled interrupts are +//! supported in PCI bus since its version 2.2, and in later available PCI Express bus. Some non-PCI +//! architectures also use message signaled interrupts. +//! +//! While IRQ is a term commonly used by Operating Systems when dealing with hardware +//! interrupts, the IRQ numbers managed by OSes are independent of the ones managed by VMM. +//! For simplicity sake, the term `Interrupt Source` is used instead of IRQ to represent both pin-based +//! interrupts and MSI interrupts. + +pub mod legacy; +pub mod msi; + +use std::fmt::{self, Display}; +use std::ops::Deref; +use std::sync::Arc; + +/// Errors associated with handling interrupts +#[derive(Debug)] +pub enum Error { + /// Operation not supported for this interrupt. + OperationNotSupported, + + /// The specified configuration is not valid. + InvalidConfiguration, + + /// The interrupt state was not changed. + InterruptNotChanged, + + /// The interrupt could not be triggered, asserted or de-asserted. + InterruptNotTriggered, + + /// An error occured during interrupt allocation. + InterruptAllocationError, + + /// An error occured during interrupt release. + InterruptFreeError, +} + +impl std::error::Error for Error {} + +/// Reuse std::io::Result to simplify interoperability among crates. +pub type Result = std::result::Result; + +impl Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Interrupt error: ")?; + match self { + Error::OperationNotSupported => write!(f, "operation not supported"), + Error::InvalidConfiguration => write!(f, "invalid configuration"), + Error::InterruptNotChanged => write!(f, "the interrupt state could not be changed"), + Error::InterruptNotTriggered => write!(f, "the interrupt could not be triggered"), + Error::InterruptAllocationError => write!(f, "the interrupt could not be allocated"), + Error::InterruptFreeError => write!(f, "the interrupt could not be released"), + } + } +} + +/// Trait used by interrupt producers to emulate an edge triggered interrupt. +/// +/// This trait allows for a device to signal an interrupt event to the guest VM. +/// These events are sampled by the interrupt controller synchronously when `trigger()` is called. +/// +/// Edge triggered interrupts cannot be shared. +pub trait EdgeInterrupt { + /// Signal an interrupt to the guest VM. + fn trigger(&self) -> Result<()>; +} + +/// Trait used by interrupt producers to emulate a level triggered interrupt. +/// +/// This trait allows for a device to assert an interrupt line as long as it needs service +/// from the CPU. +/// A level triggered interrupt is held asserted until the device clears the interrupt signal. +/// +/// Assertion of the interrupt signal starts when `assert()` is called by the device and ends when +/// `clear()` is called by the device. +/// The object implementing this trait must hold the assertion state internally and should return +/// from `assert()` and `clear()` once the state is changed. +pub trait LevelInterrupt { + /// Assert the interrupt line to signal an interrupt to the guest VM. + /// This method sets the interrupt in an asserted state. + fn assert(&self) -> Result<()>; + + /// Deassert the interrupt line to signal that the device no longer requires service. + fn clear(&self) -> Result<()>; +} + +/// Trait that allows access to a device interrupt status. +/// +/// A device will implement this trait if it wants to allow other components to check its +/// interrupt status. +/// This allows implementation of auto-triggered shared level interrupts that poll the interrupt +/// state from the device in order to re-trigger the interrupt when resampled. +pub trait InterruptStatusChecker { + /// Check if the device requires service. + /// Returns `true` if the device has not deasserted the interrupt line and still + /// requires service. + fn is_active(&self) -> bool; +} + +/// Trait used by interrupt controllers to configure interrupts. +/// +/// An object having the `Interrupt` trait is shared between the VMM (which typically implements +/// the interrupt mechanisms) and interrupt control components. +/// It offers a control interface through the `enable()` and `disable()` methods that allow an +/// interrupt to be registered with the interrupt controllers or mechanisms in the VMM. +/// +/// Objects implementing this trait are required to have internal mutability. +pub trait Interrupt { + /// Enable generation of interrupts on this line. + fn enable(&self) -> Result<()> { + Err(Error::OperationNotSupported) + } + + /// Disable generation of interrupts on this line. + fn disable(&self) -> Result<()> { + Err(Error::OperationNotSupported) + } +} + +/// Trait that allows interrupt controllers to configure interrupt parameters. +/// +/// This enhances the control plane interface of the `Interrupt` by allowing a device to configure +/// the behavior of the interrupt. +/// +/// Objects implementing this trait are required to have internal mutability. +pub trait ConfigurableInterrupt: Interrupt { + /// Type describing the configuration spec of the interrupt. + type Cfg; + + /// Update configuration of the interrupt. + fn update(&self, config: &Self::Cfg) -> Result<()>; + + /// Returns the current configuration of the interrupt. + fn get_config(&self) -> Result; +} + +/// Trait for interrupts that can be masked or unmasked. +/// +/// Objects implementing this trait are required to have internal mutability. +pub trait MaskableInterrupt: Interrupt { + /// Mask the interrupt. Masked interrupts are remembered but + /// not delivered. + fn mask(&self) -> Result<()>; + + /// Unmask the interrupt, delivering it if it was pending. + fn unmask(&self) -> Result<()>; +} + +/// Trait for interrupts that can be auto-retriggered when resampled. +/// +/// In some implementations of shared level-triggered interrupts the interrupt can be resampled +/// as a result of different events (e.g. an EOI) before a device explicitly deasserts the +/// interrupt. If the device still requires service, the interrupt should be reasserted. +/// +/// This trait allows implementation of the interrupt mechanism described above. +/// It requires that the user of this trait configures an `InterruptStatusChecker`. +/// When the interrupt is resampled, the state of the device will be checked by the implementation +/// of this trait. If the device still requires service, the interrupt is reasserted. +/// +/// An example of such mechanism is provided by KVM when using KVM_IRQFD with +/// KVM_CAP_IRQFD_RESAMPLE. +/// +/// Objects implementing this trait are required to have internal mutability. +pub trait AutoRetriggerInterrupt: Interrupt { + /// Set the `InterruptStatusChecker` object through which the interrupt can poll the device + /// interrupt status. + fn set_status_checker(&self, status_checker: Arc) -> Result<()>; +} + +/// Trait that provides access to the underlying trigger notification object used by the hypervisor. +/// +/// The type of the underlying notification mechanism used by the interrupt is defined by the +/// `NotifierType` associated type. +/// This enables some use cases where the device may want to bypass the VMM completely or when the +/// device crate acts only as a control plane and the actual emulation is implemented in some other +/// component that understands the underlying mechanism. +/// +/// The usage of the resulted notifier object is speciffic to the hypervisor but the semantics of +/// the object returned by the `trigger_notifier()` method should follow the semantics from +/// `EdgeInterrupt::trigger()` or `LevelInterrupt::assert()` (e.g. when the user changes the state +/// of the notifier object, an interrupt is queued for the guest). +/// +/// A notable example is VFIO that allows a device to register the irqfd so that interrupts follow +/// a fast path that doesn't require going through the VMM. Another example is XEN evtchn. +/// +/// Implementations of this trait must provide the trigger notifier object. +pub trait AsRefTriggerNotifier { + /// The type of the underlying mechanism used for trigger notifications by this interrupt. + type NotifierType; + + /// Returns a reference to a trigger notifier from this interrupt. + /// + /// An interrupt notifier allows for external components and processes to inject interrupts + /// into a guest through a different interface other than `EdgeInterrupt::trigger()`. + fn trigger_notifier(&self) -> &Self::NotifierType; +} + +/// Trait that provides access to the underlying resample notification object used by +/// the hypervisor. +/// +/// This enables use cases where the notification that the interrupt was resampled is +/// handled by a component that understands the underlying hypervisor interrupt implementation +/// and wants to bypass the VMM. +/// +/// The semantics of the object returned by `resample_notifier()` are similar to those of +/// `AutoRetriggerInterrupt` (when the state of the notifier object changes it means that +/// the interrupt was resampled and the device should reassert the interrupt). +/// +/// VFIO supports the registration of a `resamplefd` which would be returned by +/// `resample_notifier`. +/// +/// Implementations of this trait must provide the resample notifier object. +pub trait AsRefResampleNotifier { + /// The type of the underlying mechanism used for resample notifications by an interrupt. + type NotifierType; + + /// Returns a reference to a resample notifier from an interrupt. + /// + /// An end-of-interrupt notifier allows for external components and processes to be notified + /// when a guest acknowledges an interrupt. This can be used to resample and inject a + /// level-triggered interrupt, or to mitigate the effect of lost timer interrupts. + fn resample_notifier(&self) -> &Self::NotifierType; +} + +/// Trait to manage a group of interrupt sources for a device. +/// +/// A device may use an InterruptSourceGroup to manage multiple interrupts of the same type. +/// The group allows a device to request and release interrupts and perform actions on the +/// whole collection of interrupts like enable and disable for cases where enabling or disabling +/// a single interrupt in the group does not make sense. For example, PCI MSI interrupts must be +/// enabled as a group. +pub trait InterruptSourceGroup: Send { + /// Type of the interrupts contained in this group. + type InterruptType: Interrupt; + + /// Interrupt Type returned by get + type InterruptWrapper: Deref; + + /// Return whether the group manages no interrupts. + fn is_empty(&self) -> bool; + + /// Get number of interrupt sources managed by the group. + fn len(&self) -> usize; + + /// Enable the interrupt sources in the group to generate interrupts. + fn enable(&self) -> Result<()>; + + /// Disable the interrupt sources in the group to generate interrupts. + fn disable(&self) -> Result<()>; + + /// Return the index-th interrupt in the group, or `None` if the index is out + /// of bounds. + fn get(&self, index: usize) -> Option; + + /// Request new interrupts within this group. + fn allocate_interrupts(&mut self, size: usize) -> Result<()>; + + /// Release all interrupts within this group. + fn free_interrupts(&mut self) -> Result<()>; +} diff --git a/src/interrupt/msi.rs b/src/interrupt/msi.rs new file mode 100644 index 0000000..1b1913f --- /dev/null +++ b/src/interrupt/msi.rs @@ -0,0 +1,32 @@ +// Copyright (C) 2021 Amazon.com, Inc. or its affiliates. +// All Rights Reserved. + +// SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause + +//! Traits and Structs to manage MSI interrupt sources for devices. +//! +//! MSI interrupts are typically used by PCI devices. +//! These structs and traits can be used to configure both MSI and MSIX interrupts. + +use crate::interrupt::{ConfigurableInterrupt, MaskableInterrupt}; + +/// Configuration data for MSI/MSI-X interrupts. +/// +/// On x86 platforms, these interrupts are vectors delivered directly to the LAPIC. +#[derive(Copy, Clone, Debug, Default, Eq, PartialEq)] +pub struct MsiIrqConfig { + /// High address to delivery message signaled interrupt. + pub high_addr: u32, + /// Low address to delivery message signaled interrupt. + pub low_addr: u32, + /// Data to write to delivery message signaled interrupt. + pub data: u32, + /// Unique ID of the device to delivery message signaled interrupt. + pub devid: u32, +} + +/// Trait for defining properties of MSI interrupts. +pub trait MsiInterrupt: ConfigurableInterrupt + MaskableInterrupt {} + +/// Blanket implementation for Interrupts that use a MsiIrqConfig. +impl MsiInterrupt for T where T: ConfigurableInterrupt + MaskableInterrupt {} diff --git a/src/lib.rs b/src/lib.rs index 70e2451..58f3c29 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -52,6 +52,7 @@ pub mod bus; pub mod device_manager; +pub mod interrupt; pub mod resources; use std::ops::Deref; From b2c18707307ad67f08066d2da584fdd598bf8902 Mon Sep 17 00:00:00 2001 From: Alexandru-Cezar Sardan Date: Mon, 10 Jan 2022 16:50:28 +0200 Subject: [PATCH 2/3] README.md: Add design for interrupt abstractions Add a brief description of the design of the Interrupt abstractions. Signed-off-by: Alexandru-Cezar Sardan --- README.md | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/README.md b/README.md index 69c9889..33f3de6 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ address range, IRQ number, etc) ## Design +### I/O + The virtual device model is built around four traits, `DevicePio` and `MutDevicePio` for [Programmed I/O](https://en.wikipedia.org/wiki/Programmed_input%E2%80%93output) @@ -42,8 +44,40 @@ that there’s a virtual device registered on the bus for the requested address, and finally sends the request to that device. ![vm-device](https://user-images.githubusercontent.com/241037/143853115-b1526028-6836-4845-a311-71cf989c60ef.png) +### Interrupts + +Interrupt configuration is built around the `Interrupt` and `InterruptSourceGroup` +traits. These traits allow device control code that is developed in separate +crates from the VMM (which typically implements the interrupt mechanisms) to +configure interrupt delivery to the guest VM without having a dependency on the +implementation of the interrupt mechanism. +The `EdgeInterrupt` and `LevelInterrupt` are traits that provide interrupt +assertion mechanisms and can be used by device implementations directly in order +to inject interrupts in the guest VM. + +The `Interrupt` trait provides methods that are used by devices to enable and +disable interrupt generation by registering the interrupt with the VMM. Access +to additional interrupt properties is defined in new super-traits. +`ConfigurableInterrupt` allows for devices to send or receive interrupt +configuration parameters to/from the implementation inside the VMM. This is useful +when devices need to specify custom data that the VMM will use when delivering the +interrupt (e.g. MSI device id, PCI INTx pin etc). +`MaskableInterrupt` is also defined as a super-trait for use with interrupts that +can be masked/unmasked. + +An `InterruptSourceGroup` stores a collection of interrupts of the same type. It +is the interface through which a device may request or release interrupts and +perform group related actions like enabling or disabling all interrupts at once. +Each device that generates interrupts can be assigned one or more +`InterruptSourceGroup`s (depending on the types of interrupts it uses or logical +grouping). The following diagram depicts the interaction between the components +that use the interrupt interface: + +![vm-device-interrupts](https://user-images.githubusercontent.com/86006646/148783015-fea49a7c-cff8-4ec7-8766-00b0baed41c5.png) + ## Usage +### I/O A device is usually attached to a particular bus and thus needs to implement a trait of only one type. For example, serial port on x86 is a PIO device, while VirtIO devices use MMIO. It’s also possible for a device to implement both. Once @@ -67,6 +101,44 @@ address range to the device. The requests are dispatched by the client code, for example when handling VM exits, using `IoManager`'s methods `pio_read`, `pio_write`, `mmio_read` and `mmio_write`. +### Interrupts + +To allow configuration of interrupts, a VMM must implement the `Interrupt` and +`InterruptSourceGroup` traits for the interrupt mechanisms that the device +requires. Implementation is machine or VMM specific and may depend on the types +and number of IRQ chips that the machine has or interrupt delivery mechanisms +(e.g. `EventFd`s). +The device interrupt configuration code generally does not concern itself with +the actual implementation of the interrupts and will be initialized with one or +more `InterruptSourceGroup`s by the VMM. + +In order to allow devices to inject interrupt lines in the guest, a VMM must +also implement either the `EdgeInterrupt` or `LevelInterrupt` traits, depending +on the type of interrupt assertion mechanism that is supported. Devices use +objects that have these traits to signal an interrupt to the guest. + +The device configuration code may define constraints for the types of interrupts +that it needs by combining the supertraits defined into trait bounds (e.g. if it +needs a `ConfigurableInterrupt` that can receive the `LegacyIrqConfig` +configuration struct). MSI and legacy interrupt traits are added in this crate +for ease of use. Configuration of these interrupt types are standardised. +`MsiInterrupt` can also be used for MSI-X interrupts. These traits only define +the configuration bounds for these interrupt types. `EdgeInterrupt` or +`LevelInterrupt` will still need to be implemented in order to allow devices +to use these interrupts. MSI interrupts are considered edge-triggered while +`LegacyInterrupt` can be either edge-triggered (typically in the case of ISA +interrupts) or level triggered (in the case of INTx interrupts). + +In order to have access to the underlying notification mechanisms used by the +hypervisor, the device configuration may use the `AsRefTriggerNotifier` and the +`AsRefResampleNotifier` conversion traits and specify the `NotifierType` associated +type. This type defines the interrupt delivery mechanism and is specific to the +Hypervisor (e.g. KVM irqfd, Xen evtchn etc). + +One example of this requirement is the development of a VFIO device. Since VFIO +can trigger a KVM irqfd directly, the VFIO device would need to get access to the +underlying irqfd in order to register it with VFIO. + ## Examples ### Implementing a simple log PIO device @@ -119,3 +191,4 @@ This project is licensed under either of: - [Apache License](http://www.apache.org/licenses/LICENSE-2.0), Version 2.0 - [BSD-3-Clause License](https://opensource.org/licenses/BSD-3-Clause) + From be03951ff5290af8fe6760aa14e9440f1b3192c5 Mon Sep 17 00:00:00 2001 From: Alexandru-Cezar Sardan Date: Tue, 8 Feb 2022 17:40:09 +0000 Subject: [PATCH 3/3] Update coverage Interrupt abstractions currently don't have unit tests so coverage is decreased. Signed-off-by: Alexandru-Cezar Sardan --- coverage_config_x86_64.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coverage_config_x86_64.json b/coverage_config_x86_64.json index d0bb900..4526313 100644 --- a/coverage_config_x86_64.json +++ b/coverage_config_x86_64.json @@ -1,5 +1,5 @@ { - "coverage_score": 91.4, + "coverage_score": 87.3, "exclude_path": "", "crate_features": "" }