sriov: add support as special PCI devices

Add support to report SRIOV devices.
Differently from GPU devices, we model SRIOV devices as special
PCI devices, extending the `pci` package instead of introducing
a new top-level package.
This design emerged during the review of a previous proposal:
9058f61#r755312597

SRIOV devices are either Physical Functions or Virtual functions.
The preferred representation for ghw is Physical Functions, whose
dependent devices will be Virtual Functions; however, for the sake of
practicality, the API also exposes soft references to Virtual Functions,
so consumers of the API can access them directly and not navigating
the parent devices.

This patch also adds support in `ghwc`, to report the sriov information,
and in the `snapshot` package, to make sure to capture all the files
in sysfs that ghw cares about.

Last but not least, lacking access to suitable non-linux systems,
support is provided only on linux OS, even though the API tries hard
not to be linux-specific.

Resolves: #92
Signed-off-by: Francesco Romani <[email protected]>

README.md

@@ -1115,6 +1115,39 @@ information
 `ghw.TopologyNode` struct if you'd like to dig deeper into the NUMA/topology
 subsystem
 
+### SRIOV
+
+SRIOV (Single-Root Input/Output Virtualization) is a class of PCI devices that ghw models explicitly.
+
+```go
+package main
+
+import (
+	"fmt"
+
+	"github.com/jaypipes/ghw"
+)
+
+func main() {
+	pci, err := ghw.PCI()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error getting SRIOV info through PCI: %v", err)
+	}
+
+	fmt.Printf("%v\n", pci)
+
+	for _, dev := range pci.Functions {
+		fmt.Printf(" %v\n", dev)
+	}
+}
+```
+
+ghw discovers the SRIOV devices scanning PCI devices. Thus, you need to make sure to have scanned the PCI devices before
+to query for SRIOV devices (aka "Functions", "PCI Functions").
+Virtual Functions (VFs) are handled by Physical Functions (PFs).
+Virtual Functions are available both as entries in the `pci.Functions` slice and as properties of their parent Physical Functions.
+Both references are aliases to the same object.
+
 ### Chassis
 
 The host's chassis information is accessible with the `ghw.Chassis()` function.  This

cmd/ghwc/commands/sriov.go

@@ -0,0 +1,35 @@
+//
+// Use and distribution licensed under the Apache license version 2.
+//
+// See the COPYING file in the root project directory for full text.
+//
+
+package commands
+
+import (
+	"github.com/jaypipes/ghw"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+)
+
+// sriovCmd represents the install command
+var sriovCmd = &cobra.Command{
+	Use:   "sriov",
+	Short: "Show Single Root I/O Virtualization device information for the host system",
+	RunE:  showSRIOV,
+}
+
+// showSRIOV show SRIOV information for the host system.
+func showSRIOV(cmd *cobra.Command, args []string) error {
+	info, err := ghw.PCI()
+	if err != nil {
+		return errors.Wrap(err, "error getting SRIOV info through PCI")
+	}
+
+	printInfo(info.DescribeFunctions())
+	return nil
+}
+
+func init() {
+	rootCmd.AddCommand(sriovCmd)
+}

pkg/pci/function.go

@@ -0,0 +1,109 @@
+//
+// Use and distribution licensed under the Apache license version 2.
+//
+// See the COPYING file in the root project directory for full text.
+//
+
+package pci
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/jaypipes/ghw/pkg/context"
+	"github.com/jaypipes/ghw/pkg/marshal"
+)
+
+// Function describes an SR-IOV physical or virtual function. Physical functions
+// will have no Parent Function struct pointer and will have one or more Function
+// structs in the Functions field.
+type Function struct {
+	Device // All Functions are PCI Devices
+	// Parent contains a pointer to the parent physical function.
+	// Will be empty when this is a physical function
+	Parent *Function `json:"parent,omitempty"`
+	// MaxVFs contains the maximum number of supported virtual
+	// functions for this physical function
+	MaxVFs int `json:"max_vfs"`
+	// Functions contains the physical function's virtual functions
+	Functions []*Function `json:"functions"`
+}
+
+// IsPhysical returns true if the PCIe function is a physical function, false
+// if it is a virtual function
+func (f *Function) IsPhysical() bool {
+	return f.Parent == nil
+}
+
+type fnMarshallable struct {
+	Driver        string   `json:"driver"`
+	Address       string   `json:"address"`
+	Vendor        devIdent `json:"vendor"`
+	Product       devIdent `json:"product"`
+	Type          string   `json:"type"`
+	ParentAddress string   `json:"parent_address,omitempty"`
+	MaxVFs        int      `json:"virtual_functions_max,omitempty"`
+	Functions     []string `json:"virtual_functions,omitempty"`
+}
+
+func (f *Function) MarshalJSON() ([]byte, error) {
+	fm := fnMarshallable{
+		Driver:  f.Driver,
+		Address: f.Address,
+		Vendor: devIdent{
+			ID:   f.Vendor.ID,
+			Name: f.Vendor.Name,
+		},
+		Product: devIdent{
+			ID:   f.Product.ID,
+			Name: f.Product.Name,
+		},
+		MaxVFs: f.MaxVFs,
+		Type:   "virtual",
+	}
+	if f.IsPhysical() {
+		fm.Type = "physical"
+	}
+	if f.Parent != nil {
+		fm.ParentAddress = f.Parent.Address
+	}
+	var fns []string
+	for _, fn := range f.Functions {
+		fns = append(fns, fn.Address)
+	}
+	fm.Functions = fns
+	return json.Marshal(fm)
+}
+
+type FunctionsPrinter struct {
+	ctx       *context.Context
+	Functions []*Function `json:"functions"`
+}
+
+func (info *Info) DescribeFunctions() *FunctionsPrinter {
+	// this is to avoid ugly output like `{"functions":null}`
+	functions := info.Functions
+	if functions == nil {
+		functions = []*Function{}
+	}
+	return &FunctionsPrinter{
+		ctx:       info.ctx,
+		Functions: functions,
+	}
+}
+
+func (fp *FunctionsPrinter) String() string {
+	return fmt.Sprintf("SRIOV (%d devices)", len(fp.Functions))
+}
+
+// YAMLString returns a string with the SRIOV information formatted as YAML
+// under a top-level "functions:" key
+func (fp *FunctionsPrinter) YAMLString() string {
+	return marshal.SafeYAML(fp.ctx, fp)
+}
+
+// JSONString returns a string with the PCI information formatted as JSON
+// under a top-level "pci:" key
+func (fp *FunctionsPrinter) JSONString(indent bool) string {
+	return marshal.SafeJSON(fp.ctx, fp, indent)
+}

pkg/pci/function_linux.go

@@ -0,0 +1,87 @@
+// Use and distribution licensed under the Apache license version 2.
+//
+// See the COPYING file in the root project directory for full text.
+//
+
+package pci
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/jaypipes/ghw/pkg/linuxpath"
+	"github.com/jaypipes/ghw/pkg/util"
+)
+
+func (info *Info) ListFunctions() []*Function {
+	var fns []*Function
+	for _, dev := range info.Devices {
+		pf := info.physicalFunctionFromDevice(dev)
+		if pf == nil {
+			// not a physical function, nothing to do
+			continue
+		}
+		fns = append(fns, pf)
+		fns = append(fns, pf.Functions...)
+	}
+	return fns
+}
+
+func (info *Info) physicalFunctionFromDevice(dev *Device) *Function {
+	paths := linuxpath.New(info.ctx)
+	devPath := filepath.Join(paths.SysBusPciDevices, dev.Address)
+
+	buf, err := ioutil.ReadFile(filepath.Join(devPath, "sriov_totalvfs"))
+	if err != nil {
+		// is not a physfn. Since we will fill virtfn from physfn, we can give up now
+		return nil
+	}
+
+	maxVFs, err := strconv.Atoi(strings.TrimSpace(string(buf)))
+	if err != nil {
+		info.ctx.Warn("error reading sriov_totalvfn for %q: %v", devPath, err)
+		return nil
+	}
+
+	fn := &Function{
+		Device: *dev,
+		MaxVFs: maxVFs,
+	}
+	fn.Functions = virtualFunctionsFromPhysicalFunction(info, fn, devPath)
+	return fn
+}
+
+func virtualFunctionsFromPhysicalFunction(info *Info, parentFn *Function, parentPath string) []*Function {
+	numVfs := util.SafeIntFromFile(info.ctx, filepath.Join(parentPath, "sriov_numvfs"))
+	if numVfs == -1 {
+		return nil
+	}
+
+	var vfs []*Function
+	for vfnIdx := 0; vfnIdx < numVfs; vfnIdx++ {
+		virtFn := fmt.Sprintf("virtfn%d", vfnIdx)
+		vfnDest, err := os.Readlink(filepath.Join(parentPath, virtFn))
+		if err != nil {
+			info.ctx.Warn("error reading backing device for virtfn %q physfn %q: %v", virtFn, parentPath, err)
+			return nil
+		}
+
+		vfnAddr := filepath.Base(vfnDest)
+		vfnDev := info.GetDevice(vfnAddr)
+		if vfnDev == nil {
+			info.ctx.Warn("error finding the PCI device for virtfn %s physfn %s", vfnAddr, parentFn.Address)
+			return nil
+		}
+
+		// functions must be ordered by their index
+		vfs = append(vfs, &Function{
+			Device: *vfnDev,
+			Parent: parentFn,
+		})
+	}
+	return vfs
+}

pkg/pci/function_stub.go

@@ -0,0 +1,13 @@
+//go:build !linux
+// +build !linux
+
+// Use and distribution licensed under the Apache license version 2.
+//
+// See the COPYING file in the root project directory for full text.
+//
+
+package pci
+
+func (info *Info) ListFunctions() []*Function {
+	return nil
+}

pkg/pci/pci.go

@@ -127,6 +127,8 @@ type Info struct {
 	ctx  *context.Context
 	// All PCI devices on the host system
 	Devices []*Device
+	// All SRIOV devices on the host system (subset of Devices)
+	Functions []*Function `json:"-"`
 	// hash of class ID -> class information
 	// DEPRECATED. Will be removed in v1.0. Please use
 	// github.com/jaypipes/pcidb to explore PCIDB information

pkg/pci/pci_linux.go

@@ -48,6 +48,7 @@ func (i *Info) load() error {
 	i.Vendors = db.Vendors
 	i.Products = db.Products
 	i.Devices = i.ListDevices()
+	i.Functions = i.ListFunctions()
 	return nil
 }
 

pkg/pci/pci_stub.go

@@ -15,7 +15,7 @@ import (
 )
 
 func (i *Info) load() error {
-	return errors.New("pciFillInfo not implemented on " + runtime.GOOS)
+	return errors.New("pci load() not implemented on " + runtime.GOOS)
 }
 
 // GetDevice returns a pointer to a Device struct that describes the PCI

pkg/snapshot/clonetree.go

@@ -100,7 +100,7 @@ func CopyFilesInto(fileSpecs []string, destDir string, opts *CopyFileOptions) er
 	if opts == nil {
 		opts = &CopyFileOptions{
 			IsSymlinkFn:       isSymlink,
-			ShouldCreateDirFn: isDriversDir,
+			ShouldCreateDirFn: shouldCreateDir,
 		}
 	}
 	for _, fileSpec := range fileSpecs {
@@ -157,6 +157,13 @@ func copyFileTreeInto(paths []string, destDir string, opts *CopyFileOptions) err
 	return nil
 }
 
+func shouldCreateDir(path string, fi os.FileInfo) bool {
+	if isDeviceNetworkDir(path, fi) {
+		return true
+	}
+	return isDriversDir(path, fi)
+}
+
 func isSymlink(path string, fi os.FileInfo) bool {
 	return fi.Mode()&os.ModeSymlink != 0
 }
@@ -165,6 +172,12 @@ func isDriversDir(path string, fi os.FileInfo) bool {
 	return strings.Contains(path, "drivers")
 }
 
+func isDeviceNetworkDir(path string, fi os.FileInfo) bool {
+	parentDir := filepath.Base(filepath.Dir(path))
+	// TODO: the "HasPrefix" check is brutal, but should work on linux
+	return parentDir == "net" && strings.HasPrefix(path, "/sys/devices")
+}
+
 func copyLink(path, targetPath string) error {
 	target, err := os.Readlink(path)
 	if err != nil {

pkg/snapshot/clonetree_pci_linux.go

@@ -70,6 +70,19 @@ func scanPCIDeviceRoot(root string) (fileSpecs []string, pciRoots []string) {
 		"revision",
 		"vendor",
 	}
+
+	perDevEntriesOpt := []string{
+		"driver",
+		"net/*",
+		"physfn",
+		"sriov_*",
+		"virtfn*",
+	}
+
+	ignoreSet := map[string]bool{
+		"sriov_vf_msix_count": true, // linux >= 5.14, write-only
+	}
+
 	entries, err := ioutil.ReadDir(root)
 	if err != nil {
 		return []string{}, []string{}
@@ -96,6 +109,25 @@ func scanPCIDeviceRoot(root string) (fileSpecs []string, pciRoots []string) {
 			fileSpecs = append(fileSpecs, filepath.Join(pciEntry, perNetEntry))
 		}
 
+		for _, perNetEntryOpt := range perDevEntriesOpt {
+			netEntryOptPath := filepath.Join(pciEntry, perNetEntryOpt)
+
+			items, err := filepath.Glob(netEntryOptPath)
+			if err != nil {
+				// TODO: we skip silently because we don't have
+				// a ctx handy, so we can't do ctx.Warn :\
+				continue
+			}
+
+			for _, item := range items {
+				globbedEntry := filepath.Base(item)
+				if _, ok := ignoreSet[globbedEntry]; ok {
+					continue
+				}
+				fileSpecs = append(fileSpecs, item)
+			}
+		}
+
 		if isPCIBridge(entryPath) {
 			trace("adding new PCI root %q\n", entryName)
 			pciRoots = append(pciRoots, pciEntry)