diff --git a/README.md b/README.md index b0c742d2..93c47fdd 100644 --- a/README.md +++ b/README.md @@ -1115,6 +1115,64 @@ information `ghw.TopologyNode` struct if you'd like to dig deeper into the NUMA/topology subsystem +### SRIOV + +SRIOV (Single-Root Input/Output Virtualization) is a class of PCI devices that ghw models explicitly, like gpus. + +```go +package main + +import ( + "fmt" + + "github.com/jaypipes/ghw" +) + +func main() { + sriov, err := ghw.SRIOV() + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting SRIOV info: %v", err) + } + + fmt.Printf("%v\n", sriov) + + for _, dev := range sriov.PhysicalFunctions { + fmt.Printf(" %v\n", dev) + } +} +``` + +ghw discovers the SRIOV devices starting from the Physical Function (PF) and exposes them in the `PhysicalFunctions` slice. +Virtual Function (VF) are exposed as properties of the PF instance with exposes them. +However, in some cases users are interested in the VFs first, so it's clumsy to navigate the PFs to learn about VFs. +To make this easier, ghw also exposes a slice of VF instances. These instances are soft references to the very same VF objects +you can find from the PF objects. + +```go +package main + +import ( + "fmt" + + "github.com/jaypipes/ghw" +) + +func main() { + sriov, err := ghw.SRIOV() + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting SRIOV info: %v", err) + } + + fmt.Printf("%v\n", sriov) + + // you will see the very same VF data data you seen from the previous example + for _, dev := range sriov.VirtualFunctions { + fmt.Printf(" %v\n", dev) + } +} +``` + + ### Chassis The host's chassis information is accessible with the `ghw.Chassis()` function. This diff --git a/cmd/ghwc/commands/sriov.go b/cmd/ghwc/commands/sriov.go new file mode 100644 index 00000000..c6048c8d --- /dev/null +++ b/cmd/ghwc/commands/sriov.go @@ -0,0 +1,48 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package commands + +import ( + "fmt" + + "github.com/jaypipes/ghw" + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +// sriovCmd represents the listing command +var sriovCmd = &cobra.Command{ + Use: "sriov", + Short: "Show SRIOV devices information for the host system", + RunE: showSRIOV, +} + +// showSRIOV show SRIOV physical device information for the host system. +func showSRIOV(cmd *cobra.Command, args []string) error { + sriov, err := ghw.SRIOV() + if err != nil { + return errors.Wrap(err, "error getting SRIOV info") + } + + switch outputFormat { + case outputFormatHuman: + fmt.Printf("%v\n", sriov) + + for _, dev := range sriov.PhysicalFunctions { + fmt.Printf(" %v\n", dev) + } + case outputFormatJSON: + fmt.Printf("%s\n", sriov.JSONString(pretty)) + case outputFormatYAML: + fmt.Printf("%s", sriov.YAMLString()) + } + return nil +} + +func init() { + rootCmd.AddCommand(sriovCmd) +} diff --git a/host.go b/host.go index 5d82a53a..f38020dd 100644 --- a/host.go +++ b/host.go @@ -8,6 +8,7 @@ package ghw import ( "fmt" + "strings" "github.com/jaypipes/ghw/pkg/context" @@ -110,20 +111,24 @@ func Host(opts ...*WithOption) (*HostInfo, error) { // String returns a newline-separated output of the HostInfo's component // structs' String-ified output func (info *HostInfo) String() string { - return fmt.Sprintf( - "%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n", - info.Block.String(), - info.CPU.String(), - info.GPU.String(), - info.Memory.String(), - info.Network.String(), - info.Topology.String(), - info.Chassis.String(), - info.BIOS.String(), - info.Baseboard.String(), - info.Product.String(), - info.PCI.String(), - ) + var b strings.Builder + for _, s := range []fmt.Stringer{ + info.Block, + info.CPU, + info.GPU, + info.Memory, + info.Network, + info.Topology, + info.Chassis, + info.BIOS, + info.Baseboard, + info.Product, + info.PCI, + } { + b.WriteString(s.String()) + b.WriteString("\n") + } + return b.String() } // YAMLString returns a string with the host information formatted as YAML diff --git a/pkg/pci/address/address.go b/pkg/pci/address/address.go index 6a8a4e45..ad371055 100644 --- a/pkg/pci/address/address.go +++ b/pkg/pci/address/address.go @@ -25,6 +25,16 @@ type Address struct { Function string } +func (addr *Address) Equal(a *Address) bool { + if addr == nil && a == nil { + return true + } + if addr != nil && a != nil { + return addr.Domain == a.Domain && addr.Bus == a.Bus && addr.Device == a.Device && addr.Function == a.Function + } + return false +} + // String() returns the canonical [D]BDF representation of this Address func (addr *Address) String() string { return addr.Domain + ":" + addr.Bus + ":" + addr.Device + "." + addr.Function diff --git a/pkg/pci/address/address_test.go b/pkg/pci/address/address_test.go index daf0eaa6..c3989dd4 100644 --- a/pkg/pci/address/address_test.go +++ b/pkg/pci/address/address_test.go @@ -80,3 +80,11 @@ func TestPCIAddressFromString(t *testing.T) { } } } + +func TestPCIAddressEqual(t *testing.T) { + addr1 := pciaddr.FromString("0000:03:00.A") + addr2 := pciaddr.FromString("03:00.A") + if addr1.Equal(addr2) == false { + t.Fatalf("addr1 %v and addr2 %v should be equal", addr1, addr2) + } +} diff --git a/pkg/pci/function.go b/pkg/pci/function.go new file mode 100644 index 00000000..d4aae167 --- /dev/null +++ b/pkg/pci/function.go @@ -0,0 +1,28 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package pci + +// Function describes an SR-IOV physical or virtual function. Physical functions +// will have no Parent Function struct pointer and will have one or more Function +// structs in the Functions field. +type Function struct { + Device // All Functions are PCI Devices + // Parent contains a pointer to the parent physical function. + // Will be empty when this is a physical function + Parent *Function `json:"parent,omitempty"` + // MaxVFs contains the maximum number of supported virtual + // functions for this physical function + MaxVFs int `json:"max_vfs"` + // Functions contains the physical function's virtual functions + Functions []*Function `json:"functions"` +} + +// IsPhysical returns true if the PCIe function is a physical function, false +// if it is a virtual function +func (f *Function) IsPhysical() bool { + return f.Parent == nil +} diff --git a/pkg/pci/function_linux.go b/pkg/pci/function_linux.go new file mode 100644 index 00000000..bd51738e --- /dev/null +++ b/pkg/pci/function_linux.go @@ -0,0 +1,109 @@ +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package pci + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/jaypipes/ghw/pkg/context" + "github.com/jaypipes/ghw/pkg/linuxpath" + pciaddress "github.com/jaypipes/ghw/pkg/pci/address" + "github.com/jaypipes/ghw/pkg/util" +) + +func (info *Info) scanDevice(dev *pci.Device) error { + paths := linuxpath.New(info.ctx) + devPath := filepath.Join(paths.SysBusPciDevices, dev.Address) + + buf, err := ioutil.ReadFile(filepath.Join(devPath, "sriov_totalvfs")) + if err != nil { + // is not a physfn. Since we will fill virtfn from physfn, we can give up now + return nil + } + + maxVFs, err := strconv.Atoi(strings.TrimSpace(string(buf))) + if err != nil { + info.ctx.Warn("error reading sriov_totalvfn for %q: %v", devPath, err) + return nil + } + + virtFNs := findVFsFromPF(info, info, dev.Address, devPath) + physFN := PhysicalFunction{ + Device: info.newDevice(dev, devPath), + MaxVFNum: maxVFs, + VFs: virtFNs, + } + + info.PhysicalFunctions = append(info.PhysicalFunctions, &physFN) + for idx := 0; idx < len(virtFNs); idx++ { + info.VirtualFunctions = append(info.VirtualFunctions, &virtFNs[idx]) + } + + return nil +} + +func findVFsFromPF(info *Info, pciInfo *pci.Info, parentAddr, parentPath string) []VirtualFunction { + numVfs := util.SafeIntFromFile(info.ctx, filepath.Join(parentPath, "sriov_numvfs")) + if numVfs == -1 { + return nil + } + + var vfs []VirtualFunction + for vfnIdx := 0; vfnIdx < numVfs; vfnIdx++ { + virtFn := fmt.Sprintf("virtfn%d", vfnIdx) + vfnDest, err := os.Readlink(filepath.Join(parentPath, virtFn)) + if err != nil { + info.ctx.Warn("error reading backing device for virtfn %q physfn %q: %v", virtFn, parentPath, err) + return nil + } + + vfnPath := filepath.Clean(filepath.Join(parentPath, vfnDest)) + vfnAddr := filepath.Base(vfnDest) + vfnDev := pciInfo.GetDevice(vfnAddr) + if vfnDev == nil { + info.ctx.Warn("error finding the PCI device for virtfn %s physfn %s", vfnAddr, parentAddr) + return nil + } + + vfs = append(vfs, VirtualFunction{ + Device: info.newDevice(vfnDev, vfnPath), + Index: vfnIdx, + ParentAddress: pciaddress.FromString(parentAddr), + }) + } + return vfs +} + +func (info *Info) newDevice(dev *pci.Device, devPath string) Device { + // see: https://doc.dpdk.org/guides/linux_gsg/linux_drivers.html + return Device{ + Address: pciaddress.FromString(dev.Address), + Interfaces: findNetworkInterfaceNames(info.ctx, devPath), + PCI: dev, + } +} + +func findNetworkInterfaceNames(ctx *context.Context, devPath string) []string { + netPath := filepath.Join(devPath, "net") + + netEntries, err := ioutil.ReadDir(netPath) + if err != nil { + ctx.Warn("cannot enumerate network names for %q: %v", devPath, err) + return nil + } + + var networks []string + for _, netEntry := range netEntries { + networks = append(networks, netEntry.Name()) + } + + return networks +} diff --git a/pkg/pci/function_linux_test.go b/pkg/pci/function_linux_test.go new file mode 100644 index 00000000..49f90131 --- /dev/null +++ b/pkg/pci/function_linux_test.go @@ -0,0 +1,198 @@ +// +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package sriov_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/jaypipes/ghw/pkg/option" + pciaddr "github.com/jaypipes/ghw/pkg/pci/address" + "github.com/jaypipes/ghw/pkg/sriov" + + "github.com/jaypipes/ghw/testdata" +) + +// nolint: gocyclo +func TestStringify(t *testing.T) { + info := sriovTestSetup(t) + + for _, physFn := range info.PhysicalFunctions { + s := physFn.String() + if s == "" || !strings.Contains(s, "function") || !strings.Contains(s, "physical") { + t.Errorf("Wrong string representation %q", s) + } + } + + for _, virtFn := range info.VirtualFunctions { + s := virtFn.String() + if s == "" || !strings.Contains(s, "function") || !strings.Contains(s, "virtual") { + t.Errorf("Wrong string representation %q", s) + } + } + +} + +// nolint: gocyclo +func TestCountDevices(t *testing.T) { + info := sriovTestSetup(t) + + // Check the content of + // GHW_SNAPSHOT_PATH="/path/to/linux-amd64-intel-xeon-L5640.tar.gz" ghwc sriov + // to verify these magic numbers + expectedPhysDevs := 2 + expectedVirtDevsPerPhys := 4 + numPhysDevs := len(info.PhysicalFunctions) + if numPhysDevs != expectedPhysDevs { + t.Errorf("Expected %d physical devices found %d", expectedPhysDevs, numPhysDevs) + } + numVirtDevs := len(info.VirtualFunctions) + if numPhysDevs*expectedVirtDevsPerPhys != numVirtDevs { + t.Errorf("Expected %d=(%d*%d) virtual devices found %d", numPhysDevs*expectedVirtDevsPerPhys, numPhysDevs, expectedVirtDevsPerPhys, numVirtDevs) + } + + for _, physDev := range info.PhysicalFunctions { + numVFs := len(physDev.VFs) + if numVFs != expectedVirtDevsPerPhys { + t.Errorf("Expected %d virtual devices for PF %s found %d", expectedVirtDevsPerPhys, physDev.Address.String(), numVFs) + } + } +} + +type pfTestCase struct { + addr string + netname string +} + +// nolint: gocyclo +func TestMatchPhysicalFunction(t *testing.T) { + info := sriovTestSetup(t) + + // Check the content of + // GHW_SNAPSHOT_PATH="/path/to/linux-amd64-intel-xeon-L5640.tar.gz" ghwc sriov + // to verify these magic numbers + for _, pfTC := range []pfTestCase{ + { + addr: "0000:05:00.0", + netname: "enp5s0f0", + }, + { + addr: "0000:05:00.1", + netname: "enp5s0f1", + }, + } { + addr := pciaddr.FromString(pfTC.addr) + pf := findPF(info.PhysicalFunctions, addr) + if pf == nil { + t.Fatalf("missing PF at addr %q", addr.String()) + } + if pf.PCI == nil { + t.Errorf("missing PCI device for %q", addr.String()) + } + if pf.PCI.Driver != "igb" { + t.Errorf("unexpected driver for %#v: %q", pf, pf.PCI.Driver) + } + if len(pf.Interfaces) != 1 || pf.Interfaces[0] != pfTC.netname { + t.Errorf("unexpected interfaces for %#v: %v", pf, pf.Interfaces) + } + if pf.MaxVFNum != 7 { + t.Errorf("unexpected MaxVFNum for %#v: %d", pf, pf.MaxVFNum) + } + if len(pf.VFs) != 4 { + t.Errorf("unexpected VF count for %#v: %d", pf, len(pf.VFs)) + } + for _, vfInst := range pf.VFs { + vf := findVF(info.VirtualFunctions, vfInst.Address) + if vf == nil { + t.Errorf("VF %#v from %#v not found among info.VirtualFunctions", vfInst, pf) + } + } + } +} + +func TestMatchVirtualFunction(t *testing.T) { + info := sriovTestSetup(t) + + // Check the content of + // GHW_SNAPSHOT_PATH="/path/to/linux-amd64-intel-xeon-L5640.tar.gz" ghwc sriov + // to verify these magic numbers + + for _, vf := range info.VirtualFunctions { + if vf.PCI == nil { + t.Errorf("missing PCI device for %q", vf.Address.String()) + } + if vf.PCI.Driver != "igbvf" { + t.Errorf("unexpected driver for %#v: %q", vf, vf.PCI.Driver) + } + + pf := findPF(info.PhysicalFunctions, vf.ParentAddress) + if pf == nil { + t.Fatalf("missing parent device for %q", vf.Address.String()) + } + if vf2 := findVFInst(pf.VFs, vf.Address); vf2 == nil { + t.Errorf("VF %#v not included in parent %#v VFs", vf, pf) + } + } +} + +func findPF(pfs []*sriov.PhysicalFunction, addr *pciaddr.Address) *sriov.PhysicalFunction { + for _, pf := range pfs { + if pf.Address.Equal(addr) { + return pf + } + } + return nil +} + +func findVF(vfs []*sriov.VirtualFunction, addr *pciaddr.Address) *sriov.VirtualFunction { + for _, vf := range vfs { + if vf.Address.Equal(addr) { + return vf + } + } + return nil +} + +func findVFInst(vfs []sriov.VirtualFunction, addr *pciaddr.Address) *sriov.VirtualFunction { + for idx := 0; idx < len(vfs); idx++ { + if vfs[idx].Address.Equal(addr) { + return &vfs[idx] + } + } + return nil +} + +func sriovTestSetup(t *testing.T) *sriov.Info { + if _, ok := os.LookupEnv("GHW_TESTING_SKIP_SRIOV"); ok { + t.Skip("Skipping SRIOV tests.") + } + + testdataPath, err := testdata.SnapshotsDirectory() + if err != nil { + t.Fatalf("Expected nil err, but got %v", err) + } + + multiNumaSnapshot := filepath.Join(testdataPath, "linux-amd64-intel-xeon-L5640.tar.gz") + // from now on we use constants reflecting the content of the snapshot we requested, + // which we reviewed beforehand. IOW, you need to know the content of the + // snapshot to fully understand this test. Inspect it using + // GHW_SNAPSHOT_PATH="/path/to/linux-amd64-intel-xeon-L5640.tar.gz" ghwc sriov + + info, err := sriov.New(option.WithSnapshot(option.SnapshotOptions{ + Path: multiNumaSnapshot, + })) + + if err != nil { + t.Fatalf("Expected nil err, but got %v", err) + } + if info == nil { + t.Fatalf("Expected non-nil SRIOVInfo, but got nil") + } + return info +} diff --git a/pkg/pci/function_stub.go b/pkg/pci/function_stub.go new file mode 100644 index 00000000..943a2710 --- /dev/null +++ b/pkg/pci/function_stub.go @@ -0,0 +1,19 @@ +//go:build !linux +// +build !linux + +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. +// + +package sriov + +import ( + "runtime" + + "github.com/pkg/errors" +) + +func (i *Info) load() error { + return errors.New("SRIOV load() not implemented on " + runtime.GOOS) +} diff --git a/pkg/pci/pci.go b/pkg/pci/pci.go index 86cc7b25..584f4ce3 100644 --- a/pkg/pci/pci.go +++ b/pkg/pci/pci.go @@ -127,6 +127,8 @@ type Info struct { ctx *context.Context // All PCI devices on the host system Devices []*Device + // All SRIOV devices on the host system + Functions []*Function // hash of class ID -> class information // DEPRECATED. Will be removed in v1.0. Please use // github.com/jaypipes/pcidb to explore PCIDB information diff --git a/pkg/snapshot/clonetree.go b/pkg/snapshot/clonetree.go index 519a874d..587c6b03 100644 --- a/pkg/snapshot/clonetree.go +++ b/pkg/snapshot/clonetree.go @@ -100,7 +100,7 @@ func CopyFilesInto(fileSpecs []string, destDir string, opts *CopyFileOptions) er if opts == nil { opts = &CopyFileOptions{ IsSymlinkFn: isSymlink, - ShouldCreateDirFn: isDriversDir, + ShouldCreateDirFn: shouldCreateDir, } } for _, fileSpec := range fileSpecs { @@ -157,6 +157,13 @@ func copyFileTreeInto(paths []string, destDir string, opts *CopyFileOptions) err return nil } +func shouldCreateDir(path string, fi os.FileInfo) bool { + if isDeviceNetworkDir(path, fi) { + return true + } + return isDriversDir(path, fi) +} + func isSymlink(path string, fi os.FileInfo) bool { return fi.Mode()&os.ModeSymlink != 0 } @@ -165,6 +172,12 @@ func isDriversDir(path string, fi os.FileInfo) bool { return strings.Contains(path, "drivers") } +func isDeviceNetworkDir(path string, fi os.FileInfo) bool { + parentDir := filepath.Base(filepath.Dir(path)) + // TODO: the "HasPrefix" check is brutal, but should work on linux + return parentDir == "net" && strings.HasPrefix(path, "/sys/devices") +} + func copyLink(path, targetPath string) error { target, err := os.Readlink(path) if err != nil { diff --git a/pkg/snapshot/clonetree_pci_linux.go b/pkg/snapshot/clonetree_pci_linux.go index dbc3fc83..e39e8c09 100644 --- a/pkg/snapshot/clonetree_pci_linux.go +++ b/pkg/snapshot/clonetree_pci_linux.go @@ -70,6 +70,19 @@ func scanPCIDeviceRoot(root string) (fileSpecs []string, pciRoots []string) { "revision", "vendor", } + + perDevEntriesOpt := []string{ + "driver", + "net/*", + "physfn", + "sriov_*", + "virtfn*", + } + + ignoreSet := map[string]bool{ + "sriov_vf_msix_count": true, // linux >= 5.14, write-only + } + entries, err := ioutil.ReadDir(root) if err != nil { return []string{}, []string{} @@ -96,6 +109,25 @@ func scanPCIDeviceRoot(root string) (fileSpecs []string, pciRoots []string) { fileSpecs = append(fileSpecs, filepath.Join(pciEntry, perNetEntry)) } + for _, perNetEntryOpt := range perDevEntriesOpt { + netEntryOptPath := filepath.Join(pciEntry, perNetEntryOpt) + + items, err := filepath.Glob(netEntryOptPath) + if err != nil { + // TODO: we skip silently because we don't have + // a ctx handy, so we can't do ctx.Warn :\ + continue + } + + for _, item := range items { + globbedEntry := filepath.Base(item) + if _, ok := ignoreSet[globbedEntry]; ok { + continue + } + fileSpecs = append(fileSpecs, item) + } + } + if isPCIBridge(entryPath) { trace("adding new PCI root %q\n", entryName) pciRoots = append(pciRoots, pciEntry) diff --git a/testdata/snapshots/linux-amd64-8581cf3a529e5d8b97ea876eade2f60d.tar.gz b/testdata/snapshots/linux-amd64-8581cf3a529e5d8b97ea876eade2f60d.tar.gz index 83cba8a0..2052b9c7 100644 Binary files a/testdata/snapshots/linux-amd64-8581cf3a529e5d8b97ea876eade2f60d.tar.gz and b/testdata/snapshots/linux-amd64-8581cf3a529e5d8b97ea876eade2f60d.tar.gz differ