Merge pull request #225 from fromanirh/pci-expose-numa-node

pci: report NUMA node locality for the PCI devices

pkg/pci/pci.go

@@ -17,6 +17,7 @@ import (
 	"github.com/jaypipes/ghw/pkg/context"
 	"github.com/jaypipes/ghw/pkg/marshal"
 	"github.com/jaypipes/ghw/pkg/option"
+	"github.com/jaypipes/ghw/pkg/topology"
 	"github.com/jaypipes/ghw/pkg/util"
 )
 
@@ -39,6 +40,9 @@ type Device struct {
 	Subclass *pcidb.Subclass `json:"subclass"`
 	// optional programming interface
 	ProgrammingInterface *pcidb.ProgrammingInterface `json:"programming_interface"`
+	// Topology node that the PCI device is affined to. Will be nil if the
+	// architecture is not NUMA.
+	Node *topology.Node `json:"node,omitempty"`
 }
 
 type devIdent struct {
@@ -116,7 +120,8 @@ func (d *Device) String() string {
 }
 
 type Info struct {
-	ctx *context.Context
+	arch topology.Architecture
+	ctx  *context.Context
 	// All PCI devices on the host system
 	Devices []*Device
 	// hash of class ID -> class information
@@ -178,7 +183,19 @@ func AddressFromString(address string) *Address {
 // PCI devices on the host system
 func New(opts ...*option.Option) (*Info, error) {
 	ctx := context.New(opts...)
-	info := &Info{ctx: ctx}
+	// by default we don't report NUMA information;
+	// we will only if are sure we are running on NUMA architecture
+	arch := topology.ARCHITECTURE_SMP
+	topo, err := topology.NewWithContext(ctx)
+	if err == nil {
+		arch = topo.Architecture
+	} else {
+		ctx.Warn("error detecting system topology: %v", err)
+	}
+	info := &Info{
+		arch: arch,
+		ctx:  ctx,
+	}
 	if err := ctx.Do(info.load); err != nil {
 		return nil, err
 	}

pkg/pci/pci_linux.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/jaypipes/ghw/pkg/context"
 	"github.com/jaypipes/ghw/pkg/linuxpath"
+	"github.com/jaypipes/ghw/pkg/topology"
 	"github.com/jaypipes/ghw/pkg/util"
 )
 
@@ -66,6 +67,28 @@ func getDeviceRevision(ctx *context.Context, address string) string {
 	return string(revision)
 }
 
+func getDeviceNUMANode(ctx *context.Context, address string) *topology.Node {
+	paths := linuxpath.New(ctx)
+	pciAddr := AddressFromString(address)
+	if pciAddr == nil {
+		return nil
+	}
+	numaNodePath := filepath.Join(paths.SysBusPciDevices, pciAddr.String(), "numa_node")
+
+	if _, err := os.Stat(numaNodePath); err != nil {
+		return nil
+	}
+
+	nodeIdx := util.SafeIntFromFile(ctx, numaNodePath)
+	if nodeIdx == -1 {
+		return nil
+	}
+
+	return &topology.Node{
+		ID: nodeIdx,
+	}
+}
+
 type deviceModaliasInfo struct {
 	vendorID     string
 	productID    string
@@ -263,6 +286,9 @@ func (info *Info) GetDevice(address string) *Device {
 
 	device := info.getDeviceFromModaliasInfo(address, modaliasInfo)
 	device.Revision = getDeviceRevision(info.ctx, address)
+	if info.arch == topology.ARCHITECTURE_NUMA {
+		device.Node = getDeviceNUMANode(info.ctx, address)
+	}
 	return device
 }
 

pkg/pci/pci_linux_test.go

@@ -0,0 +1,87 @@
+//
+// Use and distribution licensed under the Apache license version 2.
+//
+// See the COPYING file in the root project directory for full text.
+//
+
+package pci_test
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/jaypipes/ghw/pkg/option"
+	"github.com/jaypipes/ghw/pkg/pci"
+
+	"github.com/jaypipes/ghw/testdata"
+)
+
+type pciNumaTestCase struct {
+	addr string
+	node int
+}
+
+// nolint: gocyclo
+func TestPCINUMANode(t *testing.T) {
+	if _, ok := os.LookupEnv("GHW_TESTING_SKIP_PCI"); ok {
+		t.Skip("Skipping PCI tests.")
+	}
+
+	testdataPath, err := testdata.SnapshotsDirectory()
+	if err != nil {
+		t.Fatalf("Expected nil err, but got %v", err)
+	}
+
+	multiNumaSnapshot := filepath.Join(testdataPath, "linux-amd64-intel-xeon-L5640.tar.gz")
+	// from now on we use constants reflecting the content of the snapshot we requested,
+	// which we reviewed beforehand. IOW, you need to know the content of the
+	// snapshot to fully understand this test. Inspect it using
+	// GHW_SNAPSHOT_PATH="/path/to/linux-amd64-intel-xeon-L5640.tar.gz" ghwc topology
+
+	info, err := pci.New(option.WithSnapshot(option.SnapshotOptions{
+		Path: multiNumaSnapshot,
+	}))
+
+	if err != nil {
+		t.Fatalf("Expected nil err, but got %v", err)
+	}
+	if info == nil {
+		t.Fatalf("Expected non-nil PCIInfo, but got nil")
+	}
+
+	tCases := []pciNumaTestCase{
+		{
+			addr: "0000:07:03.0",
+			// -1 is actually what we get out of the box on the snapshotted box
+			node: -1,
+		},
+		{
+			addr: "0000:05:11.0",
+			node: 0,
+		},
+		{
+			addr: "0000:05:00.1",
+			node: 1,
+		},
+	}
+	for _, tCase := range tCases {
+		t.Run(fmt.Sprintf("%s (%d)", tCase.addr, tCase.node), func(t *testing.T) {
+			dev := info.GetDevice(tCase.addr)
+			if dev == nil {
+				t.Fatalf("got nil device for address %q", tCase.addr)
+			}
+			if dev.Node == nil {
+				if tCase.node != -1 {
+					t.Fatalf("got nil numa NODE for address %q", tCase.addr)
+				}
+			} else {
+				if dev.Node.ID != tCase.node {
+					t.Errorf("got NUMA node info %#v, expected on node %d", dev.Node, tCase.node)
+				}
+			}
+		})
+	}
+
+}

pkg/topology/topology.go

@@ -79,7 +79,13 @@ type Info struct {
 // New returns a pointer to an Info struct that contains information about the
 // NUMA topology on the host system
 func New(opts ...*option.Option) (*Info, error) {
-	ctx := context.New(opts...)
+	return NewWithContext(context.New(opts...))
+}
+
+// NewWithContext returns a pointer to an Info struct that contains information about
+// the NUMA topology on the host system. Use this function when you want to consume
+// the topology package from another package (e.g. pci, gpu)
+func NewWithContext(ctx *context.Context) (*Info, error) {
 	info := &Info{ctx: ctx}
 	if err := ctx.Do(info.load); err != nil {
 		return nil, err