DAOS-14793 control: Fail if VMD has no backing storage (#13503) (#13546)

Report the specific case where VMD address has no backing devices. A
bdev scan of PCI address specified in server config file bdev_list
should report an error indicating if any of the addresses are for a
VMD which has no backing storage (backing NVMe SSDs).

Signed-off-by: Tom Nabarro <[email protected]>

src/control/server/server_utils_test.go

@@ -858,9 +858,9 @@ func TestServer_scanBdevStorage(t *testing.T) {
 		},
 		"bdev in config not found by spdk": {
 			bmbc: &bdev.MockBackendConfig{
-				ScanErr: storage.FaultBdevNotFound(test.MockPCIAddr()),
+				ScanErr: storage.FaultBdevNotFound(false, test.MockPCIAddr()),
 			},
-			expErr: storage.FaultBdevNotFound(test.MockPCIAddr()),
+			expErr: storage.FaultBdevNotFound(false, test.MockPCIAddr()),
 		},
 		"successful scan": {
 			bmbc: &bdev.MockBackendConfig{

src/control/server/storage/bdev/backend.go

@@ -273,7 +273,7 @@ func (sb *spdkBackend) Prepare(req storage.BdevPrepareRequest) (*storage.BdevPre
 
 // groomDiscoveredBdevs ensures that for a non-empty device list, restrict output controller data
 // to only those devices discovered and in device list and confirm that the devices specified in
-// the device list have all been discovered.
+// the device list have all been discovered. VMD addresses with no backing devices return error.
 func groomDiscoveredBdevs(reqDevs *hardware.PCIAddressSet, discovered storage.NvmeControllers, vmdEnabled bool) (storage.NvmeControllers, error) {
 	// if the request does not specify a device filter, return all discovered controllers
 	if reqDevs.IsEmpty() {
@@ -319,7 +319,7 @@ func groomDiscoveredBdevs(reqDevs *hardware.PCIAddressSet, discovered storage.Nv
 	}
 
 	if !missing.IsEmpty() {
-		return nil, storage.FaultBdevNotFound(missing.Strings()...)
+		return nil, storage.FaultBdevNotFound(vmdEnabled, missing.Strings()...)
 	}
 
 	return out, nil

src/control/server/storage/bdev/backend_test.go

@@ -125,14 +125,22 @@ func TestBackend_groomDiscoveredBdevs(t *testing.T) {
 		"missing": {
 			reqAddrList: []string{ctrlr1.PciAddr, ctrlr2.PciAddr, ctrlr3.PciAddr},
 			inCtrlrs:    storage.NvmeControllers{ctrlr1, ctrlr3},
-			expErr:      storage.FaultBdevNotFound(ctrlr2.PciAddr),
+			expErr:      storage.FaultBdevNotFound(false, ctrlr2.PciAddr),
 		},
 		"vmd devices; vmd not enabled": {
 			reqAddrList: []string{"0000:85:05.5"},
 			inCtrlrs: ctrlrsFromPCIAddrs("850505:07:00.0", "850505:09:00.0",
 				"850505:0b:00.0", "850505:0d:00.0", "850505:0f:00.0",
 				"850505:11:00.0", "850505:14:00.0", "5d0505:03:00.0"),
-			expErr: storage.FaultBdevNotFound("0000:85:05.5"),
+			expErr: storage.FaultBdevNotFound(false, "0000:85:05.5"),
+		},
+		"vmd enabled; missing backing devices": {
+			vmdEnabled:  true,
+			reqAddrList: []string{"0000:85:05.5", "0000:05:05.5"},
+			inCtrlrs: ctrlrsFromPCIAddrs("850505:07:00.0", "850505:09:00.0",
+				"850505:0b:00.0", "850505:0d:00.0", "850505:0f:00.0",
+				"850505:11:00.0", "850505:14:00.0", "5d0505:03:00.0"),
+			expErr: storage.FaultBdevNotFound(true, "0000:05:05.5"),
 		},
 		"vmd devices; vmd enabled": {
 			vmdEnabled:  true,
@@ -205,7 +213,7 @@ func TestBackend_Scan(t *testing.T) {
 				DiscoverCtrlrs: storage.NvmeControllers{ctrlr1},
 			},
 			req:    mockScanReq(storage.MockNvmeController(2).PciAddr),
-			expErr: storage.FaultBdevNotFound(storage.MockNvmeController(2).PciAddr),
+			expErr: storage.FaultBdevNotFound(false, storage.MockNvmeController(2).PciAddr),
 		},
 		"emulated nvme; AIO-file": {
 			req:     mockScanReq(storage.MockNvmeAioFile(2).Path),

src/control/server/storage/bdev/firmware.go

@@ -91,7 +91,7 @@ func getDeviceController(pciAddr string, controllers storage.NvmeControllers) (*
 		}
 	}
 
-	return nil, storage.FaultBdevNotFound(pciAddr)
+	return nil, storage.FaultBdevNotFound(false, pciAddr)
 }
 
 func filterControllersByModelFirmware(controllers storage.NvmeControllers, modelID, fwRev string) storage.NvmeControllers {

src/control/server/storage/bdev/firmware_test.go

@@ -293,7 +293,7 @@ func TestProvider_UpdateFirmware(t *testing.T) {
 			backendCfg: &MockBackendConfig{
 				ScanRes: &storage.BdevScanResponse{Controllers: defaultDevs},
 			},
-			expErr: storage.FaultBdevNotFound("fake"),
+			expErr: storage.FaultBdevNotFound(false, "fake"),
 		},
 		"request duplicates": {
 			input: storage.NVMeFirmwareUpdateRequest{

src/control/server/storage/faults.go

@@ -189,14 +189,18 @@ func FaultBdevConfigBadNrRoles(role string, gotNr, wantNr int) *fault.Fault {
 			role, wantNr))
 }
 
-// FaultBdevNotFound creates a Fault for the case where no NVMe storage devices
-// match expected PCI addresses.
-func FaultBdevNotFound(bdevs ...string) *fault.Fault {
+// FaultBdevNotFound creates a Fault for the case where no NVMe storage devices match expected PCI
+// addresses. VMD addresses are expected to have backing devices.
+func FaultBdevNotFound(vmdEnabled bool, bdevs ...string) *fault.Fault {
+	msg := fmt.Sprintf("NVMe SSD%s", common.Pluralise("", len(bdevs)))
+	if vmdEnabled {
+		msg = "backing devices for VMDs"
+	}
+
 	return storageFault(
 		code.BdevNotFound,
-		fmt.Sprintf("NVMe SSD%s %v not found", common.Pluralise("", len(bdevs)), bdevs),
-		fmt.Sprintf("check SSD%s %v that are specified in server config exist",
-			common.Pluralise("", len(bdevs)), bdevs),
+		fmt.Sprintf("%s %v not found", msg, bdevs),
+		fmt.Sprintf("check %s %v that are specified in server config exist", msg, bdevs),
 	)
 }