From ddd576406a5574002ed55871f3fee73394dfbc2f Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Sat, 14 Dec 2024 00:20:02 +0100 Subject: [PATCH] in progress Signed-off-by: Ronan Abhamon --- Makefile | 3 +- drivers/CephFSSR.py | 1 - drivers/FileSR.py | 216 +++++----- drivers/LVHDSR.py | 84 ++-- drivers/LinstorSR.py | 24 +- drivers/blktap2.py | 21 +- drivers/cleanup.py | 99 +++-- drivers/cowutil.py | 321 ++++++++++++++ drivers/linstor-manager | 37 +- drivers/linstorvhdutil.py | 120 +++--- drivers/lvhdutil.py | 76 ++-- drivers/tapdisk-pause | 38 +- drivers/vditype.py | 6 +- drivers/verifyVHDsOnSR.py | 7 +- drivers/vhdutil.py | 866 ++++++++++++++++++++------------------ tests/test_FileSR.py | 2 +- tests/test_LVHDSR.py | 3 + tests/test_vhdutil.py | 19 +- 18 files changed, 1185 insertions(+), 758 deletions(-) create mode 100755 drivers/cowutil.py diff --git a/Makefile b/Makefile index cbbf50129..5b86039f7 100755 --- a/Makefile +++ b/Makefile @@ -34,11 +34,12 @@ SM_LIBS += util SM_LIBS += verifyVHDsOnSR SM_LIBS += scsiutil SM_LIBS += scsi_host_rescan +SM_LIBS += cowutil SM_LIBS += vhdutil SM_LIBS += linstorjournaler SM_LIBS += linstorvhdutil SM_LIBS += linstorvolumemanager -SM_LIBS += lvhdutil +SM_LIBS += lvhdutil # TODO: Split SM_LIBS += cifutils SM_LIBS += xs_errors SM_LIBS += nfs diff --git a/drivers/CephFSSR.py b/drivers/CephFSSR.py index 574190d6c..8e7c173d6 100644 --- a/drivers/CephFSSR.py +++ b/drivers/CephFSSR.py @@ -39,7 +39,6 @@ import cleanup import lock import util -import vhdutil import xs_errors CAPABILITIES = ["SR_PROBE", "SR_UPDATE", diff --git a/drivers/FileSR.py b/drivers/FileSR.py index 88812ddc3..65fcb0d42 100755 --- a/drivers/FileSR.py +++ b/drivers/FileSR.py @@ -24,7 +24,6 @@ import SRCommand import util import scsiutil -import vhdutil import lock import os import errno @@ -34,7 +33,8 @@ import time import glob from uuid import uuid4 -from vditype import VdiType, VdiTypeExtension, VDI_TYPE_TO_EXTENSION +from cowutil import CowImageInfo, CowUtil, ImageFormat, getCowUtil, getVdiTypeFromImageFormat, parseImageFormats +from vditype import VdiType, VdiTypeExtension, VDI_COW_TYPES, VDI_TYPE_TO_EXTENSION import xmlrpc.client import XenAPI # pylint: disable=import-error from constants import CBTLOG_TAG @@ -46,11 +46,16 @@ "VDI_GENERATE_CONFIG", "ATOMIC_PAUSE", "VDI_CONFIG_CBT", "VDI_ACTIVATE", "VDI_DEACTIVATE", "THIN_PROVISIONING"] -CONFIGURATION = [['location', 'local directory path (required)']] +CONFIGURATION = [ + ['location', 'local directory path (required)'], + ['preferred-image-formats', 'list of preferred image formats to use (default: VHD,QCOW2)'] +] + +DEFAULT_IMAGE_FORMATS = [ImageFormat.VHD, ImageFormat.QCOW2] DRIVER_INFO = { - 'name': 'Local Path VHD', - 'description': 'SR plugin which represents disks as VHD files stored on a local path', + 'name': 'Local Path VHD and QCOW2', + 'description': 'SR plugin which represents disks as VHD and QCOW2 files stored on a local path', 'vendor': 'Citrix Systems Inc', 'copyright': '(C) 2008 Citrix Systems Inc', 'driver_version': '1.0', @@ -91,6 +96,7 @@ def __init__(self, srcmd, sr_uuid): # We call SR.SR.__init__ explicitly because # "super" sometimes failed due to circular imports SR.SR.__init__(self, srcmd, sr_uuid) + self.image_info = {} self._check_o_direct() @override @@ -101,6 +107,10 @@ def load(self, sr_uuid) -> None: if 'location' not in self.dconf or not self.dconf['location']: raise xs_errors.XenError('ConfigLocationMissing') self.remotepath = self.dconf['location'] + self.preferred_image_formats = parseImageFormats( + self.dconf.get('preferred-image-formats'), + DEFAULT_IMAGE_FORMATS + ) self.path = os.path.join(SR.MOUNT_BASE, sr_uuid) self.linkpath = self.path self.mountpoint = self.path @@ -109,7 +119,7 @@ def load(self, sr_uuid) -> None: @override def create(self, sr_uuid, size) -> None: - """ Create the SR. The path must not already exist, or if it does, + """ Create the SR. The path must not already exist, or if it does, it must be empty. (This accounts for the case where the user has mounted a device onto a directory manually and want to use this as the root of a file-based SR.) """ @@ -273,32 +283,49 @@ def replay(self, uuid) -> None: except: raise xs_errors.XenError('SRLog') - def _loadvdis(self): - if self.vdis: - return + def _load_vdis_from_type(self, vdi_type: str) -> List[CowImageInfo]: + extension = VDI_TYPE_TO_EXTENSION[vdi_type] - pattern = os.path.join(self.path, "*%s" % VdiTypeExtension.VHD) + pattern = os.path.join(self.path, "*%s" % extension) + info: List[CowImageInfo] = [] + + cowutil = getCowUtil(vdi_type) try: - self.vhds = vhdutil.getAllVHDs(pattern, FileVDI.extractUuid) + info = cowutil.getAllInfoFromVG(pattern, FileVDI.extractUuid) except util.CommandException as inst: - raise xs_errors.XenError('SRScan', opterr="error VHD-scanning " \ + raise xs_errors.XenError('SRScan', opterr="error VDI-scanning " \ "path %s (%s)" % (self.path, inst)) try: - list_vhds = [FileVDI.extractUuid(v) for v in util.ioretry(lambda: glob.glob(pattern))] - if len(self.vhds) != len(list_vhds): - util.SMlog("VHD scan returns %d VHDs: %s" % (len(self.vhds), sorted(self.vhds))) - util.SMlog("VHD list returns %d VHDs: %s" % (len(list_vhds), sorted(list_vhds))) + vdi_uuids = [FileVDI.extractUuid(v) for v in util.ioretry(lambda: glob.glob(pattern))] + if len(info) != len(vdi_uuids): + util.SMlog("VDI scan of %s returns %d VDIs: %s" % (extension, len(info), sorted(info))) + util.SMlog("VDI list of %s returns %d VDIs: %s" % (extension, len(vdi_uuids), sorted(vdi_uuids))) except: pass - for uuid in self.vhds.keys(): - if self.vhds[uuid].error: + + for uuid in info.keys(): + if info[uuid].error: raise xs_errors.XenError('SRScan', opterr='uuid=%s' % uuid) - self.vdis[uuid] = self.vdi(uuid) + + file_vdi = self.vdi(uuid) + file_vdi.cowutil = cowutil + self.vdis[uuid] = file_vdi + # Get the key hash of any encrypted VDIs: - vhd_path = os.path.join(self.path, self.vhds[uuid].path) - key_hash = vhdutil.getKeyHash(vhd_path) + vdi_path = os.path.join(self.path, info[uuid].path) + key_hash = cowutil.getKeyHash(vdi_path) self.vdis[uuid].sm_config_override['key_hash'] = key_hash + return info + + def _loadvdis(self): + if self.vdis: + return + + self.image_info = {} + for vdi_type in VDI_COW_TYPES: + self.image_info.update(self._load_vdis_from_type(vdi_type)) + # raw VDIs and CBT log files files = util.ioretry(lambda: util.listdir(self.path)) for fn in files: @@ -418,18 +445,22 @@ def _check_hardlinks(self) -> bool: return True class FileVDI(VDI.VDI): - PARAM_VHD = "vhd" PARAM_RAW = "raw" + PARAM_VHD = "vhd" + PARAM_QCOW2 = "qcow2" VDI_TYPE = { + PARAM_RAW: VdiType.RAW, PARAM_VHD: VdiType.VHD, - PARAM_RAW: VdiType.RAW + PARAM_QCOW2: VdiType.QCOW2 } def _find_path_with_retries(self, vdi_uuid, maxretry=5, period=2.0): - vhd_path = os.path.join(self.sr.path, "%s.%s" % \ - (vdi_uuid, self.PARAM_VHD)) raw_path = os.path.join(self.sr.path, "%s.%s" % \ (vdi_uuid, self.PARAM_RAW)) + vhd_path = os.path.join(self.sr.path, "%s.%s" % \ + (vdi_uuid, self.PARAM_VHD)) + qcow2_path = os.path.join(self.sr.path, "%s.%s" % \ + (vdi_uuid, self.PARAM_QCOW2)) cbt_path = os.path.join(self.sr.path, "%s.%s" % (vdi_uuid, CBTLOG_TAG)) found = False @@ -440,6 +471,10 @@ def _find_path_with_retries(self, vdi_uuid, maxretry=5, period=2.0): self.vdi_type = VdiType.VHD self.path = vhd_path found = True + elif util.ioretry(lambda: util.pathexists(qcow2_path)): + self.vdi_type = VdiType.QCOW2 + self.path = qcow2_path + found = True elif util.ioretry(lambda: util.pathexists(raw_path)): self.vdi_type = VdiType.RAW self.path = raw_path @@ -451,8 +486,13 @@ def _find_path_with_retries(self, vdi_uuid, maxretry=5, period=2.0): self.hidden = False found = True - if not found: - util.SMlog("VHD %s not found, retry %s of %s" % (vhd_path, tries, maxretry)) + if found: + try: + self.cowutil = getCowUtil(self.vdi_type) + except: + pass + else: + util.SMlog("VDI %s not found, retry %s of %s" % (vdi_uuid, tries, maxretry)) time.sleep(period) return found @@ -464,7 +504,7 @@ def load(self, vdi_uuid) -> None: self.sr.srcmd.params['o_direct'] = self.sr.o_direct if self.sr.srcmd.cmd == "vdi_create": - self.vdi_type = VdiType.VHD + self.vdi_type = getVdiTypeFromImageFormat(self.sr.preferred_image_formats[0]) self.key_hash = None if "vdi_sm_config" in self.sr.srcmd.params: if "key_hash" in self.sr.srcmd.params["vdi_sm_config"]: @@ -483,25 +523,22 @@ def load(self, vdi_uuid) -> None: if not found: if self.sr.srcmd.cmd == "vdi_delete": # Could be delete for CBT log file - self.path = os.path.join(self.sr.path, "%s.%s" % - (vdi_uuid, self.PARAM_VHD)) + self.path = os.path.join(self.sr.path, f"{vdi_uuid}.deleted") return if self.sr.srcmd.cmd == "vdi_attach_from_config": return raise xs_errors.XenError('VDIUnavailable', opterr="VDI %s not found" % vdi_uuid) - - if VdiType.isCowImage(self.vdi_type) and \ - self.sr.__dict__.get("vhds") and self.sr.vhds.get(vdi_uuid): - # VHD info already preloaded: use it instead of querying directly - vhdInfo = self.sr.vhds[vdi_uuid] - self.utilisation = vhdInfo.sizePhys - self.size = vhdInfo.sizeVirt - self.hidden = vhdInfo.hidden + image_info = VdiType.isCowImage(self.vdi_type) and self.sr.image_info.get(vdi_uuid) + if image_info: + # Image info already preloaded: use it instead of querying directly + self.utilisation = image_info.sizePhys + self.size = image_info.sizeVirt + self.hidden = image_info.hidden if self.hidden: self.managed = False - self.parent = vhdInfo.parentUuid + self.parent = image_info.parentUuid if self.parent: self.sm_config_override = {'vhd-parent': self.parent} else: @@ -548,18 +585,16 @@ def load(self, vdi_uuid) -> None: try: # The VDI might be activated in R/W mode so the VHD footer # won't be valid, use the back-up one instead. - diskinfo = util.ioretry( - lambda: self._query_info(self.path, True), - errlist=[errno.EIO, errno.ENOENT]) + image_info = self.cowutil.getInfo(self.path, FileVDI.extractUuid, useBackupFooter=True) - if 'parent' in diskinfo: - self.parent = diskinfo['parent'] + if image_info.parentUuid: + self.parent = image_info.parentUuid self.sm_config_override = {'vhd-parent': self.parent} else: + self.parent = "" self.sm_config_override = {'vhd-parent': None} - self.parent = '' - self.size = int(diskinfo['size']) * 1024 * 1024 - self.hidden = int(diskinfo['hidden']) + self.size = image_info.sizeVirt + self.hidden = image_info.hidden if self.hidden: self.managed = False self.exists = True @@ -580,12 +615,11 @@ def create(self, sr_uuid, vdi_uuid, size) -> str: raise xs_errors.XenError('VDIExists') if VdiType.isCowImage(self.vdi_type): + self.cowutil = getCowUtil(self.vdi_type) try: - size = vhdutil.validate_and_round_vhd_size(int(size)) - mb = 1024 * 1024 - size_mb = size // mb - util.ioretry(lambda: self._create(str(size_mb), self.path)) - self.size = util.ioretry(lambda: self._query_v(self.path)) + size = self.cowutil.validateAndRoundImageSize(int(size)) + util.ioretry(lambda: self._create(size, self.path)) + self.size = self.cowutil.getSizeVirt(self.path) except util.CommandException as inst: raise xs_errors.XenError('VDICreate', opterr='error %d' % inst.code) @@ -679,19 +713,19 @@ def resize(self, sr_uuid, vdi_uuid, size) -> str: if size == self.size: return VDI.VDI.get_params(self) - # We already checked it is a VHD - size = vhdutil.validate_and_round_vhd_size(int(size)) - + # We already checked it is a cow image. + size = self.cowutil.validateAndRoundImageSize(int(size)) + jFile = JOURNAL_FILE_PREFIX + self.uuid try: - vhdutil.setSizeVirt(self.path, size, jFile) + self.cowutil.setSizeVirt(self.path, size, jFile) except: # Revert the operation - vhdutil.revert(self.path, jFile) + self.cowutil.revert(self.path, jFile) raise xs_errors.XenError('VDISize', opterr='resize operation failed') old_size = self.size - self.size = vhdutil.getSizeVirt(self.path) + self.size = self.cowutil.getSizeVirt(self.path) st = util.ioretry(lambda: os.stat(self.path)) self.utilisation = int(st.st_size) @@ -708,14 +742,12 @@ def clone(self, sr_uuid, vdi_uuid) -> str: def compose(self, sr_uuid, vdi1, vdi2) -> None: if not VdiType.isCowImage(self.vdi_type): raise xs_errors.XenError('Unimplemented') - parent_fn = vdi1 + VDI_TYPE_TO_EXTENSION[VdiType.VHD] + parent_fn = vdi1 + VDI_TYPE_TO_EXTENSION[self.vdi_type] parent_path = os.path.join(self.sr.path, parent_fn) assert(util.pathexists(parent_path)) - vhdutil.setParent(self.path, parent_path, False) - vhdutil.setHidden(parent_path) + self.cowutil.setParent(self.path, parent_path, False) + self.cowutil.setHidden(parent_path) self.sr.session.xenapi.VDI.set_managed(self.sr.srcmd.params['args'][0], False) - util.pread2([vhdutil.VHD_UTIL, "modify", "-p", parent_path, - "-n", self.path]) # Tell tapdisk the chain has changed if not blktap2.VDI.tap_refresh(self.session, sr_uuid, vdi2): raise util.SMException("failed to refresh VDI %s" % self.uuid) @@ -726,11 +758,11 @@ def reset_leaf(self, sr_uuid, vdi_uuid): raise xs_errors.XenError('Unimplemented') # safety check - if not vhdutil.hasParent(self.path): + if not self.cowutil.hasParent(self.path): raise util.SMException("ERROR: VDI %s has no parent, " + \ "will not reset contents" % self.uuid) - vhdutil.killData(self.path) + self.cowutil.killData(self.path) @override def _do_snapshot(self, sr_uuid, vdi_uuid, snapType, @@ -776,7 +808,7 @@ def _create_new_parent(self, src, newsrc): self._rename(src, newsrc) def __fist_enospace(self): - raise util.CommandException(28, "vhd-util snapshot", reason="No space") + raise util.CommandException(28, "cowutil snapshot", reason="No space") def _snapshot(self, snap_type, cbtlog=None, cbt_consistency=None): util.SMlog("FileVDI._snapshot for %s (type %s)" % (self.uuid, snap_type)) @@ -796,11 +828,11 @@ def _snapshot(self, snap_type, cbtlog=None, cbt_consistency=None): if self.hidden: raise xs_errors.XenError('VDIClone', opterr='hidden VDI') - depth = vhdutil.getDepth(self.path) + depth = self.cowutil.getDepth(self.path) if depth == -1: raise xs_errors.XenError('VDIUnavailable', \ - opterr='failed to get VHD depth') - elif depth >= vhdutil.MAX_CHAIN_SIZE: + opterr='failed to get image depth') + elif depth >= self.cowutil.getMaxChainLength(): raise xs_errors.XenError('SnapshotChainTooLong') newuuid = util.gen_uuid() @@ -844,10 +876,10 @@ def _snapshot(self, snap_type, cbtlog=None, cbt_consistency=None): #Verify parent locator field of both children and delete newsrc if unused introduce_parent = True try: - srcparent = util.ioretry(lambda: self._query_p_uuid(src)) + srcparent = self.cowutil.getParent(src, FileVDI.extractUuid) dstparent = None if snap_type == VDI.SNAPSHOT_DOUBLE: - dstparent = util.ioretry(lambda: self._query_p_uuid(dst)) + dstparent = self.cowutil.getParent(dst, FileVDI.extractUuid) if srcparent != newuuid and \ (snap_type == VDI.SNAPSHOT_SINGLE or \ snap_type == VDI.SNAPSHOT_INTERNAL or \ @@ -887,7 +919,7 @@ def _snapshot(self, snap_type, cbtlog=None, cbt_consistency=None): base_vdi.size = self.size base_vdi.utilisation = self.utilisation base_vdi.sm_config = {} - grandparent = util.ioretry(lambda: self._query_p_uuid(newsrc)) + grandparent = self.cowutil.getParent(newSrc, FileVDI.extractUuid) if grandparent.find("no parent") == -1: base_vdi.sm_config['vhd-parent'] = grandparent @@ -946,8 +978,7 @@ def get_params(self) -> str: return super(FileVDI, self).get_params() def _snap(self, child, parent): - cmd = [SR.TAPDISK_UTIL, "snapshot", VdiType.VHD, child, parent] - text = util.pread(cmd) + self.cowutil.snapshot(child, parent, parent.vdi_type == VdiType.RAW) def _clonecleanup(self, src, dst, newsrc): try: @@ -975,48 +1006,21 @@ def _checkpath(self, path): raise xs_errors.XenError('EIO', \ opterr='IO error checking path %s' % path) - def _query_v(self, path): - cmd = [SR.TAPDISK_UTIL, "query", VdiType.VHD, "-v", path] - return int(util.pread(cmd)) * 1024 * 1024 - - def _query_p_uuid(self, path): - cmd = [SR.TAPDISK_UTIL, "query", VdiType.VHD, "-p", path] - parent = util.pread(cmd) - parent = parent[:-1] - ls = parent.split('/') - return ls[len(ls) - 1].replace(VdiTypeExtension.VHD, '') - - def _query_info(self, path, use_bkp_footer=False): - diskinfo = {} - qopts = '-vpf' - if use_bkp_footer: - qopts += 'b' - cmd = [SR.TAPDISK_UTIL, "query", VdiType.VHD, qopts, path] - txt = util.pread(cmd).split('\n') - diskinfo['size'] = txt[0] - lst = [txt[1].split('/')[-1].replace(VdiTypeExtension.VHD, "")] - for val in filter(util.exactmatch_uuid, lst): - diskinfo['parent'] = val - diskinfo['hidden'] = txt[2].split()[1] - return diskinfo - def _create(self, size, path): - cmd = [SR.TAPDISK_UTIL, "create", VdiType.VHD, size, path] - text = util.pread(cmd) + self.cowutil.create(path, size, False) if self.key_hash: - vhdutil.setKey(path, self.key_hash) + self.cowutil.setKey(path, self.key_hash) def _mark_hidden(self, path): - vhdutil.setHidden(path, True) + self.cowutil.setHidden(path, True) self.hidden = 1 def _is_hidden(self, path): - return vhdutil.getHidden(path) == 1 + return self.cowutil.getHidden(path) == 1 def extractUuid(path): fileName = os.path.basename(path) - uuid = fileName.replace(VdiTypeExtension.VHD, "") - return uuid + return os.path.splitext(fileName)[0] extractUuid = staticmethod(extractUuid) @override diff --git a/drivers/LVHDSR.py b/drivers/LVHDSR.py index d6abb7cbe..be8b240df 100755 --- a/drivers/LVHDSR.py +++ b/drivers/LVHDSR.py @@ -725,6 +725,8 @@ def scan(self, uuid) -> None: activated = True lvPath = os.path.join(self.path, lvname) + cowutil = None + if not VdiType.isCowImage(vdi_type): size = self.lvmCache.getSize( \ lvhdutil.LV_PREFIX[vdi_type] + vdi_uuid) @@ -732,17 +734,16 @@ def scan(self, uuid) -> None: util.roundup(lvutil.LVM_SIZE_INCREMENT, int(size)) else: - parent = \ - vhdutil._getVHDParentNoCheck(lvPath) + parent = cowutil.getParentNoCheck(lvPath) if parent is not None: sm_config['vhd-parent'] = parent[len( \ lvhdutil.LV_PREFIX[VdiType.VHD]):] - size = vhdutil.getSizeVirt(lvPath) + size = cowutil.getSizeVirt(lvPath) if self.provision == "thin": utilisation = \ util.roundup(lvutil.LVM_SIZE_INCREMENT, - vhdutil.calcOverheadEmpty(lvhdutil.MSIZE)) + cowutil.calcOverheadEmpty(lvhdutil.MSIZE)) else: utilisation = lvhdutil.calcSizeVHDLV(int(size)) @@ -929,6 +930,8 @@ def _handleInterruptedCloneOp(self, origUuid, jval, forceUndo=False): lvs = lvhdutil.getLVInfo(self.lvmCache) baseUuid, clonUuid = jval.split("_") + cowutil = None # TODO + # is there a "base copy" VDI? if not lvs.get(baseUuid): # no base copy: make sure the original is there @@ -962,7 +965,7 @@ def _handleInterruptedCloneOp(self, origUuid, jval, forceUndo=False): parent = vdis[orig.parentUuid] self.lvActivator.activate(parent.uuid, parent.lvName, False) origPath = os.path.join(self.path, orig.lvName) - if not vhdutil.check(origPath): + if cowutil.check(origPath) != cowutil.CheckResult.Success: util.SMlog("Orig VHD invalid => revert") self._undoCloneOp(lvs, origUuid, baseUuid, clonUuid) return @@ -971,7 +974,7 @@ def _handleInterruptedCloneOp(self, origUuid, jval, forceUndo=False): clon = vdis[clonUuid] clonPath = os.path.join(self.path, clon.lvName) self.lvActivator.activate(clonUuid, clon.lvName, False) - if not vhdutil.check(clonPath): + if cowutil.check(clonPath) != cowutil.CheckResult.Success: util.SMlog("Clon VHD invalid => revert") self._undoCloneOp(lvs, origUuid, baseUuid, clonUuid) return @@ -991,13 +994,16 @@ def _undoCloneOp(self, lvs, origUuid, baseUuid, clonUuid): origRefcountBinary = RefCounter.check(origUuid, ns)[1] origRefcountNormal = 0 + # TODO + cowutil = None + # un-hide the parent if VdiType.isCowImage(base.vdiType): self.lvActivator.activate(baseUuid, base.name, False) origRefcountNormal = 1 - vhdInfo = vhdutil.getVHDInfo(basePath, lvhdutil.extractUuid, False) - if vhdInfo.hidden: - vhdutil.setHidden(basePath, False) + cow_info = cowutil.getInfo(basePath, lvhdutil.extractUuid, False) + if cow_info.hidden: + cowutil.setHidden(basePath, False) elif base.hidden: self.lvmCache.setHidden(base.name, False) @@ -1048,13 +1054,15 @@ def _completeCloneOp(self, vdis, origUuid, baseUuid, clonUuid): cleanup.abort(self.uuid) + cowutil = None # TODO + # make sure the parent is hidden and read-only if not base.hidden: if not VdiType.isCowImage(base.vdiType): self.lvmCache.setHidden(base.lvName) else: basePath = os.path.join(self.path, base.lvName) - vhdutil.setHidden(basePath) + cowutil.setHidden(basePath) if not base.lvReadonly: self.lvmCache.setReadonly(base.lvName, True) @@ -1197,6 +1205,8 @@ def _undoAllVHDJournals(self): if len(journals) == 0: return self._loadvdis() + # TODO + cowutil = None for uuid, jlvName in journals: vdi = self.vdis[uuid] util.SMlog("Found VHD journal %s, reverting %s" % (uuid, vdi.path)) @@ -1206,13 +1216,15 @@ def _undoAllVHDJournals(self): lvhdutil.inflate(self.journaler, self.uuid, vdi.uuid, fullSize) try: jFile = os.path.join(self.path, jlvName) - vhdutil.revert(vdi.path, jFile) + cowutil.revert(vdi.path, jFile) except util.CommandException: util.logException("VHD journal revert") - vhdutil.check(vdi.path) + cowutil.check(vdi.path) util.SMlog("VHD revert failed but VHD ok: removing journal") # Attempt to reclaim unused space - vhdInfo = vhdutil.getVHDInfo(vdi.path, lvhdutil.extractUuid, False) + + + vhdInfo = cowutil.getInfo(vdi.path, lvhdutil.extractUuid, False) NewSize = lvhdutil.calcSizeVHDLV(vhdInfo.sizeVirt) if NewSize < fullSize: lvhdutil.deflate(self.lvmCache, vdi.lvname, int(NewSize)) @@ -1374,7 +1386,9 @@ def create(self, sr_uuid, vdi_uuid, size) -> str: if self.exists: raise xs_errors.XenError('VDIExists') - size = vhdutil.validate_and_round_vhd_size(int(size)) + self._cowutil = None # TODO + + size = self._cowutil.validateAndRoundImageSize(int(size)) util.SMlog("LVHDVDI.create: type = %s, %s (size=%s)" % \ (self.vdi_type, self.path, size)) @@ -1385,7 +1399,7 @@ def create(self, sr_uuid, vdi_uuid, size) -> str: else: if self.sr.provision == "thin": lvSize = util.roundup(lvutil.LVM_SIZE_INCREMENT, - vhdutil.calcOverheadEmpty(lvhdutil.MSIZE)) + self._cowutil.calcOverheadEmpty(lvhdutil.MSIZE)) elif self.sr.provision == "thick": lvSize = lvhdutil.calcSizeVHDLV(int(size)) @@ -1396,8 +1410,8 @@ def create(self, sr_uuid, vdi_uuid, size) -> str: if not VdiType.isCowImage(self.vdi_type): self.size = self.sr.lvmCache.getSize(self.lvname) else: - vhdutil.create(self.path, int(size), False, lvhdutil.MSIZE_MB) - self.size = vhdutil.getSizeVirt(self.path) + self._cowutil.create(self.path, int(size), False, lvhdutil.MSIZE_MB) + self.size = self._cowutil.getSizeVirt(self.path) self.sr.lvmCache.deactivateNoRefcount(self.lvname) except util.CommandException as e: util.SMlog("Unable to create VDI") @@ -1555,7 +1569,7 @@ def resize(self, sr_uuid, vdi_uuid, size) -> str: '(current size: %d, new size: %d)' % (self.size, size)) raise xs_errors.XenError('VDISize', opterr='shrinking not allowed') - size = vhdutil.validate_and_round_vhd_size(int(size)) + size = self._cowutil.validateAndRoundImageSize(int(size)) if size == self.size: return VDI.VDI.get_params(self) @@ -1582,8 +1596,8 @@ def resize(self, sr_uuid, vdi_uuid, size) -> str: if lvSizeNew != lvSizeOld: lvhdutil.inflate(self.sr.journaler, self.sr.uuid, self.uuid, lvSizeNew) - vhdutil.setSizeVirtFast(self.path, size) - self.size = vhdutil.getSizeVirt(self.path) + self._cowutil.setSizeVirtFast(self.path, size) + self.size = self._cowutil.getSizeVirt(self.path) self.utilisation = self.sr.lvmCache.getSize(self.lvname) vdi_ref = self.sr.srcmd.params['vdi_ref'] @@ -1613,8 +1627,8 @@ def compose(self, sr_uuid, vdi1, vdi2) -> None: self.sr.lvActivator.activate(self.uuid, self.lvname, False) self.sr.lvActivator.activate(parent_uuid, parent_lvname, False) - vhdutil.setParent(self.path, parent_path, False) - vhdutil.setHidden(parent_path) + self._cowutil.setParent(self.path, parent_path, False) + self._cowutil.setHidden(parent_path) self.sr.session.xenapi.VDI.set_managed(self.sr.srcmd.params['args'][0], False) if not blktap2.VDI.tap_refresh(self.session, self.sr.uuid, self.uuid, @@ -1631,11 +1645,11 @@ def reset_leaf(self, sr_uuid, vdi_uuid): self.sr.lvActivator.activate(self.uuid, self.lvname, False) # safety check - if not vhdutil.hasParent(self.path): + if not self._cowutil.hasParent(self.path): raise util.SMException("ERROR: VDI %s has no parent, " + \ "will not reset contents" % self.uuid) - vhdutil.killData(self.path) + self._cowutil.killData(self.path) def _attach(self): self._chainSetActive(True, True, True) @@ -1723,11 +1737,11 @@ def _snapshot(self, snapType, cloneOp=False, cbtlog=None, cbt_consistency=None): opterr='VDI unavailable: %s' % (self.path)) if VdiType.isCowImage(self.vdi_type): - depth = vhdutil.getDepth(self.path) + depth = self._cowutil.getDepth(self.path) if depth == -1: raise xs_errors.XenError('VDIUnavailable', \ opterr='failed to get VHD depth') - elif depth >= vhdutil.MAX_CHAIN_SIZE: + elif depth >= self._cowutil.getMaxChainLength(): raise xs_errors.XenError('SnapshotChainTooLong') self.issnap = self.session.xenapi.VDI.get_is_a_snapshot( \ @@ -1735,7 +1749,7 @@ def _snapshot(self, snapType, cloneOp=False, cbtlog=None, cbt_consistency=None): fullpr = lvhdutil.calcSizeVHDLV(self.size) thinpr = util.roundup(lvutil.LVM_SIZE_INCREMENT, \ - vhdutil.calcOverheadEmpty(lvhdutil.MSIZE)) + self._cowutil.calcOverheadEmpty(lvhdutil.MSIZE)) lvSizeOrig = thinpr lvSizeClon = thinpr @@ -1816,7 +1830,7 @@ def _snapshot(self, snapType, cloneOp=False, cbtlog=None, cbt_consistency=None): if not VdiType.isCowImage(self.vdi_type): self.sr.lvmCache.setHidden(self.lvname) else: - vhdutil.setHidden(self.path) + self._cowutil.setHidden(self.path) util.fistpoint.activate("LVHDRT_clone_vdi_after_parent_hidden", self.sr.uuid) # set the base copy to ReadOnly @@ -1860,8 +1874,8 @@ def _createSnap(self, snapUuid, snapSizeLV, isNew): RefCounter.set(snapUuid, 1, 0, lvhdutil.NS_PREFIX_LVM + self.sr.uuid) self.sr.lvActivator.add(snapUuid, snapLV, False) parentRaw = (self.vdi_type == VdiType.RAW) - vhdutil.snapshot(snapPath, self.path, parentRaw, lvhdutil.MSIZE_MB) - snapParent = vhdutil.getParent(snapPath, lvhdutil.extractUuid) + self._cowutil.snapshot(snapPath, self.path, parentRaw, lvhdutil.MSIZE_MB) + snapParent = self._cowutil.getParent(snapPath, lvhdutil.extractUuid) snapVDI = LVHDVDI(self.sr, snapUuid) snapVDI.read_only = False @@ -2068,15 +2082,15 @@ def _determineType(self): # LVM commands can be costly, so check the file directly first in case # the LV is active found = False - for t in lvhdutil.VDI_TYPES: - lvname = "%s%s" % (lvhdutil.LV_PREFIX[t], self.uuid) + for vdiType, prefix in lvhdutil.LV_PREFIX: + lvname = "%s%s" % (prefix, self.uuid) path = os.path.join(self.sr.path, lvname) if util.pathexists(path): if found: raise xs_errors.XenError('VDILoad', opterr="multiple VDI's: uuid %s" % self.uuid) found = True - self.vdi_type = t + self.vdi_type = vdiType self.lvname = lvname self.path = path if found: @@ -2110,7 +2124,7 @@ def _loadThis(self): self._initFromLVInfo(lvs[self.uuid]) if VdiType.isCowImage(self.vdi_type): self.sr.lvActivator.activate(self.uuid, self.lvname, False) - vhdInfo = vhdutil.getVHDInfo(self.path, lvhdutil.extractUuid, False) + vhdInfo = self._cowutil.getInfo(self.path, lvhdutil.extractUuid, False) if not vhdInfo: raise xs_errors.XenError('VDIUnavailable', \ opterr='getVHDInfo failed') @@ -2154,7 +2168,7 @@ def _markHidden(self): if not VdiType.isCowImage(self.vdi_type): self.sr.lvmCache.setHidden(self.lvname) else: - vhdutil.setHidden(self.path) + self._cowutil.setHidden(self.path) self.hidden = 1 def _prepareThin(self, attach): diff --git a/drivers/LinstorSR.py b/drivers/LinstorSR.py index f80437d8b..6648c49b2 100755 --- a/drivers/LinstorSR.py +++ b/drivers/LinstorSR.py @@ -305,6 +305,8 @@ class LinstorSR(SR.SR): # SR methods. # -------------------------------------------------------------------------- + _linstor: Optional[LinstorVolumeManager] = None + @override @staticmethod def handles(type) -> bool: @@ -1286,7 +1288,7 @@ def _get_vdi_path_and_parent(self, vdi_uuid, volume_name): return (device_path, None) # Otherwise it's a VHD and a parent can exist. - if not self._vhdutil.check(vdi_uuid): + if self._cowutil.check(vdi_uuid) != cowutil.CheckResult.Success: return (None, None) vhd_info = self._vhdutil.get_vhd_info(vdi_uuid) @@ -1660,8 +1662,10 @@ def create(self, sr_uuid, vdi_uuid, size) -> str: assert self.ty assert self.vdi_type + self._cowutil = None # TODO + # 2. Compute size and check space available. - size = vhdutil.validate_and_round_vhd_size(int(size)) + size = self._cowutil.validateAndRoundImageSize(int(size)) volume_size = LinstorVhdUtil.compute_volume_size(size, self.vdi_type) util.SMlog( 'LinstorVDI.create: type={}, vhd-size={}, volume-size={}' @@ -1692,16 +1696,18 @@ def create(self, sr_uuid, vdi_uuid, size) -> str: self._update_device_name(volume_info.name) + self.cowutil = None # TODO + if not VdiType.isCowImage(self.vdi_type): self.size = volume_info.virtual_size else: - self.sr._vhdutil.create( + self._cowutil.create( self.path, size, False, self.MAX_METADATA_VIRT_SIZE ) - self.size = self.sr._vhdutil.get_size_virt(self.uuid) + self.size = self._cowutil.get_size_virt(self.uuid) if self._key_hash: - self.sr._vhdutil.set_key(self.path, self._key_hash) + self._cowutil.set_key(self.path, self._key_hash) # TODO: Check supported before call # Because vhdutil commands modify the volume data, # we must retrieve a new time the utilization size. @@ -1933,7 +1939,7 @@ def resize(self, sr_uuid, vdi_uuid, size) -> str: raise xs_errors.XenError('VDIUnavailable', opterr='hidden VDI') # Compute the virtual VHD and DRBD volume size. - size = vhdutil.validate_and_round_vhd_size(int(size)) + size = self._cowutil.validateAndRoundImageSize(int(size)) volume_size = LinstorVhdUtil.compute_volume_size(size, self.vdi_type) util.SMlog( 'LinstorVDI.resize: type={}, vhd-size={}, volume-size={}' @@ -2286,12 +2292,12 @@ def _create_snapshot(self, snap_uuid, snap_of_uuid=None): # 2. Write the snapshot content. is_raw = (self.vdi_type == VdiType.RAW) - self.sr._vhdutil.snapshot( + self._cowutil.snapshot( snap_path, self.path, is_raw, self.MAX_METADATA_VIRT_SIZE ) # 3. Get snapshot parent. - snap_parent = self.sr._vhdutil.get_parent(snap_uuid) + snap_parent = self._cowutil.get_parent(snap_uuid) # 4. Update metadata. util.SMlog('Set VDI {} metadata of snapshot'.format(snap_uuid)) @@ -2384,7 +2390,7 @@ def _snapshot(self, snap_type, cbtlog=None, cbt_consistency=None): 'VDIUnavailable', opterr='failed to get VHD depth' ) - elif depth >= vhdutil.MAX_CHAIN_SIZE: + elif depth >= self._cowutil.getMaxChainLength(): raise xs_errors.XenError('SnapshotChainTooLong') # Ensure we have a valid path if we don't have a local diskful. diff --git a/drivers/blktap2.py b/drivers/blktap2.py index 1a84f11be..693e4bdb3 100755 --- a/drivers/blktap2.py +++ b/drivers/blktap2.py @@ -47,7 +47,6 @@ import nfs import resetvdis -import vhdutil import lvhdutil import VDI as sm @@ -668,7 +667,7 @@ def get_tapdisk(self): class Tapdisk(object): - TYPES = ['aio', 'vhd'] + TYPES = ['aio', 'vhd', 'qcow2'] def __init__(self, pid, minor, _type, path, state): self.pid = pid @@ -1090,6 +1089,7 @@ def _tap_type(vdi_type): return { 'raw': 'aio', 'vhd': 'vhd', + 'qcow2': 'qcow2', 'iso': 'aio', # for ISO SR 'aio': 'aio', # for LVHD 'file': 'aio', @@ -1120,7 +1120,8 @@ def __str__(self) -> str: 'aio': 'tap', # for LVHD raw nodes 'iso': 'tap', # for ISOSR 'file': 'tap', - 'vhd': 'tap'} + 'vhd': 'tap', + 'qcow2': 'tap'} def tap_wanted(self): # 1. Let the target vdi_type decide @@ -1929,7 +1930,7 @@ def _setup_cache(self, session, sr_uuid, vdi_uuid, local_sr_uuid, from lock import Lock from FileSR import FileVDI - parent_uuid = vhdutil.getParent(self.target.vdi.path, + parent_uuid = self._cowutil.getParent(self.target.vdi.path, FileVDI.extractUuid) if not parent_uuid: util.SMlog("ERROR: VDI %s has no parent, not enabling" % \ @@ -1958,14 +1959,14 @@ def _setup_cache(self, session, sr_uuid, vdi_uuid, local_sr_uuid, read_cache_path) else: try: - vhdutil.snapshot(read_cache_path, shared_target.path, False) + self._cowutil.snapshot(read_cache_path, shared_target.path, False) except util.CommandException as e: util.SMlog("Error creating parent cache: %s" % e) self.alert_no_cache(session, vdi_uuid, local_sr_uuid, e.code) return None # local write node - leaf_size = vhdutil.getSizeVirt(self.target.vdi.path) + leaf_size = self._cowutil.getSizeVirt(self.target.vdi.path) local_leaf_path = "%s/%s.vhdcache" % \ (local_sr.path, self.target.vdi.uuid) if util.pathexists(local_leaf_path): @@ -1973,18 +1974,18 @@ def _setup_cache(self, session, sr_uuid, vdi_uuid, local_sr_uuid, local_leaf_path) os.unlink(local_leaf_path) try: - vhdutil.snapshot(local_leaf_path, read_cache_path, False, + self._cowutil.snapshot(local_leaf_path, read_cache_path, False, msize=leaf_size // 1024 // 1024, checkEmpty=False) except util.CommandException as e: util.SMlog("Error creating leaf cache: %s" % e) self.alert_no_cache(session, vdi_uuid, local_sr_uuid, e.code) return None - local_leaf_size = vhdutil.getSizeVirt(local_leaf_path) + local_leaf_size = self._cowutil.getSizeVirt(local_leaf_path) if leaf_size > local_leaf_size: util.SMlog("Leaf size %d > local leaf cache size %d, resizing" % (leaf_size, local_leaf_size)) - vhdutil.setSizeVirtFast(local_leaf_path, leaf_size) + self._cowutil.setSizeVirtFast(local_leaf_path, leaf_size) vdi_type = self.target.get_vdi_type() @@ -2076,7 +2077,7 @@ def _remove_cache(self, session, local_sr_uuid): from lock import Lock from FileSR import FileVDI - parent_uuid = vhdutil.getParent(self.target.vdi.path, + parent_uuid = self._cowutil.getParent(self.target.vdi.path, FileVDI.extractUuid) if not parent_uuid: util.SMlog("ERROR: No parent for VDI %s, ignore" % \ diff --git a/drivers/cleanup.py b/drivers/cleanup.py index 5abf672da..718724556 100755 --- a/drivers/cleanup.py +++ b/drivers/cleanup.py @@ -37,7 +37,6 @@ import XenAPI # pylint: disable=import-error import util import lvutil -import vhdutil import lvhdutil import lvmcache import journaler @@ -51,7 +50,9 @@ from srmetadata import LVMMetadataHandler, VDI_TYPE_TAG from functools import reduce from time import monotonic as _time -from vditype import VdiType, VdiTypeExtension, VDI_TYPE_TO_EXTENSION + +from cowutil import getCowUtil +from vditype import VdiType, VdiTypeExtension, VDI_COW_TYPES, VDI_TYPE_TO_EXTENSION try: from linstorjournaler import LinstorJournaler @@ -548,6 +549,7 @@ def __init__(self, sr, uuid, vdi_type): self.parent = None self.children = [] self._vdiRef = None + self.cowutil = None # TODO: SET self._clearRef() @staticmethod @@ -770,10 +772,10 @@ def delete(self) -> None: self._clear() def getParent(self) -> str: - return vhdutil.getParent(self.path, lambda x: x.strip()) + return self.cowutil.getParent(self.path, lambda x: x.strip()) def repair(self, parent) -> None: - vhdutil.repair(parent) + self.cowutil.repair(parent) @override def __str__(self) -> str: @@ -798,7 +800,7 @@ def __str__(self) -> str: strSizePhys, strSizeAllocated, strType) def validate(self, fast=False) -> None: - if not vhdutil.check(self.path, fast=fast): + if not self.cowutil.check(self.path, fast=fast) != cowutil.CheckResult.Success: raise util.SMException("VHD %s corrupted" % self) def _clear(self): @@ -895,7 +897,7 @@ def _reportCoalesceError(vdi, ce): def coalesce(self) -> int: # size is returned in sectors - return vhdutil.coalesce(self.path) * 512 + return self.cowutil.coalesce(self.path) * 512 @staticmethod def _doCoalesceCowImage(vdi): @@ -1021,12 +1023,12 @@ def _tagChildrenForRelink(self): child._tagChildrenForRelink() def _loadInfoParent(self): - ret = vhdutil.getParent(self.path, lvhdutil.extractUuid) + ret = self.cowutil.getParent(self.path, lvhdutil.extractUuid) if ret: self.parentUuid = ret def _setParent(self, parent) -> None: - vhdutil.setParent(self.path, parent.path, False) + self.cowutil.setParent(self.path, parent.path, False) self.parent = parent self.parentUuid = parent.uuid parent.children.append(self) @@ -1039,11 +1041,11 @@ def _setParent(self, parent) -> None: (self.uuid, self.parentUuid)) def _loadInfoHidden(self) -> None: - hidden = vhdutil.getHidden(self.path) + hidden = self.cowutil.getHidden(self.path) self.hidden = (hidden != 0) def _setHidden(self, hidden=True) -> None: - vhdutil.setHidden(self.path, hidden) + self.cowutil.setHidden(self.path, hidden) self.hidden = hidden def _increaseSizeVirt(self, size, atomic=True) -> None: @@ -1058,9 +1060,9 @@ def _increaseSizeVirt(self, size, atomic=True) -> None: Util.log(" Expanding VHD virt size for VDI %s: %s -> %s" % \ (self, Util.num2str(self.sizeVirt), Util.num2str(size))) - msize = vhdutil.getMaxResizeSize(self.path) * 1024 * 1024 + msize = self.cowutil.getMaxResizeSize(self.path) if (size <= msize): - vhdutil.setSizeVirtFast(self.path, size) + self.cowutil.setSizeVirtFast(self.path, size) else: if atomic: vdiList = self._getAllSubtree() @@ -1076,17 +1078,17 @@ def _increaseSizeVirt(self, size, atomic=True) -> None: else: self._setSizeVirt(size) - self.sizeVirt = vhdutil.getSizeVirt(self.path) + self.sizeVirt = self.cowutil.getSizeVirt(self.path) def _setSizeVirt(self, size) -> None: """WARNING: do not call this method directly unless all VDIs in the subtree are guaranteed to be unplugged (and remain so for the duration of the operation): this operation is only safe for offline VHDs""" jFile = os.path.join(self.sr.path, self.uuid) - vhdutil.setSizeVirt(self.path, size, jFile) + self.cowutil.setSizeVirt(self.path, size, jFile) def _queryVHDBlocks(self) -> bytes: - return vhdutil.getBlockBitmap(self.path) + return self.cowutil.getBlockBitmap(self.path) def _getCoalescedSizeData(self): """Get the data size of the resulting VHD if we coalesce self onto @@ -1100,14 +1102,14 @@ def _getCoalescedSizeData(self): blocksParent = self.parent.getVDIBlocks() numBlocks = Util.countBits(blocksChild, blocksParent) Util.log("Num combined blocks = %d" % numBlocks) - sizeData = numBlocks * vhdutil.VHD_BLOCK_SIZE + sizeData = numBlocks * self.cowutil.getBlockSize(self.path) assert(sizeData <= self.sizeVirt) return sizeData def _calcExtraSpaceForCoalescing(self) -> int: sizeData = self._getCoalescedSizeData() - sizeCoalesced = sizeData + vhdutil.calcOverheadBitmap(sizeData) + \ - vhdutil.calcOverheadEmpty(self.sizeVirt) + sizeCoalesced = sizeData + self.cowutil.calcOverheadBitmap(sizeData) + \ + self.cowutil.calcOverheadEmpty(self.sizeVirt) Util.log("Coalesced size = %s" % Util.num2str(sizeCoalesced)) return sizeCoalesced - self.parent.getSizePhys() @@ -1121,7 +1123,7 @@ def _calcExtraSpaceForSnapshotCoalescing(self) -> int: """How much extra space in the SR will be required to snapshot-coalesce this VDI""" return self._calcExtraSpaceForCoalescing() + \ - vhdutil.calcOverheadEmpty(self.sizeVirt) # extra snap leaf + self.cowutil.calcOverheadEmpty(self.sizeVirt) # extra snap leaf def _getAllSubtree(self): """Get self and all VDIs in the subtree of self as a flat list""" @@ -1155,7 +1157,7 @@ def load(self, info=None) -> None: if not util.pathexists(self.path): raise util.SMException("%s not found" % self.path) try: - info = vhdutil.getVHDInfo(self.path, self.extractUuid) + info = self.cowutil.getInfo(self.path, self.extractUuid) except util.SMException: Util.log(" [VDI %s: failed to read VHD metadata]" % self.uuid) return @@ -1199,7 +1201,7 @@ def delete(self) -> None: @override def getAllocatedSize(self) -> int: if self._sizeAllocated == -1: - self._sizeAllocated = vhdutil.getAllocatedSize(self.path) + self._sizeAllocated = self.cowutil.getAllocatedSize(self.path) return self._sizeAllocated @@ -1325,7 +1327,7 @@ def _loadInfoSizePhys(self): if not VdiType.isCowImage(self.vdi_type): return self._activate() - self._sizePhys = vhdutil.getSizePhys(self.path) + self._sizePhys = self.cowutil.getSizePhys(self.path) if self._sizePhys <= 0: raise util.SMException("phys size of %s = %d" % \ (self, self._sizePhys)) @@ -1343,7 +1345,7 @@ def _loadInfoSizeAllocated(self): if not VdiType.isCowImage(self.vdi_type): return self._activate() - self._sizeAllocated = vhdutil.getAllocatedSize(self.path) + self._sizeAllocated = self.cowutil.getAllocatedSize(self.path) @override def _loadInfoHidden(self) -> None: @@ -1409,7 +1411,7 @@ def _setParent(self, parent) -> None: self.sr.lvmCache.setReadonly(self.fileName, False) try: - vhdutil.setParent(self.path, parent.path, parent.vdi_type == VdiType.RAW) + self.cowutil.setParent(self.path, parent.path, parent.vdi_type == VdiType.RAW) finally: if self.lvReadonly: self.sr.lvmCache.setReadonly(self.fileName, True) @@ -1480,8 +1482,7 @@ def _setSizeVirt(self, size) -> None: subtree are guaranteed to be unplugged (and remain so for the duration of the operation): this operation is only safe for offline VHDs""" self._activate() - jFile = lvhdutil.createVHDJournalLV(self.sr.lvmCache, self.uuid, - vhdutil.MAX_VHD_JOURNAL_SIZE) + jFile = lvhdutil.createVHDJournalLV(self.sr.lvmCache, self.uuid, self.cowutil.getResizeJournalSize()) try: lvhdutil.setSizeVirt(self.sr.journaler, self.sr.uuid, self.uuid, size, jFile) @@ -1546,6 +1547,7 @@ def load(self, info=None) -> None: self.drbd_size = -1 self.hidden = info.hidden self.scanError = False + self.vdi_type = VdiType.VHD @override @@ -1624,7 +1626,7 @@ def delete(self) -> None: @override def validate(self, fast=False) -> None: - if VdiType.isCowImage(self.vdi_type) and not self.sr._vhdutil.check(self.uuid, fast=fast): + if VdiType.isCowImage(self.vdi_type) and self.cowutil.check(self.uuid, fast=fast) != cowutil.CheckResult.Success: raise util.SMException('VHD {} corrupted'.format(self)) @override @@ -1729,7 +1731,7 @@ def _setHidden(self, hidden=True) -> None: def _setSizeVirt(self, size) -> None: jfile = self.uuid + '-jvhd' self.sr._linstor.create_volume( - jfile, vhdutil.MAX_VHD_JOURNAL_SIZE, persistent=False, volume_name=jfile + jfile, self.cowutil.getResizeJournalSize(), persistent=False, volume_name=jfile ) try: self.inflate(LinstorVhdUtil.compute_volume_size(size, self.vdi_type)) @@ -2651,15 +2653,20 @@ def __init__(self, uuid, xapi, createLock, force): def scan(self, force=False) -> None: if not util.pathexists(self.path): raise util.SMException("directory %s not found!" % self.uuid) - vhds = self._scan(force) - for uuid, vhdInfo in vhds.items(): - vdi = self.getVDI(uuid) - if not vdi: - self.logFilter.logNewVDI(uuid) - vdi = FileVDI(self, uuid, VdiType.VHD) - self.vdis[uuid] = vdi - vdi.load(vhdInfo) - uuidsPresent = list(vhds.keys()) + + uuidsPresent: list[str] = [] + + for vdi_type in VDI_COW_TYPES: + scan_result = self._scan(vdi_type, force) + for uuid, image_info in scan_result.items(): + vdi = self.getVDI(uuid) + if not vdi: + self.logFilter.logNewVDI(uuid) + vdi = FileVDI(self, uuid, vdi_type) + self.vdis[uuid] = vdi + vdi.load(image_info) + uuidsPresent.extend(scan_result.keys()) + rawList = [x for x in os.listdir(self.path) if x.endswith(VdiTypeExtension.RAW)] for rawName in rawList: uuid = FileVDI.extractUuid(rawName) @@ -2763,20 +2770,20 @@ def _isCacheFileName(self, name): return (len(name) == Util.UUID_LEN + len(self.CACHE_FILE_EXT)) and \ name.endswith(self.CACHE_FILE_EXT) - def _scan(self, force): + def _scan(self, vdi_type, force): for i in range(SR.SCAN_RETRY_ATTEMPTS): error = False - pattern = os.path.join(self.path, "*%s" % VdiTypeExtension.VHD) - vhds = vhdutil.getAllVHDs(pattern, FileVDI.extractUuid) - for uuid, vhdInfo in vhds.items(): + pattern = os.path.join(self.path, "*%s" % VDI_TYPE_TO_EXTENSION[vdi_type]) + scan_result = getCowUtil(vdi_type).getAllInfoFromVG(pattern, FileVDI.extractUuid) + for uuid, vhdInfo in scan_result.items(): if vhdInfo.error: error = True break if not error: - return vhds + return scan_result Util.log("Scan error on attempt %d" % i) if force: - return vhds + return scan_result raise util.SMException("Scan error") @override @@ -2846,7 +2853,7 @@ def _undoInterruptedCoalesceLeaf(self, childUuid, parentUuid): child.rename(childUuid) Util.log("Updating the VDI record") child.setConfig(VDI.DB_VDI_PARENT, parentUuid) - child.setConfig(VDI.DB_VDI_TYPE, VdiType.VHD) + child.setConfig(VDI.DB_VDI_TYPE, child.vdi_type) util.fistpoint.activate("LVHDRT_coaleaf_undo_after_rename2", self.uuid) if child.hidden: @@ -3066,7 +3073,7 @@ def _undoInterruptedCoalesceLeaf(self, childUuid, parentUuid): child.rename(childUuid) Util.log("Updating the VDI record") child.setConfig(VDI.DB_VDI_PARENT, parentUuid) - child.setConfig(VDI.DB_VDI_TYPE, VdiType.VHD) + child.setConfig(VDI.DB_VDI_TYPE, child.vdi_type) util.fistpoint.activate("LVHDRT_coaleaf_undo_after_rename2", self.uuid) # refcount (best effort - assume that it had succeeded if the @@ -3431,7 +3438,7 @@ def _undoInterruptedCoalesceLeaf(self, childUuid, parentUuid): child.rename(childUuid) Util.log('Updating the VDI record') child.setConfig(VDI.DB_VDI_PARENT, parentUuid) - child.setConfig(VDI.DB_VDI_TYPE, VdiType.VHD) + child.setConfig(VDI.DB_VDI_TYPE, child.vdi_type) # TODO: Maybe deflate here. diff --git a/drivers/cowutil.py b/drivers/cowutil.py new file mode 100755 index 000000000..000d47a4a --- /dev/null +++ b/drivers/cowutil.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 Vates SAS +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from sm_typing import Any, Callable, Dict, Final, List, Optional, Sequence, Union, override + +from abc import ABC, abstractmethod +from enum import IntEnum + +import errno +import time + +import util + +from vditype import VdiType + +# ------------------------------------------------------------------------------ + +IMAGE_FORMAT_COW_FLAG: Final = 1 << 8 + +class ImageFormat(IntEnum): + RAW = 1 + VHD = 2 | IMAGE_FORMAT_COW_FLAG + QCOW2 = 3 | IMAGE_FORMAT_COW_FLAG + +IMAGE_FORMAT_TO_STR: Final = { + ImageFormat.RAW: "raw", + ImageFormat.VHD: "vhd", + ImageFormat.QCOW2: "qcow2" +} + +STR_TO_IMAGE_FORMAT = {v: k for k, v in IMAGE_FORMAT_TO_STR.items()} + +# ------------------------------------------------------------------------------ + +def parseImageFormats(str_formats: Optional[str], default_formats: List[ImageFormat]) -> List[ImageFormat]: + if not str_formats: + return default_formats + + entries = [entry.strip() for entry in str_formats.split(",")] + + image_formats: List[ImageFormat] = [] + for entry in entries: + image_format = STR_TO_IMAGE_FORMAT.get(entry) + if image_format: + image_formats.append(image_format) + + if image_formats: + return image_formats + + return default_formats + +# ------------------------------------------------------------------------------ + +class CowImageInfo(object): + uuid = "" + path = "" + sizeVirt = -1 + sizePhys = -1 + sizeAllocated = -1 + hidden = False + parentUuid = "" + parentPath = "" + error: Any = 0 + + def __init__(self, uuid): + self.uuid = uuid + +# ------------------------------------------------------------------------------ + +class CowUtil(ABC): + class CheckResult(IntEnum): + Success = 0 + Fail = 1 + Unavailable = 2 + + @abstractmethod + def getMinImageSize(self) -> int: + pass + + @abstractmethod + def getMaxImageSize(self) -> int: + pass + + @abstractmethod + def getBlockSize(self, path: str) -> int: + pass + + @abstractmethod + def getFooterSize(self, path: str) -> int: + pass + + @abstractmethod + def getMaxChainLength(self) -> int: + pass + + @abstractmethod + def calcOverheadEmpty(self, virtual_size: int) -> int: + pass + + @abstractmethod + def calcOverheadBitmap(self, virtual_size: int) -> int: + pass + + @abstractmethod + def getInfo( + self, + path: str, + extractUuidFunction: Callable[[str], str], + includeParent: bool = True, + resolveParent: bool = True, + useBackupFooter: bool = False + ) -> CowImageInfo: + pass + + @abstractmethod + def getInfoFromLVM( + self, lvName: str, extractUuidFunction: Callable[[str], str], vgName: str + ) -> Optional[CowImageInfo]: + pass + + @abstractmethod + def getAllInfoFromVG( + self, + pattern: str, + extractUuidFunction: Callable[[str], str], + vgName: Optional[str] = None, + parents: bool = False, + exitOnError: bool = False + ) -> Dict[str, CowImageInfo]: + pass + + @abstractmethod + def getParent(self, path: str, extractUuidFunction: Callable[[str], str]) -> Optional[str]: + pass + + @abstractmethod + def getParentNoCheck(self, path: str) -> Optional[str]: + pass + + @abstractmethod + def hasParent(self, path: str) -> bool: + pass + + @abstractmethod + def setParent(self, path: str, parentPath: str, parentRaw: bool) -> None: + pass + + @abstractmethod + def getHidden(self, path: str) -> bool: + pass + + @abstractmethod + def setHidden(self, path: str, hidden: bool = True) -> None: + pass + + @abstractmethod + def getSizeVirt(self, path: str) -> int: + pass + + @abstractmethod + def setSizeVirt(self, path: str, size: int, jFile: str) -> None: + pass + + @abstractmethod + def setSizeVirtFast(self, path: str, size: int) -> None: + pass + + @abstractmethod + def getMaxResizeSize(self, path: str) -> int: + pass + + @abstractmethod + def getSizePhys(self, path: str) -> int: + pass + + @abstractmethod + def setSizePhys(self, path: str, size: int, debug: bool = True) -> None: + pass + + @abstractmethod + def getAllocatedSize(self, path: str) -> int: + pass + + @abstractmethod + def getResizeJournalSize(self) -> int: + pass + + @abstractmethod + def killData(self, path: str) -> None: + pass + + @abstractmethod + def getDepth(self, path: str) -> int: + pass + + @abstractmethod + def getBlockBitmap(self, path: str) -> bytes: + pass + + @abstractmethod + def coalesce(self, path: str) -> int: + pass + + @abstractmethod + def create(self, path: str, size: int, static: bool, msize: int = 0) -> None: + pass + + @abstractmethod + def snapshot( + self, + path: str, + parent: str, + parentRaw: bool, + msize: int = 0, + checkEmpty: Optional[bool] = True + ) -> None: + pass + + @abstractmethod + def check( + self, + path: str, + ignoreMissingFooter: Optional[bool] = False, + fast: Optional[bool] = False + ) -> CheckResult: + pass + + @abstractmethod + def revert(self, path: str, jFile: str) -> None: + pass + + @abstractmethod + def repair(self, path: str) -> None: + pass + + @abstractmethod + def validateAndRoundImageSize(self, size: int) -> int: + pass + + @abstractmethod + def getKeyHash(self, path: str) -> Optional[str]: + pass + + @abstractmethod + def setKey(self, path: str, key_hash: str) -> None: + pass + + def getParentChain(self, lvName: str, extractUuidFunction: Callable[[str], str], vgName: str) -> Dict[str, str]: + """ + Get the chain of all parents of 'path'. Safe to call for raw VDI's as well. + """ + chain = {} + vdis: Dict[str, CowImageInfo] = {} + retries = 0 + while (not vdis): + if retries > 60: + util.SMlog('ERROR: getAllInfoFromVG returned 0 VDIs after %d retries' % retries) + util.SMlog('ERROR: the image metadata might be corrupted') + break + vdis = self.getAllInfoFromVG(lvName, extractUuidFunction, vgName, True, True) + if (not vdis): + retries = retries + 1 + time.sleep(1) + for uuid, vdi in vdis.items(): + chain[uuid] = vdi.path + #util.SMlog("Parent chain for %s: %s" % (lvName, chain)) + return chain + + @staticmethod + def isCowImage(image_format: ImageFormat) -> bool: + return bool(image_format & IMAGE_FORMAT_COW_FLAG) + + @staticmethod + def _ioretry(cmd: Sequence[str], text: bool = True) -> Union[str, bytes]: + return util.ioretry( + lambda: util.pread2(cmd, text=text), + errlist=[errno.EIO, errno.EAGAIN] + ) + +# ------------------------------------------------------------------------------ + +def getImageFormatFromVdiType(vdi_type: str) -> ImageFormat: + if vdi_type == VdiType.RAW: + return ImageFormat.RAW + if vdi_type == VdiType.VHD: + return ImageFormat.VHD + if vdi_type == VdiType.QCOW2: + return ImageFormat.QCOW2 + + assert False, f"Unsupported vdi type: {vdi_type}" + +def getVdiTypeFromImageFormat(image_format: ImageFormat) -> str: + if image_format == ImageFormat.RAW: + return VdiType.RAW + if image_format == ImageFormat.VHD: + return VdiType.VHD + if image_format == ImageFormat.QCOW2: + return VdiType.QCOW2 + + assert False, f"Unsupported image format: {IMAGE_FORMAT_TO_STR[image_format]}" + +def getCowUtil(vdi_type: str) -> CowUtil: + import vhdutil + + if getImageFormatFromVdiType(vdi_type) in (ImageFormat.RAW, ImageFormat.VHD): + return vhdutil.VhdUtil() + + assert False, f"Unsupported VDI type: {vdi_type}" diff --git a/drivers/linstor-manager b/drivers/linstor-manager index 4366057e6..e3bea1278 100755 --- a/drivers/linstor-manager +++ b/drivers/linstor-manager @@ -399,7 +399,7 @@ def check(session, args): raise -def get_vhd_info(session, args): +def get_vhd_info(session, args): # TODO: rename try: device_path = args['devicePath'] group_name = args['groupName'] @@ -411,16 +411,19 @@ def get_vhd_info(session, args): logger=util.SMlog ) + # TODO + cowutil = None + def extract_uuid(device_path): # TODO: Remove new line in the vhdutil module. Not here. return linstor.get_volume_uuid_from_device_path( device_path.rstrip('\n') ) - vhd_info = vhdutil.getVHDInfo( + cow_info = cowutil.getInfo( device_path, extract_uuid, include_parent, False ) - return json.dumps(vhd_info.__dict__) + return json.dumps(cow_info.__dict__) except Exception as e: util.SMlog('linstor-manager:get_vhd_info error: {}'.format(e)) raise @@ -429,7 +432,8 @@ def get_vhd_info(session, args): def has_parent(session, args): try: device_path = args['devicePath'] - return str(vhdutil.hasParent(device_path)) + cowutil = None + return str(cowutil.hasParent(device_path)) except Exception as e: util.SMlog('linstor-manager:has_parent error: {}'.format(e)) raise @@ -446,13 +450,15 @@ def get_parent(session, args): logger=util.SMlog ) + cowutil = None # TODO + def extract_uuid(device_path): # TODO: Remove new line in the vhdutil module. Not here. return linstor.get_volume_uuid_from_device_path( device_path.rstrip('\n') ) - return vhdutil.getParent(device_path, extract_uuid) + return cowutil.getParent(device_path, extract_uuid) except Exception as e: util.SMlog('linstor-manager:get_parent error: {}'.format(e)) raise @@ -461,7 +467,8 @@ def get_parent(session, args): def get_size_virt(session, args): try: device_path = args['devicePath'] - return str(vhdutil.getSizeVirt(device_path)) + cowutil = None # TODO + return str(cowutil.getSizeVirt(device_path)) except Exception as e: util.SMlog('linstor-manager:get_size_virt error: {}'.format(e)) raise @@ -479,7 +486,8 @@ def get_size_phys(session, args): def get_allocated_size(session, args): try: device_path = args['devicePath'] - return str(vhdutil.getAllocatedSize(device_path)) + cowutil = None + return str(cowutil.getAllocatedSize(device_path)) except Exception as e: util.SMlog('linstor-manager:get_allocated_size error: {}'.format(e)) raise @@ -488,7 +496,8 @@ def get_allocated_size(session, args): def get_depth(session, args): try: device_path = args['devicePath'] - return str(vhdutil.getDepth(device_path)) + cowutil = None + return str(cowutil.getDepth(device_path)) except Exception as e: util.SMlog('linstor-manager:get_depth error: {}'.format(e)) raise @@ -497,7 +506,8 @@ def get_depth(session, args): def get_key_hash(session, args): try: device_path = args['devicePath'] - return vhdutil.getKeyHash(device_path) or '' + cowutil = None + return cowutil.getKeyHash(device_path) or '' except Exception as e: util.SMlog('linstor-manager:get_key_hash error: {}'.format(e)) raise @@ -506,7 +516,8 @@ def get_key_hash(session, args): def get_block_bitmap(session, args): try: device_path = args['devicePath'] - return base64.b64encode(vhdutil.getBlockBitmap(device_path)).decode('ascii') + cowutil = None + return base64.b64encode(cowutil.getBlockBitmap(device_path)).decode('ascii') except Exception as e: util.SMlog('linstor-manager:get_block_bitmap error: {}'.format(e)) raise @@ -528,7 +539,8 @@ def set_parent(session, args): try: device_path = args['devicePath'] parent_path = args['parentPath'] - vhdutil.setParent(device_path, parent_path, False) + cowutil = None # TODO + cowutil.setParent(device_path, parent_path, False) return '' except Exception as e: util.SMlog('linstor-manager:set_parent error: {}'.format(e)) @@ -538,7 +550,8 @@ def set_parent(session, args): def coalesce(session, args): try: device_path = args['devicePath'] - return str(vhdutil.coalesce(device_path)) + cowutil = None + return str(cowutil.coalesce(device_path)) except Exception as e: util.SMlog('linstor-manager:coalesce error: {}'.format(e)) raise diff --git a/drivers/linstorvhdutil.py b/drivers/linstorvhdutil.py index b3069b43f..1df3b8a1f 100644 --- a/drivers/linstorvhdutil.py +++ b/drivers/linstorvhdutil.py @@ -16,16 +16,18 @@ from sm_typing import override -from linstorjournaler import LinstorJournaler -from linstorvolumemanager import LinstorVolumeManager import base64 import errno import json import socket import time + +from cowutil import CowImageInfo, CowUtil, getCowUtil import util -import vhdutil import xs_errors + +from linstorjournaler import LinstorJournaler +from linstorvolumemanager import LinstorVolumeManager from vditype import VdiType MANAGER_PLUGIN = 'linstor-manager' @@ -49,14 +51,20 @@ def call_remote_method(session, host_ref, method, device_path, args): return response + def check_ex(path, ignoreMissingFooter = False, fast = False): - cmd = [vhdutil.VHD_UTIL, "check", vhdutil.OPT_LOG_ERR, "-n", path] - if ignoreMissingFooter: - cmd.append("-i") - if fast: - cmd.append("-B") + cowutil = None # TODO + result = self._cowutil.check(path, ignoreMissingFooter, fast) + if result == CowUtil.CheckResult.Success: + return True + if result == CowUtil.CheckResult.Fail: + return False - vhdutil.ioretry(cmd) + if not os.path.exists(path): + raise ErofsLinstorCallException(e) # Break retry calls. + if e.code == errno.ENOENT: + raise NoPathLinstorCallException(e) + raise e class LinstorCallException(util.SMException): @@ -148,7 +156,8 @@ def wrapper(*args, **kwargs): class LinstorVhdUtil: MAX_SIZE = 2 * 1024 * 1024 * 1024 * 1024 # Max VHD size. - def __init__(self, session, linstor): + def __init__(self, session, linstor, vdi_type: str): + self._cowutil = getCowUtil(vdi_type) self._session = session self._linstor = linstor @@ -219,53 +228,53 @@ def get_vhd_info(self, vdi_uuid, include_parent=True): # https://github.com/PyCQA/pylint/pull/2926 return self._get_vhd_info(vdi_uuid, self._extract_uuid, **kwargs) # pylint: disable = E1123 - @linstorhostcall(vhdutil.getVHDInfo, 'getVHDInfo') + @linstorhostcall(CowUtil.getInfo, 'getInfo') def _get_vhd_info(self, vdi_uuid, response): obj = json.loads(response) - vhd_info = vhdutil.VHDInfo(vdi_uuid) - vhd_info.sizeVirt = obj['sizeVirt'] - vhd_info.sizePhys = obj['sizePhys'] + image_info = CowImageInfo(vdi_uuid) + image_info.sizeVirt = obj['sizeVirt'] + image_info.sizePhys = obj['sizePhys'] if 'parentPath' in obj: - vhd_info.parentPath = obj['parentPath'] - vhd_info.parentUuid = obj['parentUuid'] - vhd_info.hidden = obj['hidden'] - vhd_info.path = obj['path'] + image_info.parentPath = obj['parentPath'] + image_info.parentUuid = obj['parentUuid'] + image_info.hidden = obj['hidden'] + image_info.path = obj['path'] - return vhd_info + return image_info - @linstorhostcall(vhdutil.hasParent, 'hasParent') + @linstorhostcall(CowUtil.hasParent, 'hasParent') def has_parent(self, vdi_uuid, response): return util.strtobool(response) def get_parent(self, vdi_uuid): return self._get_parent(vdi_uuid, self._extract_uuid) - @linstorhostcall(vhdutil.getParent, 'getParent') + @linstorhostcall(CowUtil.getParent, 'getParent') def _get_parent(self, vdi_uuid, response): return response - @linstorhostcall(vhdutil.getSizeVirt, 'getSizeVirt') + @linstorhostcall(CowUtil.getSizeVirt, 'getSizeVirt') def get_size_virt(self, vdi_uuid, response): return int(response) - @linstorhostcall(vhdutil.getSizePhys, 'getSizePhys') + @linstorhostcall(CowUtil.getSizePhys, 'getSizePhys') def get_size_phys(self, vdi_uuid, response): return int(response) - @linstorhostcall(vhdutil.getAllocatedSize, 'getAllocatedSize') + @linstorhostcall(CowUtil.getAllocatedSize, 'getAllocatedSize') def get_allocated_size(self, vdi_uuid, response): return int(response) - @linstorhostcall(vhdutil.getDepth, 'getDepth') + @linstorhostcall(CowUtil.getDepth, 'getDepth') def get_depth(self, vdi_uuid, response): return int(response) - @linstorhostcall(vhdutil.getKeyHash, 'getKeyHash') + @linstorhostcall(CowUtil.getKeyHash, 'getKeyHash') def get_key_hash(self, vdi_uuid, response): return response or None - @linstorhostcall(vhdutil.getBlockBitmap, 'getBlockBitmap') + @linstorhostcall(CowUtil.getBlockBitmap, 'getBlockBitmap') def get_block_bitmap(self, vdi_uuid, response): return base64.b64decode(response) @@ -273,7 +282,7 @@ def get_block_bitmap(self, vdi_uuid, response): def get_drbd_size(self, vdi_uuid, response): return int(response) - def _get_drbd_size(self, path): + def _get_drbd_size(self, cowutil_inst, path): (ret, stdout, stderr) = util.doexec(['blockdev', '--getsize64', path]) if ret == 0: return int(stdout.strip()) @@ -285,39 +294,39 @@ def _get_drbd_size(self, path): @linstormodifier() def create(self, path, size, static, msize=0): - return self._call_local_method_or_fail(vhdutil.create, path, size, static, msize) + return self._call_local_method_or_fail(CowUtil.create, path, size, static, msize) @linstormodifier() def set_size_virt(self, path, size, jfile): - return self._call_local_method_or_fail(vhdutil.setSizeVirt, path, size, jfile) + return self._call_local_method_or_fail(CowUtil.setSizeVirt, path, size, jfile) @linstormodifier() def set_size_virt_fast(self, path, size): - return self._call_local_method_or_fail(vhdutil.setSizeVirtFast, path, size) + return self._call_local_method_or_fail(CowUtil.setSizeVirtFast, path, size) @linstormodifier() def set_size_phys(self, path, size, debug=True): - return self._call_local_method_or_fail(vhdutil.setSizePhys, path, size, debug) + return self._call_local_method_or_fail(CowUtil.setSizePhys, path, size, debug) @linstormodifier() def set_parent(self, path, parentPath, parentRaw=False): - return self._call_local_method_or_fail(vhdutil.setParent, path, parentPath, parentRaw) + return self._call_local_method_or_fail(CowUtil.setParent, path, parentPath, parentRaw) @linstormodifier() def set_hidden(self, path, hidden=True): - return self._call_local_method_or_fail(vhdutil.setHidden, path, hidden) + return self._call_local_method_or_fail(CowUtil.setHidden, path, hidden) @linstormodifier() def set_key(self, path, key_hash): - return self._call_local_method_or_fail(vhdutil.setKey, path, key_hash) + return self._call_local_method_or_fail(CowUtil.setKey, path, key_hash) @linstormodifier() def kill_data(self, path): - return self._call_local_method_or_fail(vhdutil.killData, path) + return self._call_local_method_or_fail(CowUtil.killData, path) @linstormodifier() def snapshot(self, path, parent, parentRaw, msize=0, checkEmpty=True): - return self._call_local_method_or_fail(vhdutil.snapshot, path, parent, parentRaw, msize, checkEmpty) + return self._call_local_method_or_fail(CowUtil.snapshot, path, parent, parentRaw, msize, checkEmpty) def inflate(self, journaler, vdi_uuid, vdi_path, new_size, old_size): # Only inflate if the LINSTOR volume capacity is not enough. @@ -344,14 +353,14 @@ def inflate(self, journaler, vdi_uuid, vdi_path, new_size, old_size): .format(new_size, result_size) ) - self._zeroize(vdi_path, result_size - vhdutil.VHD_FOOTER_SIZE) + self._zeroize(vdi_path, result_size - self._cowutil.getFooterSize()) self.set_size_phys(vdi_path, result_size, False) journaler.remove(LinstorJournaler.INFLATE, vdi_uuid) def deflate(self, vdi_path, new_size, old_size, zeroize=False): if zeroize: - assert old_size > vhdutil.VHD_FOOTER_SIZE - self._zeroize(vdi_path, old_size - vhdutil.VHD_FOOTER_SIZE) + assert old_size > self._cowutil.getFooterSize() + self._zeroize(vdi_path, old_size - self._cowutil.getFooterSize()) new_size = LinstorVolumeManager.round_up_volume_size(new_size) if new_size >= old_size: @@ -375,15 +384,15 @@ def force_parent(self, path, parentPath, parentRaw=False): 'parentPath': str(parentPath), 'parentRaw': parentRaw } - return self._call_method(vhdutil.setParent, 'setParent', path, use_parent=False, **kwargs) + return self._call_method(CowUtil.setParent, 'setParent', path, use_parent=False, **kwargs) @linstormodifier() def force_coalesce(self, path): - return int(self._call_method(vhdutil.coalesce, 'coalesce', path, use_parent=True)) + return int(self._call_method(CowUtil.coalesce, 'coalesce', path, use_parent=True)) @linstormodifier() def force_repair(self, path): - return self._call_method(vhdutil.repair, 'repair', path, use_parent=False) + return self._call_method(CowUtil.repair, 'repair', path, use_parent=False) @linstormodifier() def force_deflate(self, path, newSize, oldSize, zeroize): @@ -394,30 +403,26 @@ def force_deflate(self, path, newSize, oldSize, zeroize): } return self._call_method('_force_deflate', 'deflate', path, use_parent=False, **kwargs) - def _force_deflate(self, path, newSize, oldSize, zeroize): + def _force_deflate(self, cowutil_inst, path, newSize, oldSize, zeroize): self.deflate(path, newSize, oldSize, zeroize) # -------------------------------------------------------------------------- - # Static helpers. + # Helpers. # -------------------------------------------------------------------------- @classmethod - def compute_volume_size(cls, virtual_size, image_type): - if VdiType.isCowImage(image_type): + def compute_volume_size(cls, virtual_size, vdi_type: str): + if VdiType.isCowImage(vdi_type): # All LINSTOR VDIs have the metadata area preallocated for # the maximum possible virtual size (for fast online VDI.resize). - meta_overhead = vhdutil.calcOverheadEmpty(cls.MAX_SIZE) - bitmap_overhead = vhdutil.calcOverheadBitmap(virtual_size) + meta_overhead = self._cowutil.calcOverheadEmpty(self.MAX_SIZE) + bitmap_overhead = self._cowutil.calcOverheadBitmap(virtual_size) virtual_size += meta_overhead + bitmap_overhead else: - raise Exception('Invalid image type: {}'.format(image_type)) + raise Exception('Invalid image type: {}'.format(vdi_type)) return LinstorVolumeManager.round_up_volume_size(virtual_size) - # -------------------------------------------------------------------------- - # Helpers. - # -------------------------------------------------------------------------- - def _extract_uuid(self, device_path): # TODO: Remove new line in the vhdutil module. Not here. return self._linstor.get_volume_uuid_from_device_path( @@ -482,7 +487,7 @@ def _call_local_method(self, local_method, device_path, *args, **kwargs): try: def local_call(): try: - return local_method(device_path, *args, **kwargs) + return local_method(self._cowutil, device_path, *args, **kwargs) except util.CommandException as e: if e.code == errno.EROFS or e.code == errno.EMEDIUMTYPE: raise ErofsLinstorCallException(e) # Break retry calls. @@ -576,7 +581,7 @@ def remote_call(): if no_host_found: try: - return local_method(device_path, *args, **kwargs) + return local_method(self._cowutil, device_path, *args, **kwargs) except Exception as e: self._raise_openers_exception(device_path, e) @@ -587,9 +592,8 @@ def remote_call(): ) return util.retry(remote_call, 5, 2) - @staticmethod def _zeroize(path, size): - if not util.zeroOut(path, size, vhdutil.VHD_FOOTER_SIZE): + if not util.zeroOut(path, size, self._cowutil.getFooterSize()): raise xs_errors.XenError( 'EIO', opterr='Failed to zero out VHD footer {}'.format(path) diff --git a/drivers/lvhdutil.py b/drivers/lvhdutil.py index 484260b8a..48f825a6f 100755 --- a/drivers/lvhdutil.py +++ b/drivers/lvhdutil.py @@ -15,7 +15,10 @@ # along with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -"""Helper functions for LVHD SR. This module knows about RAW and VHD VDI's +# TODO: Must be a class +# TODO: CHECK ALL CALLS TO LVHDUTIL FUNCTIONS, USE SELF. + +"""Helper functions for LVHD SR. This module knows about RAW and VHD VDI's that live in LV's.""" import os import sys @@ -26,7 +29,9 @@ import vhdutil from refcounter import RefCounter -from vditype import VdiType + +from cowutil import getCowUtil +from vditype import VdiType, VDI_COW_TYPES MSIZE_MB = 2 * 1024 * 1024 # max virt size for fast resize MSIZE = int(MSIZE_MB * 1024 * 1024) @@ -36,10 +41,13 @@ LVM_SIZE_INCREMENT = 4 * 1024 * 1024 LV_PREFIX = { - VdiType.VHD: "VHD-", - VdiType.RAW: "LV-", + VdiType.RAW: "LV-", + VdiType.VHD: "VHD-", + VdiType.QCOW2: "QCOW2-", } -VDI_TYPES = [VdiType.VHD, VdiType.RAW] +VDI_TYPES = [VdiType.RAW, VdiType.VHD, VdiType.QCOW2] + +LV_PREFIX_TO_VDI_TYPE = {v: k for k, v in LV_PREFIX.items()} JRN_INFLATE = "inflate" @@ -72,8 +80,7 @@ def __init__(self, uuid): def matchLV(lvName): """given LV name, return the VDI type and the UUID, or (None, None) if the name doesn't match any known type""" - for vdiType in VDI_TYPES: - prefix = LV_PREFIX[vdiType] + for vdiType, prefix in LV_PREFIX.items(): if lvName.startswith(prefix): return (vdiType, lvName.replace(prefix, "")) return (None, None) @@ -85,9 +92,9 @@ def extractUuid(path): # we are dealing with realpath uuid = uuid.replace("--", "-") uuid.replace(VG_PREFIX, "") - for t in VDI_TYPES: - if uuid.find(LV_PREFIX[t]) != -1: - uuid = uuid.split(LV_PREFIX[t])[-1] + for prefix in LV_PREFIX.values(): + if uuid.find(prefix) != -1: + uuid = uuid.split(prefix)[-1] uuid = uuid.strip() # TODO: validate UUID format return uuid @@ -98,16 +105,16 @@ def calcSizeLV(sizeVHD): return util.roundup(LVM_SIZE_INCREMENT, sizeVHD) -def calcSizeVHDLV(sizeVirt): +def calcSizeVHDLV(self, sizeVirt): # all LVHD VDIs have the metadata area preallocated for the maximum # possible virtual size (for fast online VDI.resize) - metaOverhead = vhdutil.calcOverheadEmpty(MSIZE) - bitmapOverhead = vhdutil.calcOverheadBitmap(sizeVirt) + metaOverhead = self._cowutil.calcOverheadEmpty(MSIZE) + bitmapOverhead = self._cowutil.calcOverheadBitmap(sizeVirt) return calcSizeLV(sizeVirt + metaOverhead + bitmapOverhead) def getLVInfo(lvmCache, lvName=None): - """Load LV info for all LVs in the VG or an individual LV. + """Load LV info for all LVs in the VG or an individual LV. This is a wrapper for lvutil.getLVInfo that filters out LV's that are not LVHD VDI's and adds the vdi_type information""" allLVs = lvmCache.getLVInfo(lvName) @@ -127,10 +134,10 @@ def getVDIInfo(lvmCache): vdis = {} lvs = getLVInfo(lvmCache) - haveVHDs = False + hasCowVdis = False for uuid, lvInfo in lvs.items(): if VdiType.isCowImage(lvInfo.vdiType): - haveVHDs = True + hasCowVdis = True vdiInfo = VDIInfo(uuid) vdiInfo.vdiType = lvInfo.vdiType vdiInfo.lvName = lvInfo.name @@ -142,14 +149,17 @@ def getVDIInfo(lvmCache): vdiInfo.hidden = lvInfo.hidden vdis[uuid] = vdiInfo - if haveVHDs: - pattern = "%s*" % LV_PREFIX[VdiType.VHD] - vhds = vhdutil.getAllVHDs(pattern, extractUuid, lvmCache.vgName) + if not hasCowVdis: + return vdis + + for vdi_type in VDI_COW_TYPES: + pattern = "%s*" % LV_PREFIX[vdi_type] + vdis = getCowUtil(vdi_type).getAllInfoFromVG(pattern, extractUuid, lvmCache.vgName) uuids = vdis.keys() for uuid in uuids: vdi = vdis[uuid] if VdiType.isCowImage(vdi.vdiType): - if not vhds.get(uuid): + if not vdis.get(uuid): lvmCache.refresh() if lvmCache.checkLV(vdi.lvName): util.SMlog("*** VHD info missing: %s" % uuid) @@ -157,17 +167,17 @@ def getVDIInfo(lvmCache): else: util.SMlog("LV disappeared since last scan: %s" % uuid) del vdis[uuid] - elif vhds[uuid].error: + elif vdis[uuid].error: util.SMlog("*** vhd-scan error: %s" % uuid) vdis[uuid].scanError = True else: - vdis[uuid].sizeVirt = vhds[uuid].sizeVirt - vdis[uuid].parentUuid = vhds[uuid].parentUuid - vdis[uuid].hidden = vhds[uuid].hidden + vdis[uuid].sizeVirt = vdis[uuid].sizeVirt + vdis[uuid].parentUuid = vdis[uuid].parentUuid + vdis[uuid].hidden = vdis[uuid].hidden return vdis -def inflate(journaler, srUuid, vdiUuid, size): +def inflate(self, journaler, srUuid, vdiUuid, size): """Expand a VDI LV (and its VHD) to 'size'. If the LV is already bigger than that, it's a no-op. Does not change the virtual size of the VDI""" lvName = LV_PREFIX[VdiType.VHD] + vdiUuid @@ -187,13 +197,13 @@ def inflate(journaler, srUuid, vdiUuid, size): vhdutil.VHD_FOOTER_SIZE): raise Exception('failed to zero out VHD footer') util.fistpoint.activate("LVHDRT_inflate_after_zeroOut", srUuid) - vhdutil.setSizePhys(path, newSize, False) + self._cowutil.setSizePhys(path, newSize, False) util.fistpoint.activate("LVHDRT_inflate_after_setSizePhys", srUuid) journaler.remove(JRN_INFLATE, vdiUuid) -def deflate(lvmCache, lvName, size): - """Shrink the LV and the VHD on it to 'size'. Does not change the +def deflate(self, lvmCache, lvName, size): + """Shrink the LV and the VHD on it to 'size'. Does not change the virtual size of the VDI""" currSizeLV = lvmCache.getSize(lvName) newSize = calcSizeLV(size) @@ -201,18 +211,18 @@ def deflate(lvmCache, lvName, size): return path = os.path.join(VG_LOCATION, lvmCache.vgName, lvName) # no undo necessary if this fails at any point between now and the end - vhdutil.setSizePhys(path, newSize) + self._cowutil.setSizePhys(path, newSize) lvmCache.setSize(lvName, newSize) -def setSizeVirt(journaler, srUuid, vdiUuid, size, jFile): +def setSizeVirt(self, journaler, srUuid, vdiUuid, size, jFile): """When resizing the VHD virtual size, we might have to inflate the LV in case the metadata size increases""" lvName = LV_PREFIX[VdiType.VHD] + vdiUuid vgName = VG_PREFIX + srUuid path = os.path.join(VG_LOCATION, vgName, lvName) inflate(journaler, srUuid, vdiUuid, calcSizeVHDLV(size)) - vhdutil.setSizeVirt(path, size, jFile) + self._cowutil.setSizeVirt(path, size, jFile) def _tryAcquire(lock): @@ -227,7 +237,7 @@ def _tryAcquire(lock): raise util.SRBusyException() -def attachThin(journaler, srUuid, vdiUuid): +def attachThin(self, journaler, srUuid, vdiUuid): """Ensure that the VDI LV is expanded to the fully-allocated size""" lvName = LV_PREFIX[VdiType.VHD] + vdiUuid vgName = VG_PREFIX + srUuid @@ -235,7 +245,7 @@ def attachThin(journaler, srUuid, vdiUuid): lvmCache = journaler.lvmCache _tryAcquire(sr_lock) lvmCache.refresh() - vhdInfo = vhdutil.getVHDInfoLVM(lvName, extractUuid, vgName) + vhdInfo = self._cowutil.getInfoFromLVM(lvName, extractUuid, vgName) newSize = calcSizeVHDLV(vhdInfo.sizeVirt) currSizeLV = lvmCache.getSize(lvName) if newSize <= currSizeLV: diff --git a/drivers/tapdisk-pause b/drivers/tapdisk-pause index 40c6a71ee..7a17051e1 100755 --- a/drivers/tapdisk-pause +++ b/drivers/tapdisk-pause @@ -2,13 +2,13 @@ # # Copyright (C) Citrix Systems Inc. # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published # by the Free Software Foundation; version 2.1 only. # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License @@ -26,8 +26,8 @@ from lock import Lock import xs_errors import XenAPI import lvhdutil -import vhdutil import lvmcache +from cowutil import getCowUtil from vditype import VdiType try: @@ -88,13 +88,13 @@ def tapPause(session, args): def tapUnpause(session, args): tap = Tapdisk(session, args) return tap.Unpause() - + def tapRefresh(session, args): tap = Tapdisk(session, args) if tap.Pause() != "True": return str(False) return tap.Unpause() - + class Tapdisk: def __init__(self, session, args): @@ -124,20 +124,19 @@ class Tapdisk: realpath = os.readlink(self.phypath) except OSError as e: util.SMlog("Phypath %s does not exist" % self.phypath) - return + return util.SMlog("Realpath: %s" % realpath) if realpath.startswith("/dev/VG_XenStorage-") and \ not os.path.exists(realpath): util.SMlog("Path inconsistent") pfx = "/dev/VG_XenStorage-%s/" % self.sr_uuid - for ty in ["LV","VHD"]: - p = pfx + ty + "-" + self.vdi_uuid + for ty in lvhdutil.LV_PREFIX.values(): + p = pfx + ty + self.vdi_uuid util.SMlog("Testing path: %s" % p) if os.path.exists(p): _mkphylink(self.sr_uuid, self.vdi_uuid, p) self.realpath = p - if ty == "LV": self.vdi_type = VdiType.RAW - else: self.vdi_type = VdiType.VHD + self.vdi_type = LV_PREFIX_TO_VDI_TYPE[ty] elif realpath.startswith('/dev/drbd/by-res/xcp-volume-'): if not LINSTOR_AVAILABLE: raise util.SMException( @@ -203,25 +202,26 @@ class Tapdisk: util.SMlog("No %s: nothing to unpause" % self.path) return str(True) self._pathRefresh() - self.major, self.minor = _getDevMajor_minor(self.path) + self.major, self.minor = _getDevMajor_minor(self.path) if self.major != blktap2.Tapdisk.major(): util.SMlog("Non-tap major number: %d" % self.major) return str(False) + + import VDI + vdi = VDI.VDI.from_uuid(self.session, self.vdi_uuid) + if self.activate_parents: util.SMlog("Activating parents of %s" % self.vdi_uuid) vg_name = lvhdutil.VG_PREFIX + self.sr_uuid ns = lvhdutil.NS_PREFIX_LVM + self.sr_uuid lvm_cache = lvmcache.LVMCache(vg_name) - lv_name = lvhdutil.LV_PREFIX[VdiType.VHD] + self.vdi_uuid - vdi_list = vhdutil.getParentChain(lv_name, - lvhdutil.extractUuid, vg_name) + lv_name = lvhdutil.LV_PREFIX[vdi.vdi_type] + self.vdi_uuid + vdi_list = getCowUtil(vdi.vdi_type).getParentChain(lv_name, lvhdutil.extractUuid, vg_name) for uuid, lv_name in vdi_list.items(): if uuid == self.vdi_uuid: continue lvm_cache.activate(ns, uuid, lv_name, False) - import VDI - vdi = VDI.VDI.from_uuid(self.session, self.vdi_uuid) # Check if CBT is enabled on disk we are about to unpause if vdi._get_blocktracking_status(): logname = vdi._get_cbt_logname(self.vdi_uuid) diff --git a/drivers/vditype.py b/drivers/vditype.py index 3a478c7ab..dcae45c9b 100644 --- a/drivers/vditype.py +++ b/drivers/vditype.py @@ -24,10 +24,10 @@ class VdiType(object): ISO = "iso" FILE = "file" CBTLOG = "cbtlog" - + @classmethod def isCowImage(cls, vdi_type) -> bool: - return vdi_type in (cls.VHD) + return vdi_type in VDI_COW_TYPES # TODO: Use StrEnum in python 3.11. class VdiTypeExtension(object): @@ -38,6 +38,8 @@ class VdiTypeExtension(object): FILE = ".file" CBTLOG = ".cbtlog" +VDI_COW_TYPES: Final = (VdiType.VHD, VdiType.QCOW2) + VDI_TYPE_TO_EXTENSION: Final = { VdiType.RAW: VdiTypeExtension.RAW, VdiType.VHD: VdiTypeExtension.VHD, diff --git a/drivers/verifyVHDsOnSR.py b/drivers/verifyVHDsOnSR.py index e0a56f465..88b6bb9ad 100755 --- a/drivers/verifyVHDsOnSR.py +++ b/drivers/verifyVHDsOnSR.py @@ -54,8 +54,11 @@ def activateVdiChainAndCheck(vhd_info, vg_name): return activated_list activated_list.append([vhd_info.uuid, vhd_path]) + + cowutil = None # TODO + # Do a vhdutil check with -i option, to ignore error in primary - if not vhdutil.check(vhd_path, True): + if cowutil.check(vhd_path, True) != cowutil.CheckResult.Success: util.SMlog("VHD check for %s failed, continuing with the rest!" % vg_name) VHDs_failed += 1 else: @@ -115,7 +118,7 @@ def checkAllVHD(sr_uuid): pattern = "%s*" % lvhdutil.LV_PREFIX[VdiType.VHD] # Do a vhd scan and gets all the VHDs - vhds = vhdutil.getAllVHDs(pattern, lvhdutil.extractUuid, vg_name) + vhds = CowUtil.getAllInfoFromVG(pattern, lvhdutil.extractUuid, vg_name) VHDs_total = len(vhds) # Build VHD chain, that way it will be easier to activate all the VHDs diff --git a/drivers/vhdutil.py b/drivers/vhdutil.py index 5484ad2e2..94219192e 100755 --- a/drivers/vhdutil.py +++ b/drivers/vhdutil.py @@ -16,425 +16,459 @@ # Helper functions pertaining to VHD operations # -import os -import util -import errno -import zlib -import re -import xs_errors -import time -from vditype import VdiType - -MIN_VHD_SIZE = 2 * 1024 * 1024 -MAX_VHD_SIZE = 2040 * 1024 * 1024 * 1024 -MAX_VHD_JOURNAL_SIZE = 6 * 1024 * 1024 # 2MB VHD block size, max 2TB VHD size -MAX_CHAIN_SIZE = 30 # max VHD parent chain size -VHD_UTIL = "/usr/bin/vhd-util" -OPT_LOG_ERR = "--debug" -VHD_BLOCK_SIZE = 2 * 1024 * 1024 -VHD_FOOTER_SIZE = 512 - -# lock to lock the entire SR for short ops -LOCK_TYPE_SR = "sr" - - -class VHDInfo: - uuid = "" - path = "" - sizeVirt = -1 - sizePhys = -1 - sizeAllocated = -1 - hidden = False - parentUuid = "" - parentPath = "" - error = 0 - - def __init__(self, uuid): - self.uuid = uuid - - -def calcOverheadEmpty(virtual_size): - """Calculate the VHD space overhead (metadata size) for an empty VDI of - size virtual_size""" - overhead = 0 - size_mb = virtual_size // (1024 * 1024) - - # Footer + footer copy + header + possible CoW parent locator fields - overhead = 3 * 1024 - - # BAT 4 Bytes per block segment - overhead += (size_mb // 2) * 4 - overhead = util.roundup(512, overhead) - - # BATMAP 1 bit per block segment - overhead += (size_mb // 2) // 8 - overhead = util.roundup(4096, overhead) - - return overhead - - -def calcOverheadBitmap(virtual_size): - num_blocks = virtual_size // VHD_BLOCK_SIZE - if virtual_size % VHD_BLOCK_SIZE: - num_blocks += 1 - return num_blocks * 4096 - - -def ioretry(cmd, text=True): - return util.ioretry(lambda: util.pread2(cmd, text=text), - errlist=[errno.EIO, errno.EAGAIN]) - - -def convertAllocatedSizeToBytes(size): - # Assume we have standard 2MB allocation blocks - return size * 2 * 1024 * 1024 - - -def getVHDInfo(path, extractUuidFunction, includeParent=True, resolveParent=True): - """Get the VHD info. The parent info may optionally be omitted: vhd-util - tries to verify the parent by opening it, which results in error if the VHD - resides on an inactive LV""" - opts = "-vsaf" - if includeParent: - opts += "p" - if not resolveParent: - opts += "u" - - cmd = [VHD_UTIL, "query", OPT_LOG_ERR, opts, "-n", path] - ret = ioretry(cmd) - fields = ret.strip().split('\n') - uuid = extractUuidFunction(path) - vhdInfo = VHDInfo(uuid) - vhdInfo.sizeVirt = int(fields[0]) * 1024 * 1024 - vhdInfo.sizePhys = int(fields[1]) - nextIndex = 2 - if includeParent: - if fields[nextIndex].find("no parent") == -1: - vhdInfo.parentPath = fields[nextIndex] - vhdInfo.parentUuid = extractUuidFunction(fields[nextIndex]) - nextIndex += 1 - vhdInfo.hidden = int(fields[nextIndex].replace("hidden: ", "")) - vhdInfo.sizeAllocated = convertAllocatedSizeToBytes(int(fields[nextIndex+1])) - vhdInfo.path = path - return vhdInfo - - -def getVHDInfoLVM(lvName, extractUuidFunction, vgName): - """Get the VHD info. This function does not require the container LV to be - active, but uses lvs & vgs""" - vhdInfo = None - cmd = [VHD_UTIL, "scan", "-f", "-l", vgName, "-m", lvName] - ret = ioretry(cmd) - return _parseVHDInfo(ret, extractUuidFunction) - - -def getAllVHDs(pattern, extractUuidFunction, vgName=None, \ - parentsOnly=False, exitOnError=False): - vhds = dict() - cmd = [VHD_UTIL, "scan", "-f", "-m", pattern] - if vgName: - cmd.append("-l") - cmd.append(vgName) - if parentsOnly: - cmd.append("-a") - try: - ret = ioretry(cmd) - except Exception as e: - util.SMlog("WARN: vhd scan failed: output: %s" % e) - ret = ioretry(cmd + ["-c"]) - util.SMlog("WARN: vhd scan with NOFAIL flag, output: %s" % ret) - for line in ret.split('\n'): - if len(line.strip()) == 0: - continue - vhdInfo = _parseVHDInfo(line, extractUuidFunction) - if vhdInfo: - if vhdInfo.error != 0 and exitOnError: - # Just return an empty dict() so the scan will be done - # again by getParentChain. See CA-177063 for details on - # how this has been discovered during the stress tests. - return dict() - vhds[vhdInfo.uuid] = vhdInfo - else: - util.SMlog("WARN: vhdinfo line doesn't parse correctly: %s" % line) - return vhds - - -def getParentChain(lvName, extractUuidFunction, vgName): - """Get the chain of all VHD parents of 'path'. Safe to call for raw VDI's - as well""" - chain = dict() - vdis = dict() - retries = 0 - while (not vdis): - if retries > 60: - util.SMlog('ERROR: getAllVHDs returned 0 VDIs after %d retries' % retries) - util.SMlog('ERROR: the VHD metadata might be corrupted') - break - vdis = getAllVHDs(lvName, extractUuidFunction, vgName, True, True) - if (not vdis): - retries = retries + 1 - time.sleep(1) - for uuid, vdi in vdis.items(): - chain[uuid] = vdi.path - #util.SMlog("Parent chain for %s: %s" % (lvName, chain)) - return chain - - -def getParent(path, extractUuidFunction): - cmd = [VHD_UTIL, "query", OPT_LOG_ERR, "-p", "-n", path] - ret = ioretry(cmd) - if ret.find("query failed") != -1 or ret.find("Failed opening") != -1: - raise util.SMException("VHD query returned %s" % ret) - if ret.find("no parent") != -1: - return None - return extractUuidFunction(ret) - - -def hasParent(path): - """Check if the VHD has a parent. A VHD has a parent iff its type is - 'Differencing'. This function does not need the parent to actually - be present (e.g. the parent LV to be activated).""" - cmd = [VHD_UTIL, "read", OPT_LOG_ERR, "-p", "-n", path] - ret = ioretry(cmd) - # pylint: disable=no-member - m = re.match(r".*Disk type\s+: (\S+) hard disk.*", ret, flags=re.S) - vhd_type = m.group(1) - assert(vhd_type == "Differencing" or vhd_type == "Dynamic") - return vhd_type == "Differencing" - - -def setParent(path, parentPath, parentRaw): - normpath = os.path.normpath(parentPath) - cmd = [VHD_UTIL, "modify", OPT_LOG_ERR, "-p", normpath, "-n", path] - if parentRaw: - cmd.append("-m") - ioretry(cmd) - - -def getHidden(path): - cmd = [VHD_UTIL, "query", OPT_LOG_ERR, "-f", "-n", path] - ret = ioretry(cmd) - hidden = int(ret.split(':')[-1].strip()) - return hidden +from sm_typing import Callable, Dict, Final, Optional, Sequence, cast, override +from abc import abstractmethod -def setHidden(path, hidden=True): - opt = "1" - if not hidden: - opt = "0" - cmd = [VHD_UTIL, "set", OPT_LOG_ERR, "-n", path, "-f", "hidden", "-v", opt] - ret = ioretry(cmd) - - -def getSizeVirt(path): - cmd = [VHD_UTIL, "query", OPT_LOG_ERR, "-v", "-n", path] - ret = ioretry(cmd) - size = int(ret) * 1024 * 1024 - return size - - -def setSizeVirt(path, size, jFile): - "resize VHD offline" - size_mb = size // (1024 * 1024) - cmd = [VHD_UTIL, "resize", OPT_LOG_ERR, "-s", str(size_mb), "-n", path, - "-j", jFile] - ioretry(cmd) - - -def setSizeVirtFast(path, size): - "resize VHD online" - size_mb = size // (1024 * 1024) - cmd = [VHD_UTIL, "resize", OPT_LOG_ERR, "-s", str(size_mb), "-n", path, "-f"] - ioretry(cmd) - - -def getMaxResizeSize(path): - """get the max virtual size for fast resize""" - cmd = [VHD_UTIL, "query", OPT_LOG_ERR, "-S", "-n", path] - ret = ioretry(cmd) - return int(ret) - - -def getSizePhys(path): - cmd = [VHD_UTIL, "query", OPT_LOG_ERR, "-s", "-n", path] - ret = ioretry(cmd) - return int(ret) - - -def setSizePhys(path, size, debug=True): - "set physical utilisation (applicable to VHD's on fixed-size files)" - if debug: - cmd = [VHD_UTIL, "modify", OPT_LOG_ERR, "-s", str(size), "-n", path] - else: - cmd = [VHD_UTIL, "modify", "-s", str(size), "-n", path] - ioretry(cmd) - - -def getAllocatedSize(path): - cmd = [VHD_UTIL, "query", OPT_LOG_ERR, '-a', '-n', path] - ret = ioretry(cmd) - return convertAllocatedSizeToBytes(int(ret)) - -def killData(path): - "zero out the disk (kill all data inside the VHD file)" - cmd = [VHD_UTIL, "modify", OPT_LOG_ERR, "-z", "-n", path] - ioretry(cmd) - - -def getDepth(path): - "get the VHD parent chain depth" - cmd = [VHD_UTIL, "query", OPT_LOG_ERR, "-d", "-n", path] - text = ioretry(cmd) - depth = -1 - if text.startswith("chain depth:"): - depth = int(text.split(':')[1].strip()) - return depth - - -def getBlockBitmap(path): - cmd = [VHD_UTIL, "read", OPT_LOG_ERR, "-B", "-n", path] - text = ioretry(cmd, text=False) - return zlib.compress(text) - - -def coalesce(path): - """ - Coalesce the VHD, on success it returns the number of sectors coalesced - """ - cmd = [VHD_UTIL, "coalesce", OPT_LOG_ERR, "-n", path] - text = ioretry(cmd) - match = re.match(r'^Coalesced (\d+) sectors', text) - if match: - return int(match.group(1)) - - return 0 +import errno +import os +import re +import zlib +from cowutil import CowImageInfo, CowUtil +import util +import XenAPI # pylint: disable=import-error +import xs_errors -def create(path, size, static, msize=0): - size_mb = size // (1024 * 1024) - cmd = [VHD_UTIL, "create", OPT_LOG_ERR, "-n", path, "-s", str(size_mb)] - if static: - cmd.append("-r") - if msize: - cmd.append("-S") - cmd.append(str(msize)) - ioretry(cmd) - - -def snapshot(path, parent, parentRaw, msize=0, checkEmpty=True): - cmd = [VHD_UTIL, "snapshot", OPT_LOG_ERR, "-n", path, "-p", parent] - if parentRaw: - cmd.append("-m") - if msize: - cmd.append("-S") - cmd.append(str(msize)) - if not checkEmpty: - cmd.append("-e") - ioretry(cmd) - - -def check(path, ignoreMissingFooter=False, fast=False): - cmd = [VHD_UTIL, "check", OPT_LOG_ERR, "-n", path] - if ignoreMissingFooter: - cmd.append("-i") - if fast: - cmd.append("-B") - try: - ioretry(cmd) - return True - except util.CommandException: - return False - - -def revert(path, jFile): - cmd = [VHD_UTIL, "revert", OPT_LOG_ERR, "-n", path, "-j", jFile] - ioretry(cmd) - - -def _parseVHDInfo(line, extractUuidFunction): - vhdInfo = None - valueMap = line.split() - if len(valueMap) < 1 or valueMap[0].find("vhd=") == -1: - return None - for keyval in valueMap: - (key, val) = keyval.split('=') - if key == "vhd": - uuid = extractUuidFunction(val) - if not uuid: - util.SMlog("***** malformed output, no UUID: %s" % valueMap) - return None - vhdInfo = VHDInfo(uuid) - vhdInfo.path = val - elif key == "scan-error": - vhdInfo.error = line - util.SMlog("***** VHD scan error: %s" % line) - break - elif key == "capacity": - vhdInfo.sizeVirt = int(val) - elif key == "size": - vhdInfo.sizePhys = int(val) - elif key == "hidden": - vhdInfo.hidden = int(val) - elif key == "parent" and val != "none": - vhdInfo.parentPath = val - vhdInfo.parentUuid = extractUuidFunction(val) - return vhdInfo - - -def _getVHDParentNoCheck(path): - cmd = ["vhd-util", "read", "-p", "-n", "%s" % path] - text = util.pread(cmd) - util.SMlog(text) - for line in text.split('\n'): - if line.find("decoded name :") != -1: - val = line.split(':')[1].strip() - vdi = val.replace("--", "-")[-40:] - if vdi[1:].startswith("LV-"): - vdi = vdi[1:] - return vdi - return None - - -def repair(path): - """Repairs the VHD.""" - ioretry([VHD_UTIL, 'repair', '-n', path]) - - -def validate_and_round_vhd_size(size): - """ Take the supplied vhd size, in bytes, and check it is positive and less - that the maximum supported size, rounding up to the next block boundary - """ - if size < 0 or size > MAX_VHD_SIZE: - raise xs_errors.XenError( - 'VDISize', opterr='VDI size ' + - 'must be between 1 MB and %d MB' % - (MAX_VHD_SIZE // (1024 * 1024))) - - if size < MIN_VHD_SIZE: - size = MIN_VHD_SIZE - - size = util.roundup(VHD_BLOCK_SIZE, size) - - return size - - -def getKeyHash(path): - """Extract the hash of the encryption key from the header of an encrypted VHD""" - cmd = ["vhd-util", "key", "-p", "-n", path] - ret = ioretry(cmd) - ret = ret.strip() - if ret == 'none': +# ------------------------------------------------------------------------------ + +MIN_VHD_SIZE: Final = 2 * 1024 * 1024 +MAX_VHD_SIZE: Final = 2040 * 1024 * 1024 * 1024 + +MAX_VHD_JOURNAL_SIZE: Final = 6 * 1024 * 1024 # 2MB VHD block size, max 2TB VHD size. + +VHD_BLOCK_SIZE: Final = 2 * 1024 * 1024 + +VHD_FOOTER_SIZE: Final = 512 + +MAX_VHD_CHAIN_LENGTH: Final = 30 + +VHD_UTIL: Final = "/usr/bin/vhd-util" + +OPT_LOG_ERR: Final = "--debug" + +# ------------------------------------------------------------------------------ + +class VhdUtil(CowUtil): + @override + def getMinImageSize(self) -> int: + return MIN_VHD_SIZE + + @override + def getMaxImageSize(self) -> int: + return MAX_VHD_SIZE + + @override + def getBlockSize(self, path: str) -> int: + return VHD_BLOCK_SIZE + + @override + def getFooterSize(self, path: str) -> int: + return VHD_FOOTER_SIZE + + @override + def getMaxChainLength(self) -> int: + return MAX_VHD_CHAIN_LENGTH + + @override + def calcOverheadEmpty(self, virtual_size: int) -> int: + """ + Calculate the VHD space overhead (metadata size) for an empty VDI of + size virtual_size. + """ + overhead = 0 + size_mb = virtual_size // (1024 * 1024) + + # Footer + footer copy + header + possible CoW parent locator fields + overhead = 3 * 1024 + + # BAT 4 Bytes per block segment + overhead += (size_mb // 2) * 4 + overhead = util.roundup(512, overhead) + + # BATMAP 1 bit per block segment + overhead += (size_mb // 2) // 8 + overhead = util.roundup(4096, overhead) + + return overhead + + @override + def calcOverheadBitmap(self, virtual_size: int) -> int: + num_blocks = virtual_size // VHD_BLOCK_SIZE + if virtual_size % VHD_BLOCK_SIZE: + num_blocks += 1 + return num_blocks * 4096 + + @override + def getInfo( + self, + path: str, + extractUuidFunction: Callable[[str], str], + includeParent: bool = True, + resolveParent: bool = True, + useBackupFooter: bool = False + ) -> CowImageInfo: + """ + Get the VHD info. The parent info may optionally be omitted: vhd-util + tries to verify the parent by opening it, which results in error if the VHD + resides on an inactive LV. + """ + opts = "-vsaf" + if includeParent: + opts += "p" + if not resolveParent: + opts += "u" + if useBackupFooter: + opts += "b" + + ret = cast(str, self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, opts, "-n", path])) + fields = ret.strip().split("\n") + uuid = extractUuidFunction(path) + vhdInfo = CowImageInfo(uuid) + vhdInfo.sizeVirt = int(fields[0]) * 1024 * 1024 + vhdInfo.sizePhys = int(fields[1]) + nextIndex = 2 + if includeParent: + if fields[nextIndex].find("no parent") == -1: + vhdInfo.parentPath = fields[nextIndex] + vhdInfo.parentUuid = extractUuidFunction(fields[nextIndex]) + nextIndex += 1 + vhdInfo.hidden = bool(int(fields[nextIndex].replace("hidden: ", ""))) + vhdInfo.sizeAllocated = self._convertAllocatedSizeToBytes(int(fields[nextIndex+1])) + vhdInfo.path = path + return vhdInfo + + @override + def getInfoFromLVM( + self, lvName: str, extractUuidFunction: Callable[[str], str], vgName: str + ) -> Optional[CowImageInfo]: + """ + Get the VHD info. This function does not require the container LV to be + active, but uses LVs & VGs. + """ + ret = cast(str, self._ioretry([VHD_UTIL, "scan", "-f", "-l", vgName, "-m", lvName])) + return self._parseVHDInfo(ret, extractUuidFunction) + + @override + def getAllInfoFromVG( + self, + pattern: str, + extractUuidFunction: Callable[[str], str], + vgName: Optional[str] = None, + parents: bool = False, + exitOnError: bool = False + ) -> Dict[str, CowImageInfo]: + result: Dict[str, CowImageInfo] = dict() + cmd = [VHD_UTIL, "scan", "-f", "-m", pattern] + if vgName: + cmd.append("-l") + cmd.append(vgName) + if parents: + cmd.append("-a") + try: + ret = cast(str, self._ioretry(cmd)) + except Exception as e: + util.SMlog("WARN: VHD scan failed: output: %s" % e) + ret = cast(str, self._ioretry(cmd + ["-c"])) + util.SMlog("WARN: VHD scan with NOFAIL flag, output: %s" % ret) + for line in ret.split('\n'): + if not line.strip(): + continue + info = self._parseVHDInfo(line, extractUuidFunction) + if info: + if info.error != 0 and exitOnError: + # Just return an empty dict() so the scan will be done + # again by getParentChain. See CA-177063 for details on + # how this has been discovered during the stress tests. + return dict() + result[info.uuid] = info + else: + util.SMlog("WARN: VHD info line doesn't parse correctly: %s" % line) + return result + + @override + def getParent(self, path: str, extractUuidFunction: Callable[[str], str]) -> Optional[str]: + ret = cast(str, self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, "-p", "-n", path])) + if ret.find("query failed") != -1 or ret.find("Failed opening") != -1: + raise util.SMException("VHD query returned %s" % ret) + if ret.find("no parent") != -1: + return None + return extractUuidFunction(ret) + + @override + def getParentNoCheck(self, path: str) -> Optional[str]: + text = util.pread([VHD_UTIL, "read", "-p", "-n", "%s" % path]) + util.SMlog(text) + for line in text.split("\n"): + if line.find("decoded name :") != -1: + val = line.split(":")[1].strip() + vdi = val.replace("--", "-")[-40:] + if vdi[1:].startswith("LV-"): + vdi = vdi[1:] + return vdi return None - vals = ret.split() - if len(vals) != 2: - util.SMlog('***** malformed output from vhd-util' - ' for VHD {}: "{}"'.format(path, ret)) - return None - [_nonce, key_hash] = vals - return key_hash - -def setKey(path, key_hash): - """Set the encryption key for a VHD""" - cmd = ["vhd-util", "key", "-s", "-n", path, "-H", key_hash] - ioretry(cmd) + @override + def hasParent(self, path: str) -> bool: + """ + Check if the VHD has a parent. A VHD has a parent iff its type is + 'Differencing'. This function does not need the parent to actually + be present (e.g. the parent LV to be activated). + """ + ret = cast(str, self._ioretry([VHD_UTIL, "read", OPT_LOG_ERR, "-p", "-n", path])) + # pylint: disable=no-member + m = re.match(r".*Disk type\s+: (\S+) hard disk.*", ret, flags=re.S) + if m: + vhd_type = m.group(1) + assert vhd_type == "Differencing" or vhd_type == "Dynamic" + return vhd_type == "Differencing" + assert False, f"Ill-formed {VHD_UTIL} output detected during VHD parent parsing" + + @override + def setParent(self, path: str, parentPath: str, parentRaw: bool) -> None: + normpath = os.path.normpath(parentPath) + cmd = [VHD_UTIL, "modify", OPT_LOG_ERR, "-p", normpath, "-n", path] + if parentRaw: + cmd.append("-m") + self._ioretry(cmd) + + @override + def getHidden(self, path: str) -> bool: + ret = cast(str, self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, "-f", "-n", path])) + return bool(int(ret.split(":")[-1].strip())) + + @override + def setHidden(self, path: str, hidden: bool = True) -> None: + opt = "1" + if not hidden: + opt = "0" + self._ioretry([VHD_UTIL, "set", OPT_LOG_ERR, "-n", path, "-f", "hidden", "-v", opt]) + + @override + def getSizeVirt(self, path: str) -> int: + ret = self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, "-v", "-n", path]) + return int(ret) * 1024 * 1024 + + @override + def setSizeVirt(self, path: str, size: int, jFile: str) -> None: + """ + Resize VHD offline + """ + size_mb = size // (1024 * 1024) + self._ioretry([VHD_UTIL, "resize", OPT_LOG_ERR, "-s", str(size_mb), "-n", path, "-j", jFile]) + + @override + def setSizeVirtFast(self, path: str, size: int) -> None: + """ + Resize VHD online. + """ + size_mb = size // (1024 * 1024) + self._ioretry([VHD_UTIL, "resize", OPT_LOG_ERR, "-s", str(size_mb), "-n", path, "-f"]) + + @override + def getMaxResizeSize(self, path: str) -> int: + """ + Get the max virtual size for fast resize. + """ + ret = self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, "-S", "-n", path]) + return int(ret) * 1024 * 1024 + + @override + def getSizePhys(self, path: str) -> int: + return int(self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, "-s", "-n", path])) + + @override + def setSizePhys(self, path: str, size: int, debug: bool = True) -> None: + """ + Set physical utilisation (applicable to VHD's on fixed-size files). + """ + if debug: + cmd = [VHD_UTIL, "modify", OPT_LOG_ERR, "-s", str(size), "-n", path] + else: + cmd = [VHD_UTIL, "modify", "-s", str(size), "-n", path] + self._ioretry(cmd) + + @override + def getAllocatedSize(self, path: str) -> int: + ret = self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, "-a", "-n", path]) + return self._convertAllocatedSizeToBytes(int(ret)) + + @override + def getResizeJournalSize(self) -> int: + return MAX_VHD_JOURNAL_SIZE + + @override + def killData(self, path: str) -> None: + """ + Zero out the disk (kill all data inside the VHD file). + """ + self._ioretry([VHD_UTIL, "modify", OPT_LOG_ERR, "-z", "-n", path]) + + @override + def getDepth(self, path: str) -> int: + """ + Get the VHD parent chain depth. + """ + text = cast(str, self._ioretry([VHD_UTIL, "query", OPT_LOG_ERR, "-d", "-n", path])) + depth = -1 + if text.startswith("chain depth:"): + depth = int(text.split(":")[1].strip()) + return depth + + @override + def getBlockBitmap(self, path: str) -> bytes: + text = cast(bytes, self._ioretry([VHD_UTIL, "read", OPT_LOG_ERR, "-B", "-n", path], text=False)) + return zlib.compress(text) + + @override + def coalesce(self, path: str) -> int: + """ + Coalesce the VHD, on success it returns the number of sectors coalesced. + """ + text = cast(str, self._ioretry([VHD_UTIL, "coalesce", OPT_LOG_ERR, "-n", path])) + match = re.match(r"^Coalesced (\d+) sectors", text) + if match: + return int(match.group(1)) + return 0 + + @override + def create(self, path: str, size: int, static: bool, msize: int = 0) -> None: + size_mb = size // (1024 * 1024) + cmd = [VHD_UTIL, "create", OPT_LOG_ERR, "-n", path, "-s", str(size_mb)] + if static: + cmd.append("-r") + if msize: + cmd.append("-S") + cmd.append(str(msize)) + self._ioretry(cmd) + + @override + def snapshot( + self, + path: str, + parent: str, + parentRaw: bool, + msize: int = 0, + checkEmpty: Optional[bool] = True + ) -> None: + cmd = [VHD_UTIL, "snapshot", OPT_LOG_ERR, "-n", path, "-p", parent] + if parentRaw: + cmd.append("-m") + if msize: + cmd.append("-S") + cmd.append(str(msize)) + if not checkEmpty: + cmd.append("-e") + self._ioretry(cmd) + + @override + def check( + self, + path: str, + ignoreMissingFooter: Optional[bool] = False, + fast: Optional[bool] = False + ) -> CowUtil.CheckResult: + cmd = [VHD_UTIL, "check", OPT_LOG_ERR, "-n", path] + if ignoreMissingFooter: + cmd.append("-i") + if fast: + cmd.append("-B") + try: + self._ioretry(cmd) + return CowUtil.CheckResult.Success + except util.CommandException as e: + if e.code in (errno.ENOENT, errno.EROFS, errno.EMEDIUMTYPE): + return CowUtil.CheckResult.Unavailable + return CowUtil.CheckResult.Fail + + @override + def revert(self, path: str, jFile: str) -> None: + self._ioretry([VHD_UTIL, "revert", OPT_LOG_ERR, "-n", path, "-j", jFile]) + + @override + def repair(self, path: str) -> None: + """ + Repairs a VHD. + """ + self._ioretry([VHD_UTIL, "repair", "-n", path]) + + @override + def validateAndRoundImageSize(self, size: int) -> int: + """ + Take the supplied vhd size, in bytes, and check it is positive and less + that the maximum supported size, rounding up to the next block boundary. + """ + if size < 0 or size > MAX_VHD_SIZE: + raise xs_errors.XenError( + "VDISize", + opterr="VDI size must be between 1 MB and %d MB" % (MAX_VHD_SIZE // (1024 * 1024)) + ) + + if size < MIN_VHD_SIZE: + size = MIN_VHD_SIZE + + return util.roundup(VHD_BLOCK_SIZE, size) + + @override + def getKeyHash(self, path: str) -> Optional[str]: + """ + Extract the hash of the encryption key from the header of an encrypted VHD. + """ + ret = cast(str, self._ioretry([VHD_UTIL, "key", "-p", "-n", path])).strip() + if ret == "none": + return None + vals = ret.split() + if len(vals) != 2: + util.SMlog("***** malformed output from vhd-util for VHD {}: \"{}\"".format(path, ret)) + return None + [_nonce, key_hash] = vals + return key_hash + + @override + def setKey(self, path: str, key_hash: str) -> None: + """ + Set the encryption key for a VHD. + """ + self._ioretry([VHD_UTIL, "key", "-s", "-n", path, "-H", key_hash]) + + @staticmethod + def _convertAllocatedSizeToBytes(size: int): + # Assume we have standard 2MB allocation blocks + return size * 2 * 1024 * 1024 + + @staticmethod + def _parseVHDInfo(line: str, extractUuidFunction: Callable[[str], str]) -> Optional[CowImageInfo]: + vhdInfo = None + valueMap = line.split() + + try: + (key, val) = valueMap[0].split("=") + except: + return None + + if key != "vhd": + return None + + uuid = extractUuidFunction(val) + if not uuid: + util.SMlog("***** malformed output, no UUID: %s" % valueMap) + return None + vhdInfo = CowImageInfo(uuid) + vhdInfo.path = val + + for keyval in valueMap: + (key, val) = keyval.split("=") + if key == "scan-error": + vhdInfo.error = line + util.SMlog("***** VHD scan error: %s" % line) + break + elif key == "capacity": + vhdInfo.sizeVirt = int(val) + elif key == "size": + vhdInfo.sizePhys = int(val) + elif key == "hidden": + vhdInfo.hidden = bool(int(val)) + elif key == "parent" and val != "none": + vhdInfo.parentPath = val + vhdInfo.parentUuid = extractUuidFunction(val) + return vhdInfo diff --git a/tests/test_FileSR.py b/tests/test_FileSR.py index 8dfaca9b8..0818989f2 100644 --- a/tests/test_FileSR.py +++ b/tests/test_FileSR.py @@ -329,7 +329,7 @@ def test_create_vdi_vhd(self, mock_vhdutil): sr.path = "sr_path" vdi = FakeFileVDI(sr, vdi_uuid) vdi.vdi_type = VdiType.VHD - mock_vhdutil.validate_and_round_vhd_size.side_effect = vhdutil.validate_and_round_vhd_size + mock_vhdutil.validateAndRoundImageSize.side_effect = vhdutil.validateAndRoundImageSize # Act vdi.create(sr_uuid, vdi_uuid, 20 * 1024 * 1024) diff --git a/tests/test_LVHDSR.py b/tests/test_LVHDSR.py index a3ef06eeb..c797812f3 100644 --- a/tests/test_LVHDSR.py +++ b/tests/test_LVHDSR.py @@ -265,6 +265,9 @@ def setUp(self) -> None: vhdutil_patcher = mock.patch('LVHDSR.vhdutil', autospec=True) self.mock_vhdutil = vhdutil_patcher.start() self.mock_vhdutil.MAX_CHAIN_SIZE = vhdutil.MAX_CHAIN_SIZE + + # TODO: self._cowutil.getMaxChainLength() + lvutil_patcher = mock.patch('LVHDSR.lvutil', autospec=True) self.mock_lvutil = lvutil_patcher.start() vdi_util_patcher = mock.patch('VDI.util', autospec=True) diff --git a/tests/test_vhdutil.py b/tests/test_vhdutil.py index 6aa91e15b..9f04dee1a 100644 --- a/tests/test_vhdutil.py +++ b/tests/test_vhdutil.py @@ -18,27 +18,32 @@ class TestVhdUtil(unittest.TestCase): def test_validate_and_round_min_size(self): - size = vhdutil.validate_and_round_vhd_size(2 * 1024 * 1024) + size = vhdutil.validateAndRoundImageSize(2 * 1024 * 1024) self.assertTrue(size == 2 * 1024 * 1024) def test_validate_and_round_max_size(self): - size = vhdutil.validate_and_round_vhd_size(vhdutil.MAX_VHD_SIZE) + cowutil = None + size = vhdutil.validateAndRoundImageSize(cowutil.getMaxImageSize()) - self.assertTrue(size == vhdutil.MAX_VHD_SIZE) + cowutil = None + self.assertTrue(size == cowutil.getMaxImageSize()) def test_validate_and_round_odd_size_up_to_next_boundary(self): - size = vhdutil.validate_and_round_vhd_size(vhdutil.MAX_VHD_SIZE - 1) + cowutil = None + size = vhdutil.validateAndRoundImageSize(cowutil.getMaxImageSize() - 1) - self.assertTrue(size == vhdutil.MAX_VHD_SIZE) + cowutil = None + self.assertTrue(size == cowutil.getMaxImageSize()) def test_validate_and_round_negative(self): with self.assertRaises(xs_errors.SROSError): - vhdutil.validate_and_round_vhd_size(-1) + vhdutil.validateAndRoundImageSize(-1) def test_validate_and_round_too_large(self): with self.assertRaises(xs_errors.SROSError): - vhdutil.validate_and_round_vhd_size(vhdutil.MAX_VHD_SIZE + 1) + cowutil = None + vhdutil.validateAndRoundImageSize(cowutil.getMaxImageSize() + 1) @testlib.with_context def test_calc_overhead_empty_small(self, context):