Merge pull request #193 from xcp-ng/migrate-without-xo

Do not use XO for migration tests

jobs.py

@@ -43,7 +43,7 @@
             "--vm[]": "multi/all",
             "--sr-disk": "auto",
         },
-        "paths": ["tests/misc"],
+        "paths": ["tests/misc", "tests/migration"],
         "markers": "multi_vms and not flaky and not reboot",
     },
     "packages": {

lib/host.py

@@ -74,6 +74,11 @@ def xe(self, action, args={}, check=True, simple_output=True, minimal=False, for
         def stringify(key, value):
             if isinstance(value, bool):
                 return "{}={}".format(key, to_xapi_bool(value))
+            if isinstance(value, dict):
+                ret = ""
+                for key2, value2 in value.items():
+                    ret += f"{key}:{key2}={value2} "
+                return ret.rstrip()
             return "{}={}".format(key, shlex.quote(value))
 
         command = ['xe', action] + maybe_param_minimal + maybe_param_force + \

lib/vm.py

@@ -10,7 +10,6 @@
 from lib.common import PackageManagerEnum, parse_xe_dict, safe_split, wait_for, wait_for_not
 from lib.snapshot import Snapshot
 from lib.vif import VIF
-from lib.xo import xo_object_exists, xo_cli
 
 class VM(BaseVM):
     def __init__(self, uuid, host):
@@ -178,31 +177,52 @@ def exists(self):
     def exists_on_previous_pool(self):
         return self.previous_host.pool_has_vm(self.uuid)
 
-    def migrate(self, target_host, sr=None):
-        # workaround XO bug where sometimes it loses connection without knowing it
-        self.host.pool.master.xo_server_reconnect()
-        if target_host.pool != self.host.pool:
-            target_host.pool.master.xo_server_reconnect()
-
-        # Sometimes we migrate VMs right after creating them
-        # In that case we need to ensure that XO knows about the new VM
-        # Else we risk getting a "no such VM" error
-        # Thus, let's first wait for XO to know about the VM
-        wait_for(lambda: xo_object_exists(self.uuid), "Wait for XO to know about VM %s" % self.uuid)
-
+    def migrate(self, target_host, sr=None, network=None):
         msg = "Migrate VM to host %s" % target_host
         params = {
-            'vm': self.uuid,
-            'targetHost': target_host.uuid
+            'uuid': self.uuid,
+            'host-uuid': target_host.uuid,
+            'live': self.is_running()
         }
+        cross_pool = self.host.pool.uuid != target_host.pool.uuid
         if sr is not None:
-            msg += " (SR: %s)" % sr.uuid
-            mapping = {}
-            for vdi_uuid in self.vdi_uuids():
-                mapping[vdi_uuid] = sr.uuid
-            params['mapVdisSrs'] = 'json:' + json.dumps(mapping)
+            if self.get_sr().uuid == sr.uuid:
+                # Same SR, no need to migrate storage
+                sr = None
+            else:
+                msg += " (SR: %s)" % sr.uuid
+        if network is not None:
+            msg += " (Network: %s)" % network
         logging.info(msg)
-        xo_cli('vm.migrate', params)
+
+        storage_motion = cross_pool or sr is not None or network is not None
+        if storage_motion:
+            remote_master = target_host.pool.master
+            params['remote-master'] = remote_master.hostname_or_ip
+            params['remote-username'] = remote_master.user
+            params['remote-password'] = remote_master.password
+
+            if sr is not None:
+                sr_uuid = sr.uuid
+            else:
+                sr_uuid = target_host.xe('pool-param-get', {'uuid': target_host.pool.uuid, 'param-name': 'default-SR'})
+            vdi_map = {}
+            for vdi_uuid in self.vdi_uuids():
+                vdi_map[vdi_uuid] = sr_uuid
+            params['vdi'] = vdi_map
+
+            if cross_pool:
+                # VIF mapping is only required for cross pool migration
+                if network is None:
+                    network = remote_master.management_network()
+
+                vif_map = {}
+                for vif in self.vifs():
+                    vif_map[vif.uuid] = network
+                params['vif'] = vif_map
+
+        self.host.xe('vm-migrate', params)
+
         self.previous_host = self.host
         self.host = target_host
 

tests/guest-tools/unix/test_guest_tools_unix.py

@@ -116,7 +116,6 @@ def test_clean_shutdown(self, running_vm):
         vm.start()
         vm.wait_for_vm_running_and_ssh_up()
 
-    @pytest.mark.usefixtures('hosts_with_xo')
     def test_storage_migration(self, running_vm, host, hostA2, local_sr_on_hostA2, state):
         vm = running_vm
         # migrate to default SR on hostA2

tests/misc/test_cross_pool_migration.py → tests/migration/test_cross_pool_migration.py

@@ -5,7 +5,6 @@
 
 @pytest.mark.multi_vms # run on a variety of VMs
 @pytest.mark.big_vm # and also on a really big VM ideally
-@pytest.mark.usefixtures('hosts_with_xo')
 def test_cross_pool_migration(hostB1, imported_vm):
     vm = imported_vm.clone()
     try:

tests/misc/test_basic_without_ssh.py

@@ -75,7 +75,7 @@ def test_checkpoint(self, imported_vm):
     # Live migration tests
     # We want to test storage migration (memory+disks) and live migration without storage migration (memory only).
     # The order will depend on the initial location of the VM: a local SR or a shared SR.
-    @pytest.mark.usefixtures("hostA2", 'hosts_with_xo')
+    @pytest.mark.usefixtures("hostA2")
     def test_live_migrate(self, imported_vm, existing_shared_sr):
         def live_migrate(vm, dest_host, dest_sr, check_vdis=False):
             vm.migrate(dest_host, dest_sr)

tests/misc/test_xo_connection.py

@@ -0,0 +1,12 @@
+import logging
+
+# This test is meant to test an XO connection.
+#
+# Requirements:
+# - an XCP-ng host >= 8.2 with latest updates.
+# - An XOA registered through `xo-cli`
+
+def test_xo_connection(hosts_with_xo):
+    for h in hosts_with_xo:
+        logging.debug(f"Testing connection for host: {h.hostname_or_ip}")
+        assert h.xo_get_server_id() is not None

tests/storage/cephfs/test_cephfs_sr_crosspool_migration.py

@@ -12,7 +12,7 @@
 
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
-@pytest.mark.usefixtures("hostB1", "hosts_with_xo", "local_sr_on_hostB1")
+@pytest.mark.usefixtures("hostB1", "local_sr_on_hostB1")
 class Test:
     def test_cold_crosspool_migration(self, host, hostB1, vm_on_cephfs_sr, cephfs_sr, local_sr_on_hostB1):
         cold_migration_then_come_back(vm_on_cephfs_sr, host, cephfs_sr, hostB1, local_sr_on_hostB1)

tests/storage/cephfs/test_cephfs_sr_intrapool_migration.py

@@ -12,7 +12,7 @@
 
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
-@pytest.mark.usefixtures("hostA2", "hosts_with_xo", "local_sr_on_hostA2")
+@pytest.mark.usefixtures("hostA2", "local_sr_on_hostA2")
 class Test:
     def test_live_intrapool_shared_migration(self, host, hostA2, vm_on_cephfs_sr, cephfs_sr):
         live_storage_migration_then_come_back(vm_on_cephfs_sr, host, cephfs_sr, hostA2, cephfs_sr)

tests/storage/ext/test_ext_sr_crosspool_migration.py

@@ -10,7 +10,7 @@
 
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
-@pytest.mark.usefixtures("hostB1", "hosts_with_xo", "local_sr_on_hostB1")
+@pytest.mark.usefixtures("hostB1", "local_sr_on_hostB1")
 class Test:
     def test_cold_crosspool_migration(self, host, hostB1, vm_on_ext_sr, ext_sr, local_sr_on_hostB1):
         cold_migration_then_come_back(vm_on_ext_sr, host, ext_sr, hostB1, local_sr_on_hostB1)

tests/storage/ext/test_ext_sr_intrapool_migration.py

@@ -10,7 +10,7 @@
 
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
-@pytest.mark.usefixtures("hostA2", "hosts_with_xo", "local_sr_on_hostA2")
+@pytest.mark.usefixtures("hostA2", "local_sr_on_hostA2")
 class Test:
     def test_cold_intrapool_migration(self, host, hostA2, vm_on_ext_sr, ext_sr, local_sr_on_hostA2):
         cold_migration_then_come_back(vm_on_ext_sr, host, ext_sr, hostA2, local_sr_on_hostA2)

tests/storage/glusterfs/test_glusterfs_sr_crosspool_migration.py

@@ -12,7 +12,7 @@
 
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
-@pytest.mark.usefixtures("hostB1", "hosts_with_xo", "local_sr_on_hostB1", "sr_disk_for_all_hosts")
+@pytest.mark.usefixtures("hostB1", "local_sr_on_hostB1", "sr_disk_for_all_hosts")
 class Test:
     def test_cold_crosspool_migration(self, host, hostB1, vm_on_glusterfs_sr, glusterfs_sr, local_sr_on_hostB1):
         cold_migration_then_come_back(vm_on_glusterfs_sr, host, glusterfs_sr, hostB1, local_sr_on_hostB1)

tests/storage/glusterfs/test_glusterfs_sr_intrapool_migration.py

@@ -12,7 +12,7 @@
 
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
-@pytest.mark.usefixtures("hostA2", "hosts_with_xo", "local_sr_on_hostA2", "sr_disk_for_all_hosts")
+@pytest.mark.usefixtures("hostA2", "local_sr_on_hostA2", "sr_disk_for_all_hosts")
 class Test:
     def test_live_intrapool_shared_migration(self, host, hostA2, vm_on_glusterfs_sr, glusterfs_sr):
         live_storage_migration_then_come_back(vm_on_glusterfs_sr, host, glusterfs_sr, hostA2, glusterfs_sr)

tests/storage/linstor/test_linstor_sr_crosspool_migration.py

@@ -12,7 +12,7 @@
 
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally on a big VM to test it scales
-@pytest.mark.usefixtures("hostB1", "hosts_with_xo", "local_sr_on_hostB1")
+@pytest.mark.usefixtures("hostB1", "local_sr_on_hostB1")
 class Test:
     def test_cold_crosspool_migration(self, host, hostB1, vm_on_linstor_sr, linstor_sr, local_sr_on_hostB1):
         cold_migration_then_come_back(vm_on_linstor_sr, host, linstor_sr, hostB1, local_sr_on_hostB1)

tests/storage/linstor/test_linstor_sr_intrapool_migration.py

@@ -12,7 +12,7 @@
 
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
-@pytest.mark.usefixtures("hostA2", "hosts_with_xo", "local_sr_on_hostA2")
+@pytest.mark.usefixtures("hostA2", "local_sr_on_hostA2")
 class Test:
     def test_live_intrapool_shared_migration(self, host, hostA2, vm_on_linstor_sr, linstor_sr):
         live_storage_migration_then_come_back(vm_on_linstor_sr, host, linstor_sr, hostA2, linstor_sr)

tests/storage/lvm/test_lvm_sr_crosspool_migration.py

@@ -10,7 +10,7 @@
 
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
-@pytest.mark.usefixtures("hostB1", "hosts_with_xo", "local_sr_on_hostB1")
+@pytest.mark.usefixtures("hostB1", "local_sr_on_hostB1")
 class Test:
     def test_cold_crosspool_migration(self, host, hostB1, vm_on_lvm_sr, lvm_sr, local_sr_on_hostB1):
         cold_migration_then_come_back(vm_on_lvm_sr, host, lvm_sr, hostB1, local_sr_on_hostB1)

tests/storage/lvm/test_lvm_sr_intrapool_migration.py

@@ -10,7 +10,7 @@
 
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
-@pytest.mark.usefixtures("hostA2", "hosts_with_xo", "local_sr_on_hostA2")
+@pytest.mark.usefixtures("hostA2", "local_sr_on_hostA2")
 class Test:
     def test_cold_intrapool_migration(self, host, hostA2, vm_on_lvm_sr, lvm_sr, local_sr_on_hostA2):
         cold_migration_then_come_back(vm_on_lvm_sr, host, lvm_sr, hostA2, local_sr_on_hostA2)

tests/storage/lvmoiscsi/test_lvmoiscsi_sr_crosspool_migration.py

@@ -10,7 +10,7 @@
 
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
-@pytest.mark.usefixtures("hostB1", "hosts_with_xo", "local_sr_on_hostB1")
+@pytest.mark.usefixtures("hostB1", "local_sr_on_hostB1")
 class Test:
     def test_cold_crosspool_migration(self, host, hostB1, vm_on_lvmoiscsi_sr, lvmoiscsi_sr, local_sr_on_hostB1):
         cold_migration_then_come_back(vm_on_lvmoiscsi_sr, host, lvmoiscsi_sr, hostB1, local_sr_on_hostB1)

tests/storage/lvmoiscsi/test_lvmoiscsi_sr_intrapool_migration.py

@@ -10,7 +10,7 @@
 
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
-@pytest.mark.usefixtures("hostA2", "hosts_with_xo", "local_sr_on_hostA2")
+@pytest.mark.usefixtures("hostA2", "local_sr_on_hostA2")
 class Test:
     def test_live_intrapool_shared_migration(self, host, hostA2, vm_on_lvmoiscsi_sr, lvmoiscsi_sr):
         live_storage_migration_then_come_back(vm_on_lvmoiscsi_sr, host, lvmoiscsi_sr, hostA2, lvmoiscsi_sr)

tests/storage/moosefs/test_moosefs_sr_crosspool_migration.py

@@ -12,7 +12,7 @@
 
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally on a big VM to test it scales
-@pytest.mark.usefixtures("hostB1", "hosts_with_xo", "local_sr_on_hostB1")
+@pytest.mark.usefixtures("hostB1", "local_sr_on_hostB1")
 class Test:
     def test_cold_crosspool_migration(self, host, hostB1, vm_on_moosefs_sr, moosefs_sr, local_sr_on_hostB1):
         cold_migration_then_come_back(vm_on_moosefs_sr, host, moosefs_sr, hostB1, local_sr_on_hostB1)

tests/storage/moosefs/test_moosefs_sr_intrapool_migration.py

@@ -12,7 +12,7 @@
 
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally on a big VM to test it scales
-@pytest.mark.usefixtures("hostA2", "hosts_with_xo", "local_sr_on_hostA2")
+@pytest.mark.usefixtures("hostA2", "local_sr_on_hostA2")
 class Test:
     def test_live_intrapool_shared_migration(self, host, hostA2, vm_on_moosefs_sr, moosefs_sr):
         live_storage_migration_then_come_back(vm_on_moosefs_sr, host, moosefs_sr, hostA2, moosefs_sr)

tests/storage/nfs/test_nfs_sr_crosspool_migration.py

@@ -10,7 +10,7 @@
 
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
-@pytest.mark.usefixtures("hostB1", "hosts_with_xo", "local_sr_on_hostB1")
+@pytest.mark.usefixtures("hostB1", "local_sr_on_hostB1")
 class Test:
     def test_cold_crosspool_migration(self, host, hostB1, vm_on_nfs_sr, nfs_sr, local_sr_on_hostB1):
         cold_migration_then_come_back(vm_on_nfs_sr, host, nfs_sr, hostB1, local_sr_on_hostB1)

tests/storage/nfs/test_nfs_sr_intrapool_migration.py

@@ -10,7 +10,7 @@
 
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
-@pytest.mark.usefixtures("hostA2", "hosts_with_xo", "local_sr_on_hostA2")
+@pytest.mark.usefixtures("hostA2", "local_sr_on_hostA2")
 class Test:
     def test_live_intrapool_shared_migration(self, host, hostA2, vm_on_nfs_sr, nfs_sr):
         live_storage_migration_then_come_back(vm_on_nfs_sr, host, nfs_sr, hostA2, nfs_sr)

tests/storage/xfs/test_xfs_sr_crosspool_migration.py

@@ -12,7 +12,7 @@
 
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally on a big VM to test it scales
-@pytest.mark.usefixtures("hostB1", "hosts_with_xo", "local_sr_on_hostB1")
+@pytest.mark.usefixtures("hostB1", "local_sr_on_hostB1")
 class Test:
     def test_cold_crosspool_migration(self, host, hostB1, vm_on_xfs_sr, xfs_sr, local_sr_on_hostB1):
         cold_migration_then_come_back(vm_on_xfs_sr, host, xfs_sr, hostB1, local_sr_on_hostB1)

tests/storage/xfs/test_xfs_sr_intrapool_migration.py

@@ -12,7 +12,7 @@
 
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally on a big VM to test it scales
-@pytest.mark.usefixtures("hostA2", "hosts_with_xo", "local_sr_on_hostA2")
+@pytest.mark.usefixtures("hostA2", "local_sr_on_hostA2")
 class Test:
     def test_cold_intrapool_migration(self, host, hostA2, vm_on_xfs_sr, xfs_sr, local_sr_on_hostA2):
         cold_migration_then_come_back(vm_on_xfs_sr, host, xfs_sr, hostA2, local_sr_on_hostA2)

tests/storage/zfs/test_zfs_sr_crosspool_migration.py

@@ -12,7 +12,7 @@
 
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
-@pytest.mark.usefixtures("hostB1", "hosts_with_xo", "local_sr_on_hostB1")
+@pytest.mark.usefixtures("hostB1", "local_sr_on_hostB1")
 class Test:
     def test_cold_crosspool_migration(self, host, hostB1, vm_on_zfs_sr, zfs_sr, local_sr_on_hostB1):
         cold_migration_then_come_back(vm_on_zfs_sr, host, zfs_sr, hostB1, local_sr_on_hostB1)

tests/storage/zfs/test_zfs_sr_intrapool_migration.py

@@ -12,7 +12,7 @@
 
 @pytest.mark.small_vm # run with a small VM to test the features
 @pytest.mark.big_vm # and ideally with a big VM to test it scales
-@pytest.mark.usefixtures("hostA2", "hosts_with_xo", "local_sr_on_hostA2")
+@pytest.mark.usefixtures("hostA2", "local_sr_on_hostA2")
 class Test:
     def test_cold_intrapool_migration(self, host, hostA2, vm_on_zfs_sr, zfs_sr, local_sr_on_hostA2):
         cold_migration_then_come_back(vm_on_zfs_sr, host, zfs_sr, hostA2, local_sr_on_hostA2)