From 2fe92e91625222afcc01765437ba47c5374b7a76 Mon Sep 17 00:00:00 2001 From: Radoslav Gerganov Date: Mon, 11 Jan 2016 16:35:12 +0200 Subject: [PATCH] VMware: Live migration of instances This patch implements live migration of instances across compute nodes. Each compute node must be managing a cluster in the same vCenter and ESX hosts must have vMotion enabled [1]. If the instance is located on a datastore shared between source and destination cluster, then only the host is changed. Otherwise, we select the most suitable datastore on the destination cluster and migrate the instance there. [1] https://kb.vmware.com/s/article/2054994 Co-Authored-By: gkotton@vmware.com blueprint vmware-live-migration Change-Id: I640013383e684497b2d99a9e1d6817d68c4d0a4b --- doc/source/admin/configuring-migrations.rst | 13 ++ doc/source/user/support-matrix.ini | 3 +- nova/objects/migrate_data.py | 10 ++ nova/tests/unit/objects/test_objects.py | 1 + .../unit/virt/vmwareapi/test_driver_api.py | 27 ++-- .../tests/unit/virt/vmwareapi/test_vm_util.py | 61 ++++++++ nova/tests/unit/virt/vmwareapi/test_vmops.py | 132 +++++++++++++++++- nova/virt/vmwareapi/driver.py | 66 +++++++++ nova/virt/vmwareapi/vm_util.py | 48 ++++++- nova/virt/vmwareapi/vmops.py | 98 +++++++++++++ ...mware-live-migration-c09cce337301cab0.yaml | 6 + 11 files changed, 444 insertions(+), 21 deletions(-) create mode 100644 releasenotes/notes/vmware-live-migration-c09cce337301cab0.yaml diff --git a/doc/source/admin/configuring-migrations.rst b/doc/source/admin/configuring-migrations.rst index 70d868ef76f6..dc6a9a9832e5 100644 --- a/doc/source/admin/configuring-migrations.rst +++ b/doc/source/admin/configuring-migrations.rst @@ -382,3 +382,16 @@ Block migration - Block migration works only with EXT local storage storage repositories, and the server must not have any volumes attached. + +VMware +~~~~~~ + +.. :ref:`_configuring-migrations-vmware` + +.. _configuring-migrations-vmware: + +vSphere configuration +--------------------- + +Enable vMotion on all ESX hosts which are managed by Nova by following the +instructions in `this `_ KB article. diff --git a/doc/source/user/support-matrix.ini b/doc/source/user/support-matrix.ini index 4f5dd31b053e..d11719e0a98c 100644 --- a/doc/source/user/support-matrix.ini +++ b/doc/source/user/support-matrix.ini @@ -468,8 +468,7 @@ driver.libvirt-kvm-s390x=complete driver.libvirt-qemu-x86=complete driver.libvirt-lxc=missing driver.libvirt-xen=complete -driver.vmware=missing -driver-notes.vmware=https://bugs.launchpad.net/nova/+bug/1192192 +driver.vmware=complete driver.hyperv=complete driver.ironic=missing driver.libvirt-vz-vm=complete diff --git a/nova/objects/migrate_data.py b/nova/objects/migrate_data.py index 25faae6031ac..1351bfbdf69e 100644 --- a/nova/objects/migrate_data.py +++ b/nova/objects/migrate_data.py @@ -501,3 +501,13 @@ class PowerVMLiveMigrateData(LiveMigrateData): for field in self.fields: if field in legacy: setattr(self, field, legacy[field]) + + +@obj_base.NovaObjectRegistry.register +class VMwareLiveMigrateData(LiveMigrateData): + VERSION = '1.0' + + fields = { + 'cluster_name': fields.StringField(nullable=False), + 'datastore_regex': fields.StringField(nullable=False), + } diff --git a/nova/tests/unit/objects/test_objects.py b/nova/tests/unit/objects/test_objects.py index 61f6feb3f3fc..2bdfdc5e3beb 100644 --- a/nova/tests/unit/objects/test_objects.py +++ b/nova/tests/unit/objects/test_objects.py @@ -1166,6 +1166,7 @@ object_data = { 'VirtCPUTopology': '1.0-fc694de72e20298f7c6bab1083fd4563', 'VirtualInterface': '1.3-efd3ca8ebcc5ce65fff5a25f31754c54', 'VirtualInterfaceList': '1.0-9750e2074437b3077e46359102779fc6', + 'VMwareLiveMigrateData': '1.0-a3cc858a2bf1d3806d6f57cfaa1fb98a', 'VolumeUsage': '1.0-6c8190c46ce1469bb3286a1f21c2e475', 'XenDeviceBus': '1.0-272a4f899b24e31e42b2b9a7ed7e9194', 'XenapiLiveMigrateData': '1.4-7dc9417e921b2953faa6751f18785f3f', diff --git a/nova/tests/unit/virt/vmwareapi/test_driver_api.py b/nova/tests/unit/virt/vmwareapi/test_driver_api.py index d3fd0b0838dc..94c94a7ff9a3 100644 --- a/nova/tests/unit/virt/vmwareapi/test_driver_api.py +++ b/nova/tests/unit/virt/vmwareapi/test_driver_api.py @@ -2349,23 +2349,20 @@ class VMwareAPIVMTestCase(test.NoDBTestCase, self.assertEqual(2, len(ds_util._DS_DC_MAPPING)) def test_pre_live_migration(self): - migrate_data = objects.migrate_data.LiveMigrateData() - self.assertRaises(NotImplementedError, - self.conn.pre_live_migration, self.context, - 'fake_instance', 'fake_block_device_info', - 'fake_network_info', 'fake_disk_info', migrate_data) - - def test_live_migration(self): - self.assertRaises(NotImplementedError, - self.conn.live_migration, self.context, - 'fake_instance', 'fake_dest', 'fake_post_method', - 'fake_recover_method') + migrate_data = objects.VMwareLiveMigrateData() + migrate_data.cluster_name = 'fake-cluster' + migrate_data.datastore_regex = 'datastore1' + ret = self.conn.pre_live_migration(self.context, 'fake-instance', + 'fake-block-dev-info', 'fake-net-info', + 'fake-disk-info', migrate_data) + self.assertIs(migrate_data, ret) def test_rollback_live_migration_at_destination(self): - self.assertRaises(NotImplementedError, - self.conn.rollback_live_migration_at_destination, - self.context, 'fake_instance', 'fake_network_info', - 'fake_block_device_info') + with mock.patch.object(self.conn, "destroy") as mock_destroy: + self.conn.rollback_live_migration_at_destination(self.context, + "instance", [], None) + mock_destroy.assert_called_once_with(self.context, + "instance", [], None) def test_post_live_migration(self): self.assertIsNone(self.conn.post_live_migration(self.context, diff --git a/nova/tests/unit/virt/vmwareapi/test_vm_util.py b/nova/tests/unit/virt/vmwareapi/test_vm_util.py index f2b2fff1406e..040d91c399a9 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vm_util.py +++ b/nova/tests/unit/virt/vmwareapi/test_vm_util.py @@ -181,6 +181,67 @@ class VMwareVMUtilTestCase(test.NoDBTestCase): self.assertEqual(expected, result) + def test_update_vif_spec_opaque_net(self): + fake_factory = fake.FakeFactory() + vif_info = {'network_name': 'br100', + 'mac_address': '00:00:00:ca:fe:01', + 'network_ref': {'type': 'OpaqueNetwork', + 'network-id': 'fake-network-id', + 'network-type': 'fake-net', + 'use-external-id': False}, + 'iface_id': 7, + 'vif_model': 'VirtualE1000'} + device = fake_factory.create('ns0:VirtualDevice') + actual = vm_util.update_vif_spec(fake_factory, vif_info, device) + spec = fake_factory.create('ns0:VirtualDeviceConfigSpec') + spec.device = fake_factory.create('ns0:VirtualDevice') + spec.device.backing = fake_factory.create( + 'ns0:VirtualEthernetCardOpaqueNetworkBackingInfo') + spec.device.backing.opaqueNetworkType = 'fake-net' + spec.device.backing.opaqueNetworkId = 'fake-network-id' + spec.operation = 'edit' + self.assertEqual(spec, actual) + + def test_update_vif_spec_dvpg(self): + fake_factory = fake.FakeFactory() + vif_info = {'network_name': 'br100', + 'mac_address': '00:00:00:ca:fe:01', + 'network_ref': {'type': 'DistributedVirtualPortgroup', + 'dvsw': 'fake-network-id', + 'dvpg': 'fake-group'}, + 'iface_id': 7, + 'vif_model': 'VirtualE1000'} + device = fake_factory.create('ns0:VirtualDevice') + actual = vm_util.update_vif_spec(fake_factory, vif_info, device) + spec = fake_factory.create('ns0:VirtualDeviceConfigSpec') + spec.device = fake_factory.create('ns0:VirtualDevice') + spec.device.backing = fake_factory.create( + 'ns0:VirtualEthernetCardDistributedVirtualPortBackingInfo') + spec.device.backing.port = fake_factory.create( + 'ns0:DistributedVirtualSwitchPortConnection') + spec.device.backing.port.portgroupKey = 'fake-group' + spec.device.backing.port.switchUuid = 'fake-network-id' + spec.operation = 'edit' + self.assertEqual(spec, actual) + + def test_update_vif_spec_network(self): + fake_factory = fake.FakeFactory() + vif_info = {'network_name': 'br100', + 'mac_address': '00:00:00:ca:fe:01', + 'network_ref': {'type': 'Network', + 'name': 'net1'}, + 'iface_id': 7, + 'vif_model': 'VirtualE1000'} + device = fake_factory.create('ns0:VirtualDevice') + actual = vm_util.update_vif_spec(fake_factory, vif_info, device) + spec = fake_factory.create('ns0:VirtualDeviceConfigSpec') + spec.device = fake_factory.create('ns0:VirtualDevice') + spec.device.backing = fake_factory.create( + 'ns0:VirtualEthernetCardNetworkBackingInfo') + spec.device.backing.deviceName = 'br100' + spec.operation = 'edit' + self.assertEqual(spec, actual) + def test_get_cdrom_attach_config_spec(self): fake_factory = fake.FakeFactory() datastore = fake.Datastore() diff --git a/nova/tests/unit/virt/vmwareapi/test_vmops.py b/nova/tests/unit/virt/vmwareapi/test_vmops.py index e015da0a8f28..5c45b9db77de 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vmops.py +++ b/nova/tests/unit/virt/vmwareapi/test_vmops.py @@ -82,6 +82,14 @@ class VMwareVMOpsTestCase(test.NoDBTestCase): vmFolder='fake_vm_folder') cluster = vmwareapi_fake.create_cluster('fake_cluster', fake_ds_ref) self._uuid = uuidsentinel.foo + fake_info_cache = { + 'created_at': None, + 'updated_at': None, + 'deleted_at': None, + 'deleted': False, + 'instance_uuid': self._uuid, + 'network_info': '[]', + } self._instance_values = { 'name': 'fake_name', 'display_name': 'fake_display_name', @@ -91,7 +99,8 @@ class VMwareVMOpsTestCase(test.NoDBTestCase): 'image_ref': self._image_id, 'root_gb': 10, 'node': '%s(%s)' % (cluster.mo_id, cluster.name), - 'expected_attrs': ['system_metadata'], + 'info_cache': fake_info_cache, + 'expected_attrs': ['system_metadata', 'info_cache'], } self._instance = fake_instance.fake_instance_obj( self._context, **self._instance_values) @@ -793,6 +802,127 @@ class VMwareVMOpsTestCase(test.NoDBTestCase): def test_finish_revert_migration_power_off(self): self._test_finish_revert_migration(power_on=False) + def _test_find_esx_host(self, cluster_hosts, ds_hosts): + def mock_call_method(module, method, *args, **kwargs): + if args[0] == 'fake_cluster': + ret = mock.MagicMock() + ret.ManagedObjectReference = cluster_hosts + return ret + elif args[0] == 'fake_ds': + ret = mock.MagicMock() + ret.DatastoreHostMount = ds_hosts + return ret + + with mock.patch.object(self._session, '_call_method', + mock_call_method): + return self._vmops._find_esx_host('fake_cluster', 'fake_ds') + + def test_find_esx_host(self): + ch1 = vmwareapi_fake.ManagedObjectReference(value='host-10') + ch2 = vmwareapi_fake.ManagedObjectReference(value='host-12') + ch3 = vmwareapi_fake.ManagedObjectReference(value='host-15') + dh1 = vmwareapi_fake.DatastoreHostMount('host-8') + dh2 = vmwareapi_fake.DatastoreHostMount('host-12') + dh3 = vmwareapi_fake.DatastoreHostMount('host-17') + ret = self._test_find_esx_host([ch1, ch2, ch3], [dh1, dh2, dh3]) + self.assertEqual('host-12', ret.value) + + def test_find_esx_host_none(self): + ch1 = vmwareapi_fake.ManagedObjectReference(value='host-10') + ch2 = vmwareapi_fake.ManagedObjectReference(value='host-12') + ch3 = vmwareapi_fake.ManagedObjectReference(value='host-15') + dh1 = vmwareapi_fake.DatastoreHostMount('host-8') + dh2 = vmwareapi_fake.DatastoreHostMount('host-13') + dh3 = vmwareapi_fake.DatastoreHostMount('host-17') + ret = self._test_find_esx_host([ch1, ch2, ch3], [dh1, dh2, dh3]) + self.assertIsNone(ret) + + @mock.patch.object(vm_util, 'get_vmdk_info') + @mock.patch.object(ds_obj, 'get_datastore_by_ref') + def test_find_datastore_for_migration(self, mock_get_ds, mock_get_vmdk): + def mock_call_method(module, method, *args, **kwargs): + ds1 = vmwareapi_fake.ManagedObjectReference(value='datastore-10') + ds2 = vmwareapi_fake.ManagedObjectReference(value='datastore-12') + ds3 = vmwareapi_fake.ManagedObjectReference(value='datastore-15') + ret = mock.MagicMock() + ret.ManagedObjectReference = [ds1, ds2, ds3] + return ret + ds_ref = vmwareapi_fake.ManagedObjectReference(value='datastore-12') + vmdk_dev = mock.MagicMock() + vmdk_dev.device.backing.datastore = ds_ref + mock_get_vmdk.return_value = vmdk_dev + ds = ds_obj.Datastore(ds_ref, 'datastore1') + mock_get_ds.return_value = ds + with mock.patch.object(self._session, '_call_method', + mock_call_method): + ret = self._vmops._find_datastore_for_migration(self._instance, + 'fake_vm', 'cluster_ref', + None) + self.assertIs(ds, ret) + mock_get_vmdk.assert_called_once_with(self._session, 'fake_vm', + uuid=self._instance.uuid) + mock_get_ds.assert_called_once_with(self._session, ds_ref) + + @mock.patch.object(vm_util, 'get_vmdk_info') + @mock.patch.object(ds_util, 'get_datastore') + def test_find_datastore_for_migration_other(self, mock_get_ds, + mock_get_vmdk): + def mock_call_method(module, method, *args, **kwargs): + ds1 = vmwareapi_fake.ManagedObjectReference(value='datastore-10') + ds2 = vmwareapi_fake.ManagedObjectReference(value='datastore-12') + ds3 = vmwareapi_fake.ManagedObjectReference(value='datastore-15') + ret = mock.MagicMock() + ret.ManagedObjectReference = [ds1, ds2, ds3] + return ret + ds_ref = vmwareapi_fake.ManagedObjectReference(value='datastore-18') + vmdk_dev = mock.MagicMock() + vmdk_dev.device.backing.datastore = ds_ref + mock_get_vmdk.return_value = vmdk_dev + ds = ds_obj.Datastore(ds_ref, 'datastore1') + mock_get_ds.return_value = ds + with mock.patch.object(self._session, '_call_method', + mock_call_method): + ret = self._vmops._find_datastore_for_migration(self._instance, + 'fake_vm', 'cluster_ref', + None) + self.assertIs(ds, ret) + mock_get_vmdk.assert_called_once_with(self._session, 'fake_vm', + uuid=self._instance.uuid) + mock_get_ds.assert_called_once_with(self._session, 'cluster_ref', + None) + + @mock.patch.object(vm_util, 'relocate_vm') + @mock.patch.object(vm_util, 'get_vm_ref', return_value='fake_vm') + @mock.patch.object(vm_util, 'get_cluster_ref_by_name', + return_value='fake_cluster') + @mock.patch.object(vm_util, 'get_res_pool_ref', return_value='fake_pool') + @mock.patch.object(vmops.VMwareVMOps, '_find_datastore_for_migration') + @mock.patch.object(vmops.VMwareVMOps, '_find_esx_host', + return_value='fake_host') + def test_live_migration(self, mock_find_host, mock_find_datastore, + mock_get_respool, mock_get_cluster, mock_get_vm, + mock_relocate): + post_method = mock.MagicMock() + migrate_data = objects.VMwareLiveMigrateData() + migrate_data.cluster_name = 'fake-cluster' + migrate_data.datastore_regex = 'ds1|ds2' + mock_find_datastore.return_value = ds_obj.Datastore('ds_ref', 'ds') + with mock.patch.object(self._session, '_call_method', + return_value='hardware-devices'): + self._vmops.live_migration( + self._context, self._instance, 'fake-host', + post_method, None, False, migrate_data) + mock_get_vm.assert_called_once_with(self._session, self._instance) + mock_get_cluster.assert_called_once_with(self._session, 'fake-cluster') + mock_find_datastore.assert_called_once_with(self._instance, 'fake_vm', + 'fake_cluster', mock.ANY) + mock_find_host.assert_called_once_with('fake_cluster', 'ds_ref') + mock_relocate.assert_called_once_with(self._session, 'fake_vm', + 'fake_pool', 'ds_ref', 'fake_host', + devices=[]) + post_method.assert_called_once_with(self._context, self._instance, + 'fake-host', False, migrate_data) + @mock.patch.object(vmops.VMwareVMOps, '_get_instance_metadata') @mock.patch.object(vmops.VMwareVMOps, '_get_extra_specs') @mock.patch.object(vm_util, 'reconfigure_vm') diff --git a/nova/virt/vmwareapi/driver.py b/nova/virt/vmwareapi/driver.py index 3807d2618f2a..dba10d2e2097 100644 --- a/nova/virt/vmwareapi/driver.py +++ b/nova/virt/vmwareapi/driver.py @@ -38,6 +38,7 @@ from nova.compute import utils as compute_utils import nova.conf from nova import exception from nova.i18n import _ +from nova import objects import nova.privsep.path from nova import rc_fields as fields from nova.virt import driver @@ -269,6 +270,71 @@ class VMwareVCDriver(driver.ComputeDriver): network_info, image_meta, resize_instance, block_device_info, power_on) + def ensure_filtering_rules_for_instance(self, instance, network_info): + pass + + def pre_live_migration(self, context, instance, block_device_info, + network_info, disk_info, migrate_data): + return migrate_data + + def post_live_migration_at_source(self, context, instance, network_info): + pass + + def post_live_migration_at_destination(self, context, instance, + network_info, + block_migration=False, + block_device_info=None): + pass + + def cleanup_live_migration_destination_check(self, context, + dest_check_data): + pass + + def live_migration(self, context, instance, dest, + post_method, recover_method, block_migration=False, + migrate_data=None): + """Live migration of an instance to another host.""" + self._vmops.live_migration(context, instance, dest, post_method, + recover_method, block_migration, + migrate_data) + + def check_can_live_migrate_source(self, context, instance, + dest_check_data, block_device_info=None): + cluster_name = dest_check_data.cluster_name + cluster_ref = vm_util.get_cluster_ref_by_name(self._session, + cluster_name) + if cluster_ref is None: + msg = (_("Cannot find destination cluster %s for live migration") % + cluster_name) + raise exception.MigrationPreCheckError(reason=msg) + res_pool_ref = vm_util.get_res_pool_ref(self._session, cluster_ref) + if res_pool_ref is None: + msg = _("Cannot find destination resource pool for live migration") + raise exception.MigrationPreCheckError(reason=msg) + return dest_check_data + + def check_can_live_migrate_destination(self, context, instance, + src_compute_info, dst_compute_info, + block_migration=False, + disk_over_commit=False): + # the information that we need for the destination compute node + # is the name of its cluster and datastore regex + data = objects.VMwareLiveMigrateData() + data.cluster_name = CONF.vmware.cluster_name + data.datastore_regex = CONF.vmware.datastore_regex + return data + + def unfilter_instance(self, instance, network_info): + pass + + def rollback_live_migration_at_destination(self, context, instance, + network_info, + block_device_info, + destroy_disks=True, + migrate_data=None): + """Clean up destination node after a failed live migration.""" + self.destroy(context, instance, network_info, block_device_info) + def get_instance_disk_info(self, instance, block_device_info=None): pass diff --git a/nova/virt/vmwareapi/vm_util.py b/nova/virt/vmwareapi/vm_util.py index c4cd6830d5b0..f0bf503b9c38 100644 --- a/nova/virt/vmwareapi/vm_util.py +++ b/nova/virt/vmwareapi/vm_util.py @@ -519,6 +519,44 @@ def get_network_detach_config_spec(client_factory, device, port_index): return config_spec +def update_vif_spec(client_factory, vif_info, device): + """Updates the backing for the VIF spec.""" + network_spec = client_factory.create('ns0:VirtualDeviceConfigSpec') + network_spec.operation = 'edit' + network_ref = vif_info['network_ref'] + network_name = vif_info['network_name'] + if network_ref and network_ref['type'] == 'OpaqueNetwork': + backing = client_factory.create( + 'ns0:VirtualEthernetCardOpaqueNetworkBackingInfo') + backing.opaqueNetworkId = network_ref['network-id'] + backing.opaqueNetworkType = network_ref['network-type'] + # Configure externalId + if network_ref['use-external-id']: + if hasattr(device, 'externalId'): + device.externalId = vif_info['iface_id'] + else: + dp = client_factory.create('ns0:DynamicProperty') + dp.name = "__externalId__" + dp.val = vif_info['iface_id'] + device.dynamicProperty = [dp] + elif (network_ref and + network_ref['type'] == "DistributedVirtualPortgroup"): + backing = client_factory.create( + 'ns0:VirtualEthernetCardDistributedVirtualPortBackingInfo') + portgroup = client_factory.create( + 'ns0:DistributedVirtualSwitchPortConnection') + portgroup.switchUuid = network_ref['dvsw'] + portgroup.portgroupKey = network_ref['dvpg'] + backing.port = portgroup + else: + backing = client_factory.create( + 'ns0:VirtualEthernetCardNetworkBackingInfo') + backing.deviceName = network_name + device.backing = backing + network_spec.device = device + return network_spec + + def get_storage_profile_spec(session, storage_policy): """Gets the vm profile spec configured for storage policy.""" profile_id = pbm.get_profile_id_by_name(session, storage_policy) @@ -938,20 +976,24 @@ def clone_vm_spec(client_factory, location, def relocate_vm_spec(client_factory, res_pool=None, datastore=None, host=None, - disk_move_type="moveAllDiskBackingsAndAllowSharing"): + disk_move_type="moveAllDiskBackingsAndAllowSharing", + devices=None): rel_spec = client_factory.create('ns0:VirtualMachineRelocateSpec') rel_spec.datastore = datastore rel_spec.host = host rel_spec.pool = res_pool rel_spec.diskMoveType = disk_move_type + if devices is not None: + rel_spec.deviceChange = devices return rel_spec def relocate_vm(session, vm_ref, res_pool=None, datastore=None, host=None, - disk_move_type="moveAllDiskBackingsAndAllowSharing"): + disk_move_type="moveAllDiskBackingsAndAllowSharing", + devices=None): client_factory = session.vim.client.factory rel_spec = relocate_vm_spec(client_factory, res_pool, datastore, host, - disk_move_type) + disk_move_type, devices) relocate_task = session._call_method(session.vim, "RelocateVM_Task", vm_ref, spec=rel_spec) session._wait_for_task(relocate_task) diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 91f8f79298e8..7f69dd0e8bc6 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -21,6 +21,7 @@ Class for VM tasks like spawn, snapshot, suspend, resume etc. import collections import os +import re import time import decorator @@ -1552,6 +1553,103 @@ class VMwareVMOps(object): step=6, total_steps=RESIZE_TOTAL_STEPS) + def _find_esx_host(self, cluster_ref, ds_ref): + """Find ESX host in the specified cluster which is also connected to + the specified datastore. + """ + cluster_hosts = self._session._call_method(vutil, + 'get_object_property', + cluster_ref, 'host') + ds_hosts = self._session._call_method(vutil, 'get_object_property', + ds_ref, 'host') + for ds_host in ds_hosts.DatastoreHostMount: + for cluster_host in cluster_hosts.ManagedObjectReference: + if ds_host.key.value == cluster_host.value: + return cluster_host + + def _find_datastore_for_migration(self, instance, vm_ref, cluster_ref, + datastore_regex): + """Find datastore in the specified cluster where the instance will be + migrated to. Return the current datastore if it is already connected to + the specified cluster. + """ + vmdk = vm_util.get_vmdk_info(self._session, vm_ref, uuid=instance.uuid) + ds_ref = vmdk.device.backing.datastore + cluster_datastores = self._session._call_method(vutil, + 'get_object_property', + cluster_ref, + 'datastore') + if not cluster_datastores: + LOG.warning('No datastores found in the destination cluster') + return None + # check if the current datastore is connected to the destination + # cluster + for datastore in cluster_datastores.ManagedObjectReference: + if datastore.value == ds_ref.value: + ds = ds_obj.get_datastore_by_ref(self._session, ds_ref) + if (datastore_regex is None or + datastore_regex.match(ds.name)): + LOG.debug('Datastore "%s" is connected to the ' + 'destination cluster', ds.name) + return ds + # find the most suitable datastore on the destination cluster + return ds_util.get_datastore(self._session, cluster_ref, + datastore_regex) + + def live_migration(self, context, instance, dest, + post_method, recover_method, block_migration, + migrate_data): + LOG.debug("Live migration data %s", migrate_data, instance=instance) + vm_ref = vm_util.get_vm_ref(self._session, instance) + cluster_name = migrate_data.cluster_name + cluster_ref = vm_util.get_cluster_ref_by_name(self._session, + cluster_name) + datastore_regex = re.compile(migrate_data.datastore_regex) + res_pool_ref = vm_util.get_res_pool_ref(self._session, cluster_ref) + # find a datastore where the instance will be migrated to + ds = self._find_datastore_for_migration(instance, vm_ref, cluster_ref, + datastore_regex) + if ds is None: + LOG.error("Cannot find datastore", instance=instance) + raise exception.HostNotFound(host=dest) + LOG.debug("Migrating instance to datastore %s", ds.name, + instance=instance) + # find ESX host in the destination cluster which is connected to the + # target datastore + esx_host = self._find_esx_host(cluster_ref, ds.ref) + if esx_host is None: + LOG.error("Cannot find ESX host for live migration, cluster: %s, " + "datastore: %s", migrate_data.cluster_name, ds.name, + instance=instance) + raise exception.HostNotFound(host=dest) + # Update networking backings + network_info = instance.get_network_info() + client_factory = self._session.vim.client.factory + devices = [] + hardware_devices = self._session._call_method( + vutil, "get_object_property", vm_ref, "config.hardware.device") + vif_model = instance.image_meta.properties.get('hw_vif_model', + constants.DEFAULT_VIF_MODEL) + for vif in network_info: + vif_info = vmwarevif.get_vif_dict( + self._session, cluster_ref, vif_model, utils.is_neutron(), vif) + device = vmwarevif.get_network_device(hardware_devices, + vif['address']) + devices.append(vm_util.update_vif_spec(client_factory, vif_info, + device)) + + LOG.debug("Migrating instance to cluster '%s', datastore '%s' and " + "ESX host '%s'", cluster_name, ds.name, esx_host, + instance=instance) + try: + vm_util.relocate_vm(self._session, vm_ref, res_pool_ref, + ds.ref, esx_host, devices=devices) + LOG.info("Migrated instance to host %s", dest, instance=instance) + except Exception: + with excutils.save_and_reraise_exception(): + recover_method(context, instance, dest, migrate_data) + post_method(context, instance, dest, block_migration, migrate_data) + def poll_rebooting_instances(self, timeout, instances): """Poll for rebooting instances.""" ctxt = nova_context.get_admin_context() diff --git a/releasenotes/notes/vmware-live-migration-c09cce337301cab0.yaml b/releasenotes/notes/vmware-live-migration-c09cce337301cab0.yaml new file mode 100644 index 000000000000..f2453b90bc94 --- /dev/null +++ b/releasenotes/notes/vmware-live-migration-c09cce337301cab0.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + The VMware compute driver now supports live migration. Each compute node + must be managing a cluster in the same vCenter and ESX hosts must have + vMotion enabled.