From ef655f802e5bebf8987a551e0d11766b8544ef23 Mon Sep 17 00:00:00 2001 From: Radoslav Gerganov Date: Mon, 23 Feb 2015 16:40:37 +0200 Subject: [PATCH] VMware: Do not untar OVA on the file system The OVA image is a tarball which consists of OVF descriptor and VMDKs. Currently the OVA is unpacked on the file system and then uploaded on the datastore. This is inefficient and may cause n-cpu to run out of space if large images are booted. This patch fixes this by processing the OVA image as a stream -- we subsequently read chunks of data from Glance and upload them to vCenter. Change-Id: I22f5897da0870a1d69fa153a0868000369159b37 --- nova/tests/unit/virt/vmwareapi/test_images.py | 13 ++-- nova/virt/vmwareapi/images.py | 78 +++++++++---------- 2 files changed, 41 insertions(+), 50 deletions(-) diff --git a/nova/tests/unit/virt/vmwareapi/test_images.py b/nova/tests/unit/virt/vmwareapi/test_images.py index cfd7d79eeec0..72a870af64e7 100644 --- a/nova/tests/unit/virt/vmwareapi/test_images.py +++ b/nova/tests/unit/virt/vmwareapi/test_images.py @@ -106,9 +106,8 @@ class VMwareImagesTestCase(test.NoDBTestCase): @mock.patch('oslo_vmware.rw_handles.ImageReadHandle') @mock.patch('oslo_vmware.rw_handles.VmdkWriteHandle') @mock.patch.object(tarfile, 'open') - @mock.patch.object(os, 'unlink') - def test_fetch_image_ova(self, mock_unlink, mock_tar_open, - mock_write_class, mock_read_class): + def test_fetch_image_ova(self, mock_tar_open, mock_write_class, + mock_read_class): session = mock.MagicMock() ovf_descriptor = None ovf_path = os.path.join(os.path.dirname(__file__), 'ovf.xml') @@ -150,11 +149,11 @@ class VMwareImagesTestCase(test.NoDBTestCase): mock_vmdk.name = "Damn_Small_Linux-disk1.vmdk" def fake_extract(name): - if name == mock_ovf.name: + if name == mock_ovf: m = mock.MagicMock() m.read.return_value = ovf_descriptor return m - elif name == mock_vmdk.name: + elif name == mock_vmdk: return mock_read_handle mock_tar = mock.MagicMock() @@ -167,14 +166,14 @@ class VMwareImagesTestCase(test.NoDBTestCase): context, instance, session, 'fake-vm', 'fake-datastore', vm_folder_ref, res_pool_ref) + mock_tar_open.assert_called_once_with(mode='r|', + fileobj=mock_read_handle) mock_start_transfer.assert_called_once_with(context, mock_read_handle, 512, write_file_handle=mock_write_handle) mock_call_method.assert_called_once_with( session.vim, "UnregisterVM", mock.sentinel.vm_ref) - mock_unlink.assert_called_once_with(mock.ANY) - @mock.patch('oslo_vmware.rw_handles.ImageReadHandle') @mock.patch('oslo_vmware.rw_handles.VmdkWriteHandle') def test_fetch_image_stream_optimized(self, diff --git a/nova/virt/vmwareapi/images.py b/nova/virt/vmwareapi/images.py index e5606cb5b857..524b1787dcfe 100644 --- a/nova/virt/vmwareapi/images.py +++ b/nova/virt/vmwareapi/images.py @@ -19,7 +19,6 @@ Utility functions for Image transfer and manipulation. import os import tarfile -import tempfile from lxml import etree from oslo_config import cfg @@ -417,49 +416,42 @@ def fetch_image_ova(context, instance, session, vm_name, ds_name, session, vm_name, ds_name) read_iter = IMAGE_API.download(context, image_ref) - ova_fd, ova_path = tempfile.mkstemp() + read_handle = rw_handles.ImageReadHandle(read_iter) - try: - # NOTE(arnaud): Look to eliminate first writing OVA to file system - with os.fdopen(ova_fd, 'w') as fp: - for chunk in read_iter: - fp.write(chunk) - with tarfile.open(ova_path, mode="r") as tar: - vmdk_name = None - for tar_info in tar: - if tar_info and tar_info.name.endswith(".ovf"): - extracted = tar.extractfile(tar_info.name) - xmlstr = extracted.read() - vmdk_name = get_vmdk_name_from_ovf(xmlstr) - elif vmdk_name and tar_info.name.startswith(vmdk_name): - # Actual file name is .XXXXXXX - extracted = tar.extractfile(tar_info.name) - write_handle = rw_handles.VmdkWriteHandle( - session, - session._host, - session._port, - res_pool_ref, - vm_folder_ref, - vm_import_spec, - file_size) - start_transfer(context, - extracted, - file_size, - write_file_handle=write_handle) - extracted.close() - LOG.info(_LI("Downloaded OVA image file %(image_ref)s"), - {'image_ref': instance.image_ref}, instance=instance) - imported_vm_ref = write_handle.get_imported_vm() - session._call_method(session.vim, "UnregisterVM", - imported_vm_ref) - LOG.info(_LI("The imported VM was unregistered"), - instance=instance) - return - raise exception.ImageUnacceptable( - reason=_("Extracting vmdk from OVA failed."), - image_id=image_ref) - finally: - os.unlink(ova_path) + with tarfile.open(mode="r|", fileobj=read_handle) as tar: + vmdk_name = None + for tar_info in tar: + if tar_info and tar_info.name.endswith(".ovf"): + extracted = tar.extractfile(tar_info) + xmlstr = extracted.read() + vmdk_name = get_vmdk_name_from_ovf(xmlstr) + elif vmdk_name and tar_info.name.startswith(vmdk_name): + # Actual file name is .XXXXXXX + extracted = tar.extractfile(tar_info) + write_handle = rw_handles.VmdkWriteHandle( + session, + session._host, + session._port, + res_pool_ref, + vm_folder_ref, + vm_import_spec, + file_size) + start_transfer(context, + extracted, + file_size, + write_file_handle=write_handle) + extracted.close() + LOG.info(_LI("Downloaded OVA image file %(image_ref)s"), + {'image_ref': instance.image_ref}, instance=instance) + imported_vm_ref = write_handle.get_imported_vm() + session._call_method(session.vim, "UnregisterVM", + imported_vm_ref) + LOG.info(_LI("The imported VM was unregistered"), + instance=instance) + return + raise exception.ImageUnacceptable( + reason=_("Extracting vmdk from OVA failed."), + image_id=image_ref) def upload_image_stream_optimized(context, image_id, instance, session,