From: beierl Date: Mon, 7 Oct 2019 17:09:24 +0000 (-0400) Subject: Fixes OVF converter and upload X-Git-Tag: v7.0.0rc1~57^2 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=c6b00f0466ffe9141d9b933fa131780247af8583;p=osm%2Fdevops.git Fixes OVF converter and upload Adds the ability to specify hardware version Defaults hardware version to 14 for compatibility Displays list of OS types in help Adds a working vCD upload Adds simple usage file Cleans up installation path Bug 850 Change-Id: I1f4658e48869149c523a80401487e3e6c25dc809 Signed-off-by: beierl --- diff --git a/tools/OVF_converter/Usage.txt b/tools/OVF_converter/Usage.txt new file mode 100644 index 00000000..67671b11 --- /dev/null +++ b/tools/OVF_converter/Usage.txt @@ -0,0 +1,61 @@ +## +# Copyright 2016-2019 VMware Inc. +# This file is part of ETSI OSM +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact: osslegalrouting@vmware.com +## + +The following two tools are provided to help simplify VMware based image +management: + +o ovf_converter - converts libvirt .qcow2 files to VMware .vmdk format. +o ovf_uploader - uploads .ovf and .vmdk files to vCD, similar to + openstack image upload command. + +OVF Converter Usage +=================== + +This utility can be used for both VMware Integrated Openstack (VIO) or +vCloud Director (vCD). It converts libvirt images, such as files that end +in .qcow2 or .img to VMware .vmdk format. Both an .ovf and .vmdk file are +created and can be used as needed. + +OVF Uploader Usage +================== + +This utility is only used for vCD, as there is no equivalent CLI that provides +the ability to upload images into vCD. It takes the output OVF from the +converter and uploads both it and the VMDK file to the vCD catalog. + +Examples: +========= + +o Upload a CentOS .qcow2 file to VIO + + $ ovf_converter CentOS-7-x86_64-GenericCloud.qcow2 + $ openstack image create --public \ + --file CentOS-7-x86_64-GenericCloud.vmdk \ + --disk-format vmdk CentOS-7 + +o Upload a CentOS .qcow2 file to vCD + + $ ovf_converter CentOS-7-x86_64-GenericCloud.qcow2 + $ ovf_uploader -u orgadmin -p password -o osm -l https://vcd \ + CentOS-7-x86_64-GenericCloud.ovf + +Both commands suport -h to display help options. + diff --git a/tools/OVF_converter/converter.py b/tools/OVF_converter/converter.py new file mode 100644 index 00000000..5f31ed36 --- /dev/null +++ b/tools/OVF_converter/converter.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# # +# Copyright 2016-2017 VMware Inc. +# This file is part of ETSI OSM +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact: osslegalrouting@vmware.com +# # + +import logging +import os +import subprocess +import yaml +from lxml import etree as ET + +# file paths +MODULE_DIR = os.path.dirname(__file__) +OVF_TEMPLATE_PATH = os.path.join(MODULE_DIR, + "ovf_template/template.xml") +IDE_CDROM_XML_PATH = os.path.join(MODULE_DIR, + "ovf_template/ide_cdrom.xml") +OS_INFO_FILE_PATH = os.path.join(MODULE_DIR, + "config/os_type.yaml") +DISK_CONTROLLER_INFO_FILE_PATH = os.path.join(MODULE_DIR, + "config/disk_controller.yaml") + +# Set logger +LOG_FILE = os.path.join(MODULE_DIR, "logs/ovf_converter.log") +os.makedirs(os.path.dirname(LOG_FILE), exist_ok=True) +logger = logging.getLogger(__name__) +hdlr = logging.FileHandler(LOG_FILE) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +hdlr.setFormatter(formatter) +logger.addHandler(hdlr) +logger.setLevel(10) + +__version__ = "1.2" +__description__ = "OVF Hardware Version 14 compatible" + + +def get_version(*args, **kwargs): + """ get version of this application""" + version = str(__version__) + " - " + str(__description__) + return version + + +# converter class +class OVFConverter(object): + """ Class to convert input image into OVF format """ + + def __init__(self, source_img_path, output_location=None, output_ovf_name=None, + memory=None, cpu=None, disk=None, os_type=None, + disk_controller=None, cdrom=None, hwversion=14): + """ + Constructor to initialize object of class OVFConverter + Args: + source_img_path - absolute path to source image which will get convert into ovf + output_location - location where created OVF will be kept. This location + should have write access. If not given file will get + created at source location (optional) + output_ovf_name - name of output ovf.If not given source image name will + be used (optional) + memory - required memory for VM in MB (optional) + cpu - required number of virtual cpus for VM (optional) + disk - required size of disk for VM in GB (optional) + os_type- required operating system type as specified in user document + (default os type other 32 bit) (optional) + disk_controller - required disk controller type + (default controller SCSI with lsilogicsas) + (SATA, IDE, Paravirtual, Buslogic, Lsilogic, Lsilogicsas) (optional) + hwversion - VMware ESXi hardware family version (optional) + + Returns: + Nothing. + """ + self.logger = logger + self.ovf_template_path = OVF_TEMPLATE_PATH + + self.source_img_path = source_img_path + self.source_img_filename, file_extension = os.path.splitext(os.path.basename(self.source_img_path)) + self.source_img_location = os.path.dirname(self.source_img_path) + self.source_format = file_extension[1:] + + self.output_format = "ovf" + self.output_ovf_name = output_ovf_name.split('.')[0] if output_ovf_name else self.source_img_filename + self.output_location = output_location if output_location else "." + self.output_ovf_name_ext = self.output_ovf_name + "." + self.output_format + self.output_path = os.path.join(self.output_location, self.output_ovf_name_ext) + + self.output_diskimage_format = "vmdk" + self.output_diskimage_name = self.source_img_filename + "." + self.output_diskimage_format + self.output_diskimage_path = os.path.join(self.output_location, self.output_diskimage_name) + + self.logger.info("Input parameters to Converter: \n ovf_template_path = {}, \n source_img_path = {}, \n" + "source_img_location ={} , \n source_format = {}, \n source_img_filename = {}".format( + self.ovf_template_path, + self.source_img_path, self.source_img_location, + self.source_format, self.source_img_filename)) + + self.logger.info("Output parameters to Converter: \n output_format = {}, \n output_ovf_name = {}, \n" + "output_location ={} , \n output_path = {}, \n output_diskimage_name = {} , \n" + " output_diskimage_path = {} ".format(self.output_format, self.output_ovf_name, + self.output_location, self.output_path, + self.output_diskimage_name, self.output_diskimage_path)) + + self.disk_capacity = 1 + self.disk_populated_size = 0 + + self.vm_name = self.output_ovf_name + self.memory = str(memory) if memory is not None else None + self.cpu = str(cpu) if cpu is not None else None + self.os_type = str(os_type).strip() if os_type else None + self.cdrom = cdrom + self.hwversion = hwversion + + if self.os_type: + self.osID, self.osType = self.__get_osType() + if self.osID is None or self.osType is None: + error_msg = "ERROR: Invalid input can not find OS type {} ".format(self.os_type) + self.__raise_exception(error_msg) + + self.disk_controller = str(disk_controller).strip() if disk_controller else None + + if self.disk_controller: + self.disk_controller_info = self.__get_diskcontroller() + + if not self.disk_controller_info: + error_msg = "ERROR: Invalid input can not find Disk Controller {} ".format(self.disk_controller) + self.__raise_exception(error_msg) + + if disk is not None: + # convert disk size from GB to bytes + self.disk_size = int(disk) * 1024 * 1024 * 1024 + else: + self.disk_size = None + + self.logger.info("Other input parameters to Converter: \n vm_name = {}, \n memory = {}, \n" + "disk_size ={} \n os type = {} \n disk controller = {}".format( + self.vm_name, self.memory, self.disk_size, self.os_type, self.disk_controller)) + + # check access for read input location and write output location return none if no access + if not os.access(self.source_img_path, os.F_OK): + error_msg = "ERROR: Source image file {} not present".format(self.source_img_path) + self.__raise_exception(error_msg, exception_type="IO") + + elif not os.access(self.source_img_path, os.R_OK): + error_msg = "ERROR: Cannot read source image file {}".format(self.source_img_path) + self.__raise_exception(error_msg, exception_type="IO") + + if not os.access(self.output_location, os.W_OK): + error_msg = "ERROR: No write access to location {} to write output OVF ".format(self.output_location) + self.__raise_exception(error_msg, exception_type="IO") + + def __get_image_info(self): + """ + Private method to get information about source imager. + Args : None + Return : True on success else False + """ + try: + print("Getting source image information") + command = "qemu-img info \t " + self.source_img_path + output, error, returncode = self.__execute_command(command) + + if error or returncode: + self.logger.error("ERROR: Error occurred while getting information about source image : {} \n " + "return code : {} ".format(error, returncode)) + return False + + elif output: + self.logger.info("Get Image Info Output : {} \n ".format(output)) + split_output = output.decode().split("\n") + for line in split_output: + line = line.strip() + if "virtual size" in line: + virtual_size_info = line.split(":")[1].split() + if len(virtual_size_info) == 3 and virtual_size_info[2].strip(")") == "bytes": + self.disk_capacity = int(virtual_size_info[1].strip("(")) + else: + self.disk_capacity = self.__convert_size(virtual_size_info[0]) + + elif "disk size" in line: + size = line.split(":")[1].split()[0] + self.disk_populated_size = self.__convert_size(size) + elif "file format" in line: + self.source_format = line.split(":")[1] + + self.logger.info("Updated source image virtual disk capacity : {} ," + "Updated source image populated size: {}".format(self.disk_capacity, + self.disk_populated_size)) + return True + except Exception as exp: + error_msg = "ERROR: Error occurred while getting information about source image : {}".format(exp) + self.logger.error(error_msg) + print(error_msg) + return False + + def __convert_image(self): + """ + Private method to convert source disk image into .vmdk disk image. + Args : None + Return : True on success else False + """ + + print("Converting source disk image to .vmdk ") + + command = "qemu-img convert -p -f " + self.source_format + " -O " + self.output_diskimage_format + \ + " -o subformat=streamOptimized " + self.source_img_path + " " + self.output_diskimage_path + + _, error, returncode = self.__execute_command(command, show_output=True) + + if error or returncode: + error_msg = "ERROR: Error occurred while converting source disk image into vmdk: {}\n" + \ + "return code : {} ".format(error, returncode) + self.logger.error(error_msg) + print(error_msg) + return False + else: + if os.path.isfile(self.output_diskimage_path): + self.logger.info("Successfully converted source image {} into {} \n " + "return code : {} ".format(self.source_img_path, + self.output_diskimage_path, + returncode)) + result = self.__make_image_bootable() + if result: + self.logger.info("Made {} bootable".format(self.output_diskimage_path)) + print("Output VMDK is at: {}".format(self.output_diskimage_path)) + return True + else: + self.logger.error("Cannot make {} bootable".format(self.output_diskimage_path)) + print("ERROR: Fail to convert source image into .vmdk") + return False + else: + self.logger.error("Converted vmdk disk file {} is not present \n ".format( + self.output_diskimage_path)) + print("Fail to convert source image into .vmdk") + return False + + def __make_image_bootable(self): + """ + Private method to make source disk image bootable. + Args : None + Return : True on success else False + """ + command = "printf '\x03' | dd conv=notrunc of=" + self.output_diskimage_path + "\t bs=1 seek=$((0x4))" + output, error, returncode = self.__execute_command(command) + + if error and returncode: + error_msg = "ERROR:Error occurred while making source disk image bootable : {} \n "\ + "return code : {} ".format(error, returncode) + self.logger.error(error_msg) + print(error_msg) + return False + else: + self.logger.info("Make Image Bootable Output : {} ".format(output)) + return True + + def __edit_ovf_template(self): + """ + Private method to create new OVF file by editing OVF template + Args : None + Return : True on success else False + """ + try: + print("Creating OVF") + # Read OVF template file + OVF_tree = ET.parse(self.ovf_template_path) + root = OVF_tree.getroot() + + # Collect namespaces + nsmap = {k: v for k, v in root.nsmap.items() if k} + nsmap["xmlns"] = "http://schemas.dmtf.org/ovf/envelope/1" + + # Edit OVF template + references = root.find('xmlns:References', nsmap) + if references is not None: + file_tag = references.find('xmlns:File', nsmap) + if file_tag is not None: + file_tag.attrib['{' + nsmap['ovf'] + '}href'] = self.output_diskimage_name + + disksection = root.find('xmlns:DiskSection', nsmap) + if disksection is not None: + diak_tag = disksection.find('xmlns:Disk', nsmap) + if diak_tag is not None: + if self.disk_size and self.disk_size > self.disk_capacity: + self.disk_capacity = self.disk_size + + diak_tag.attrib['{' + nsmap['ovf'] + '}capacity'] = str(self.disk_capacity) + diak_tag.attrib['{' + nsmap['ovf'] + '}populatedSize'] = str(self.disk_populated_size) + + virtuasystem = root.find('xmlns:VirtualSystem', nsmap) + if virtuasystem is not None: + name_tag = virtuasystem.find('xmlns:Name', nsmap) + if name_tag is not None: + name_tag.text = self.vm_name + + if self.os_type is not None: + operatingSystemSection = virtuasystem.find('xmlns:OperatingSystemSection', nsmap) + if self.osID and self.osType: + operatingSystemSection.attrib['{' + nsmap['ovf'] + '}id'] = self.osID + os_discription_tag = operatingSystemSection.find('xmlns:Description', nsmap) + os_discription_tag.text = self.osType + + virtualHardwareSection = virtuasystem.find('xmlns:VirtualHardwareSection', nsmap) + system = virtualHardwareSection.find('xmlns:System', nsmap) + virtualSystemIdentifier = system.find('vssd:VirtualSystemIdentifier', nsmap) + if virtualSystemIdentifier is not None: + virtualSystemIdentifier.text = self.vm_name + VirtualSystemType = system.find('vssd:VirtualSystemType', nsmap) + if VirtualSystemType is not None: + VirtualSystemType.text = "vmx-{}".format(self.hwversion) + + if self.memory is not None or self.cpu is not None or self.disk_controller is not None: + for item in virtualHardwareSection.iterfind('xmlns:Item', nsmap): + description = item.find("rasd:Description", nsmap) + + if self.cpu is not None: + if description is not None and description.text == "Number of Virtual CPUs": + cpu_item = item.find("rasd:VirtualQuantity", nsmap) + name_item = item.find("rasd:ElementName", nsmap) + if cpu_item is not None: + cpu_item.text = self.cpu + name_item.text = self.cpu + " virtual CPU(s)" + + if self.memory is not None: + if description is not None and description.text == "Memory Size": + mem_item = item.find("rasd:VirtualQuantity", nsmap) + name_item = item.find("rasd:ElementName", nsmap) + if mem_item is not None: + mem_item.text = self.memory + name_item.text = self.memory + " MB of memory" + + if self.disk_controller is not None: + if description is not None and description.text == "SCSI Controller": + if self.disk_controller_info is not None: + name_item = item.find("rasd:ElementName", nsmap) + name_item.text = str(self.disk_controller_info["controllerName"]) + "0" + + resource_type = item.find("rasd:ResourceType", nsmap) + resource_type.text = self.disk_controller_info["resourceType"] + + description.text = self.disk_controller_info["controllerName"] + resource_subtype = item.find("rasd:ResourceSubType", nsmap) + if self.disk_controller_info["controllerName"] == "IDE Controller": + # Remove resource subtype item + resource_subtype.getparent().remove(resource_subtype) + if "resourceSubType" in self.disk_controller_info: + resource_subtype.text = self.disk_controller_info["resourceSubType"] + if self.cdrom: + last_item = list(virtualHardwareSection.iterfind('xmlns:Item', nsmap))[-1] + ide_cdrom_items_etree = ET.parse(IDE_CDROM_XML_PATH) + ide_cdrom_items = list(ide_cdrom_items_etree.iterfind('Item')) + for item in ide_cdrom_items: + last_item.addnext(item) + + # Save output OVF + OVF_tree.write(self.output_path, xml_declaration=True, encoding='utf-8', + method="xml") + + if os.path.isfile(self.output_path): + logger.info("Successfully written output OVF at {}".format(self.output_path)) + print("Output OVF is at: {}".format(self.output_path)) + return self.output_path + else: + error_msg = "ERROR: Error occurred while creating OVF file" + print(error_msg) + return False + + except Exception as exp: + error_msg = "ERROR: Error occurred while editing OVF template : {}".format(exp) + self.logger.error(error_msg) + print(error_msg) + return False + + def __convert_size(self, size): + """ + Private method to convert disk size from GB,MB to bytes. + Args : + size : disk size with prefix 'G' for GB and 'M' for MB + Return : disk size in bytes + """ + byte_size = 0 + try: + if not size: + self.logger.error("No size {} to convert in bytes".format(size)) + else: + size = str(size) + disk_size = float(size[:-1]) + input_type = size[-1].strip() + + self.logger.info("Disk size : {} , size type : {} ".format(disk_size, input_type)) + + if input_type == "G": + byte_size = disk_size * 1024 * 1024 * 1024 + elif input_type == "M": + byte_size = disk_size * 1024 * 1024 + + self.logger.info("Disk size in bytes: {} ".format(byte_size)) + + return int(byte_size) + + except Exception as exp: + error_msg = "ERROR:Error occurred while converting disk size in bytes : {}".format(exp) + self.logger.error(error_msg) + print(error_msg) + return False + + def __get_osType(self): + """ + Private method to get OS ID and Type + Args : + None + Return : + osID : OS ID + osType: OS Type + """ + osID = None + osType = None + os_info = self.__read_yaml_file(OS_INFO_FILE_PATH) + + try: + if self.os_type and os_info: + for os_id, os_type in os_info.items(): + if self.os_type.lower() == os_type.lower(): + osID = os_id + osType = os_type + break + except Exception as exp: + error_msg = "ERROR:Error occurred while getting OS details : {}".format(exp) + self.logger.error(error_msg) + print(error_msg) + + return osID, osType + + def __get_diskcontroller(self): + """ + Private method to get details of Disk Controller + Args : + None + Return : + disk_controller : dict with details of Disk Controller + """ + disk_controller = {} + scsi_subtype = None + if self.disk_controller.lower() in ["paravirtual", "lsilogic", "buslogic", "lsilogicsas"]: + scsi_subtype = self.disk_controller + self.disk_controller = "SCSI" + + disk_controller_info = self.__read_yaml_file(DISK_CONTROLLER_INFO_FILE_PATH) + try: + if self.disk_controller and disk_controller_info: + for key, value in disk_controller_info.iteritems(): + if self.disk_controller.lower() in key.lower(): + disk_controller['controllerName'] = key + disk_controller['resourceType'] = str(value["ResourceType"]) + resourceSubTypes = value["ResourceSubTypes"] if "ResourceSubTypes" in value else None + if key == "SATA Controller": + disk_controller["resourceSubType"] = resourceSubTypes[0] + elif key == "SCSI Controller": + if scsi_subtype: + if scsi_subtype.lower() == "paravirtual": + scsi_subtype = "VirtualSCSI" + for subtype in resourceSubTypes: + if scsi_subtype.lower() == subtype.lower(): + disk_controller["resourceSubType"] = subtype + break + else: + error_msg = "ERROR: Invalid inputs can not "\ + "find SCSI subtype {}".format(scsi_subtype) + self.__raise_exception(error_msg) + + except KeyError as exp: + error_msg = "ERROR:Error occurred while getting Disk Controller details : {}".format(exp) + self.logger.error(error_msg) + print(error_msg) + + return disk_controller + + def __read_yaml_file(self, file_path): + """ + Private method to execute command + Args : + command : command to execute + Return : + Dict of yaml data + """ + with open(file_path) as data_file: + data = yaml.load(data_file, Loader=yaml.SafeLoader) + return data + + def __raise_exception(self, error_msg, exception_type="Generic"): + """ + Private method to execute command + Args : + command : command to execute + Return : + None + """ + if error_msg: + self.logger.debug(error_msg) + print(error_msg) + if exception_type == "Generic": + raise Exception(error_msg) + elif exception_type == "IO": + raise Exception(error_msg) + + def __execute_command(self, command, show_output=False): + """ + Private method to execute command + Args : + command : command to execute + Return : + stdout : output of command + stderr: error occurred while executing command if any + returncode : return code of command execution + """ + try: + self.logger.info("Execute command: {} ".format(command)) + + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stdin=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True, bufsize=1) + + stdout = b'' + stderr = b'' + + while True: + output = proc.stdout.read(1) + stdout += output + if show_output: + print(output.decode(), end='') + returncode = proc.poll() + if returncode is not None: + for output in proc.stdout.readlines(): + stdout += output + if show_output: + print(output.decode(), end='') + break + + for output in proc.stderr.readlines(): + stderr += output + + except Exception as exp: + self.logger.error("Error {} occurred while executing command {} ".format(exp, command)) + + return stdout, stderr, returncode + + def create_ovf(self): + """ + Method to convert source image into OVF + Args : None + Return : True on success else False + """ + # check output format + if self.source_format == self.output_format: + self.logger.info("Source format is OVF. No need to convert: {} ") + return self.source_img_path + + # Get source img properties + img_info = self.__get_image_info() + if img_info: + + # Create vmdk disk image + disk_img = self.__convert_image() + if disk_img: + + # Edit OVF tempalte + ovf_path = self.__edit_ovf_template() + return ovf_path + else: + self.logger.error("Error in getting image information cannot convert image") + raise Exception("Error in getting image information cannot convert image") + return False diff --git a/tools/OVF_converter/format_converter/__init__.py b/tools/OVF_converter/format_converter/__init__.py deleted file mode 100644 index db9e016c..00000000 --- a/tools/OVF_converter/format_converter/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -## -# Copyright 2016-2017 VMware Inc. -# This file is part of ETSI OSM -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# For those usages not covered by the Apache License, Version 2.0 please -# contact: osslegalrouting@vmware.com -## diff --git a/tools/OVF_converter/format_converter/command_progress.py b/tools/OVF_converter/format_converter/command_progress.py deleted file mode 100644 index cbfb7378..00000000 --- a/tools/OVF_converter/format_converter/command_progress.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -## -# Copyright 2016-2017 VMware Inc. -# This file is part of ETSI OSM -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# For those usages not covered by the Apache License, Version 2.0 please -# contact: osslegalrouting@vmware.com -## - -import threading -import sys - -class RepeatingTimer(threading._Timer): - """ Class to run thread parally """ - def run(self): - """ Method to run thread """ - while True: - self.finished.wait(self.interval) - if self.finished.is_set(): - return - else: - self.function(*self.args, **self.kwargs) - - -class CommandProgressbar(object): - """ Class to show progressbar while waiting fro command output """ - - def __init__(self): - self.timer = None - - def __show_progressbar(self): - """ - Private method to show progressbar while waiting for command to complete - Args : None - Return : None - """ - print '\b.', - sys.stdout.flush() - - def start_progressbar(self): - """ - Method to start progressbar thread - Args : None - Return : None - """ - self.timer = RepeatingTimer(1.0, self.__show_progressbar) - self.timer.daemon = True # Allows program to exit if only the thread is alive - self.timer.start() - - def stop_progressbar(self): - """ - Method to stop progressbar thread - Args : None - Return : None - """ - self.timer.cancel() diff --git a/tools/OVF_converter/format_converter/converter.py b/tools/OVF_converter/format_converter/converter.py deleted file mode 100644 index ca2a3691..00000000 --- a/tools/OVF_converter/format_converter/converter.py +++ /dev/null @@ -1,578 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -## -# Copyright 2016-2017 VMware Inc. -# This file is part of ETSI OSM -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# For those usages not covered by the Apache License, Version 2.0 please -# contact: osslegalrouting@vmware.com -## - -import logging -import os -import subprocess -import yaml -from lxml import etree as ET -from command_progress import CommandProgressbar - -#file paths -MODULE_DIR = os.path.dirname(__file__) -OVF_TEMPLATE_PATH = os.path.join(os.path.dirname(MODULE_DIR), - "ovf_template/template.xml") -IDE_CDROM_XML_PATH = os.path.join(os.path.dirname(MODULE_DIR), - "ovf_template/ide_cdrom.xml") -OS_INFO_FILE_PATH = os.path.join(os.path.dirname(MODULE_DIR), - "config/os_type.yaml") -DISK_CONTROLLER_INFO_FILE_PATH = os.path.join(os.path.dirname(MODULE_DIR), - "config/disk_controller.yaml") - - -#Set logger -LOG_FILE = os.path.join(os.path.dirname(MODULE_DIR),"logs/ovf_converter.log") -logger = logging.getLogger(__name__) -hdlr = logging.FileHandler(LOG_FILE) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') -hdlr.setFormatter(formatter) -logger.addHandler(hdlr) -logger.setLevel(10) - -__version__ = "1.0" -__description__ = "initial Release" - -def get_version(*args, **kwargs): - """ get version of this application""" - version = str(__version__ ) +" - "+ str( __description__ ) - return version - -#converter class -class OVFConverter(object): - """ Class to convert input image into OVF format """ - - def __init__(self, source_img_path, output_location=None, output_ovf_name=None, - memory=None, cpu=None, disk=None, os_type=None, - disk_controller=None, cdrom=None, - options={'subformat':'streamOptimized'}): - """ - Constructor to initialize object of class OVFConverter - Args: - source_img_path - absolute path to source image which will get convert into ovf - output_location - location where created OVF will be kept. This location - should have write access. If not given file will get - created at source location (optional) - output_ovf_name - name of output ovf.If not given source image name will - be used (optional) - memory - required memory for VM in MB (optional) - cpu - required number of virtual cpus for VM (optional) - disk - required size of disk for VM in GB (optional) - os_type- required operating system type as specified in user document - (default os type other 32 bit) (optional) - disk_controller - required disk controller type - (default controller SCSI with lsilogicsas) - (SATA, IDE, Paravirtual, Buslogic, Lsilogic, Lsilogicsas) (optional) - options - subformat option for OVF (optional) - - Returns: - Nothing. - """ - self.logger = logger - self.ovf_template_path = OVF_TEMPLATE_PATH - - self.source_img_path = source_img_path - file_location, file_extension = os.path.splitext(self.source_img_path) - self.source_img_location = os.path.dirname(self.source_img_path) - self.source_format = file_extension[1:] - self.source_img_filename = os.path.basename(self.source_img_path).split('.')[0] - - self.output_format = "ovf" - self.output_ovf_name = output_ovf_name.split('.')[0] if output_ovf_name else self.source_img_filename - self.output_location = output_location if output_location else self.source_img_location - self.output_ovf_name_ext = self.output_ovf_name + "." + self.output_format - self.output_path = os.path.join(self.output_location , self.output_ovf_name_ext ) - - self.output_diskimage_format = "vmdk" - self.output_diskimage_name = self.source_img_filename + "."+ self.output_diskimage_format - self.output_diskimage_path = os.path.join(self.output_location, self.output_diskimage_name) - - - self.logger.info("Input parameters to Converter: \n ovf_template_path = {}, \n source_img_path = {}, \n"\ - "source_img_location ={} , \n source_format = {}, \n source_img_filename = {}".format( - self.ovf_template_path, - self.source_img_path, self.source_img_location, - self.source_format, self.source_img_filename )) - - self.logger.info("Output parameters to Converter: \n output_format = {}, \n output_ovf_name = {}, \n"\ - "output_location ={} , \n output_path = {}, \n output_diskimage_name = {} , \n"\ - " output_diskimage_path = {} ".format(self.output_format, self.output_ovf_name, - self.output_location, self.output_path, - self.output_diskimage_name,self.output_diskimage_path )) - - - self.disk_capacity = 1 - self.disk_populated_size = 0 - - self.vm_name = self.output_ovf_name - self.memory = str(memory) if memory is not None else None - self.cpu = str(cpu) if cpu is not None else None - self.os_type=str(os_type).strip() if os_type else None - self.cdrom = cdrom - - if self.os_type: - self.osID , self.osType = self.__get_osType() - if self.osID is None or self.osType is None: - error_msg = "ERROR: Invalid input can not find OS type {} ".format(self.os_type) - self.__raise_exception(error_msg) - - self.disk_controller = str(disk_controller).strip() if disk_controller else None - - if self.disk_controller: - self.disk_controller_info = self.__get_diskcontroller() - - if not self.disk_controller_info: - error_msg = "ERROR: Invalid input can not find Disk Controller {} ".format(self.disk_controller) - self.__raise_exception(error_msg) - - if disk is not None: - #convert disk size from GB to bytes - self.disk_size = int(disk) * 1024 * 1024 * 1024 - else: - self.disk_size = None - - self.logger.info("Other input parameters to Converter: \n vm_name = {}, \n memory = {}, \n"\ - "disk_size ={} \n os type = {} \n disk controller = {}".format( - self.vm_name, self.memory, self.disk_size, self.os_type, self.disk_controller - )) - - #check access for read input location and write output location return none if no access - if not os.access(self.source_img_path, os.F_OK): - error_msg = "ERROR: Source image file {} not present".format(self.source_img_path) - self.__raise_exception(error_msg, exception_type="IO") - - elif not os.access(self.source_img_path, os.R_OK): - error_msg = "ERROR: Cannot read source image file {}".format(self.source_img_path) - self.__raise_exception(error_msg, exception_type="IO") - - if not os.access(self.output_location, os.W_OK): - error_msg = "ERROR: Not have write access to location {} to write output OVF ".format(self.source_img_path) - self.__raise_exception(error_msg, exception_type="IO") - - def __get_image_info(self): - """ - Private method to get information about source imager. - Args : None - Return : True on success else False - """ - try: - print("Getting source image information") - command = "qemu-img info \t " + self.source_img_path - output, error, returncode= self.__execute_command(command) - - if error or returncode: - self.logger.error("ERROR: Error occurred while getting information about source image : {} \n "\ - "return code : {} ".format(error, returncode)) - return False - - elif output: - self.logger.info("Get Image Info Output : {} \n ".format(output)) - split_output = output.split("\n") - for line in split_output: - line = line.strip() - if "virtual size" in line: - virtual_size_info = line.split(":")[1].split() - if len(virtual_size_info) == 3 and virtual_size_info[2].strip(")") == "bytes": - self.disk_capacity = int(virtual_size_info[1].strip("(")) - else: - self.disk_capacity = self.__convert_size(virtual_size_info[0]) - - elif "disk size" in line: - size = line.split(":")[1].split()[0] - self.disk_populated_size = self.__convert_size(size) - elif "file format" in line: - self.source_format = line.split(":")[1] - - self.logger.info("Updated source image virtual disk capacity : {} ,"\ - "Updated source image populated size: {}".format(self.disk_capacity, - self.disk_populated_size)) - return True - except Exception as exp: - error_msg = "ERROR: Error occurred while getting information about source image : {}".format(exp) - self.logger.error(error_msg) - print(error_msg) - return False - - def __convert_image(self): - """ - Private method to convert source disk image into .vmdk disk image. - Args : None - Return : True on success else False - """ - - print("Converting source disk image to .vmdk ") - - progress = CommandProgressbar() - progress.start_progressbar() - - command = "qemu-img convert -f "+ self.source_format +" -O " + self.output_diskimage_format + \ - " -o subformat=streamOptimized " + self.source_img_path + "\t" + self.output_diskimage_path - - output, error , returncode = self.__execute_command(command) - - progress.stop_progressbar() - - if error or returncode : - error_msg = "ERROR: Error occurred while converting source disk image into vmdk : {} \n "\ - "return code : {} ".format(error, returncode) - self.logger.error(error_msg) - print(error_msg) - return False - else: - if os.path.isfile(self.output_diskimage_path): - self.logger.info("Successfully converted source image {} into {} \n "\ - "return code : {} ".format(self.source_img_path, - self.output_diskimage_path, - returncode)) - result = self.__make_image_bootable() - if result: - self.logger.info("Made {} bootable".format(self.output_diskimage_path)) - return True - else: - self.logger.error("Cannot make {} bootable".format(self.output_diskimage_path)) - print("ERROR: Fail to convert source image into .vmdk") - return False - else: - self.logger.error("Converted vmdk disk file {} is not present \n ".format( - self.output_diskimage_path)) - print("Fail to convert source image into .vmdk") - return False - - def __make_image_bootable(self): - """ - Private method to make source disk image bootable. - Args : None - Return : True on success else False - """ - command = "printf '\x03' | dd conv=notrunc of="+ self.output_diskimage_path + "\t bs=1 seek=$((0x4))" - output, error, returncode = self.__execute_command(command) - - if error and returncode : - error_msg = "ERROR:Error occurred while making source disk image bootable : {} \n "\ - "return code : {} ".format(error, returncode) - self.logger.error(error_msg) - print(error_msg) - return False - else: - self.logger.info("Make Image Bootable Output : {} ".format(output)) - return True - - - def __edit_ovf_template(self): - """ - Private method to create new OVF file by editing OVF template - Args : None - Return : True on success else False - """ - try: - print("\nCreating OVF") - #Read OVF template file - OVF_tree = ET.parse(self.ovf_template_path) - root = OVF_tree.getroot() - - #Collect namespaces - nsmap = {k:v for k,v in root.nsmap.iteritems() if k} - nsmap["xmlns"]= "http://schemas.dmtf.org/ovf/envelope/1" - - #Edit OVF template - references = root.find('xmlns:References',nsmap) - if references is not None: - file_tag = references.find('xmlns:File', nsmap) - if file_tag is not None: - file_tag.attrib['{'+nsmap['ovf']+'}href'] = self.output_diskimage_name - - disksection = root.find('xmlns:DiskSection',nsmap) - if disksection is not None: - diak_tag = disksection.find('xmlns:Disk', nsmap) - if diak_tag is not None: - if self.disk_size and self.disk_size > self.disk_capacity: - self.disk_capacity = self.disk_size - - diak_tag.attrib['{'+nsmap['ovf']+'}capacity'] = str(self.disk_capacity) - diak_tag.attrib['{'+nsmap['ovf']+'}populatedSize'] = str(self.disk_populated_size) - - virtuasystem = root.find('xmlns:VirtualSystem',nsmap) - if virtuasystem is not None: - name_tag = virtuasystem.find('xmlns:Name', nsmap) - if name_tag is not None: - name_tag.text = self.vm_name - - if self.os_type is not None: - operatingSystemSection = virtuasystem.find('xmlns:OperatingSystemSection', nsmap) - if self.osID and self.osType: - operatingSystemSection.attrib['{'+nsmap['ovf']+'}id'] = self.osID - os_discription_tag = operatingSystemSection.find('xmlns:Description', nsmap) - os_discription_tag.text = self.osType - - virtualHardwareSection = virtuasystem.find('xmlns:VirtualHardwareSection', nsmap) - system = virtualHardwareSection.find('xmlns:System', nsmap) - virtualSystemIdentifier = system.find('vssd:VirtualSystemIdentifier', nsmap) - if virtualSystemIdentifier is not None: - virtualSystemIdentifier.text = self.vm_name - - if self.memory is not None or self.cpu is not None or self.disk_controller is not None: - for item in virtualHardwareSection.iterfind('xmlns:Item',nsmap): - description = item.find("rasd:Description",nsmap) - - if self.cpu is not None: - if description is not None and description.text == "Number of Virtual CPUs": - cpu_item = item.find("rasd:VirtualQuantity", nsmap) - name_item = item.find("rasd:ElementName", nsmap) - if cpu_item is not None: - cpu_item.text = self.cpu - name_item.text = self.cpu+" virtual CPU(s)" - - if self.memory is not None: - if description is not None and description.text == "Memory Size": - mem_item = item.find("rasd:VirtualQuantity", nsmap) - name_item = item.find("rasd:ElementName", nsmap) - if mem_item is not None: - mem_item.text = self.memory - name_item.text = self.memory + " MB of memory" - - if self.disk_controller is not None: - if description is not None and description.text == "SCSI Controller": - if self.disk_controller_info is not None: - name_item = item.find("rasd:ElementName", nsmap) - name_item.text = str(self.disk_controller_info["controllerName"])+"0" - - resource_type = item.find("rasd:ResourceType", nsmap) - resource_type.text = self.disk_controller_info["resourceType"] - - description.text = self.disk_controller_info["controllerName"] - resource_subtype = item.find("rasd:ResourceSubType", nsmap) - if self.disk_controller_info["controllerName"] == "IDE Controller": - #Remove resource subtype item - resource_subtype.getparent().remove(resource_subtype) - if "resourceSubType" in self.disk_controller_info: - resource_subtype.text = self.disk_controller_info["resourceSubType"] - if self.cdrom: - last_item = list(virtualHardwareSection.iterfind('xmlns:Item',nsmap))[-1] - ide_cdrom_items_etree = ET.parse(IDE_CDROM_XML_PATH) - ide_cdrom_items = list(ide_cdrom_items_etree.iterfind('Item')) - for item in ide_cdrom_items: - last_item.addnext(item) - - # Save output OVF - OVF_tree.write(self.output_path, xml_declaration=True,encoding='utf-8', - method="xml" ) - - if os.path.isfile(self.output_path): - logger.info("Successfully written output OVF at {}".format(self.output_path)) - print("Output OVF is at : {}".format(self.output_path)) - return self.output_path - else: - error_msg = "ERROR: Error occurred while creating OVF file" - print(error_msg) - return False - - except Exception as exp: - error_msg = "ERROR: Error occurred while editing OVF template : {}".format(exp) - self.logger.error(error_msg) - print(error_msg) - return False - - - def __convert_size(self,size): - """ - Private method to convert disk size from GB,MB to bytes. - Args : - size : disk size with prefix 'G' for GB and 'M' for MB - Return : disk size in bytes - """ - byte_size= 0 - try: - if not size: - self.logger.error("No size {} to convert in bytes".format(size)) - else: - size = str(size) - disk_size = float(size[:-1]) - input_type = size[-1].strip() - - self.logger.info("Disk size : {} , size type : {} ".format(disk_size,input_type)) - - if input_type == "G": - byte_size = disk_size * 1024 * 1024 *1024 - elif input_type == "M": - byte_size = disk_size * 1024 * 1024 - - self.logger.info("Disk size in bytes: {} ".format(byte_size)) - - return int(byte_size) - - except Exception as exp: - error_msg = "ERROR:Error occurred while converting disk size in bytes : {}".format(exp) - self.logger.error(error_msg) - print(error_msg) - return False - - def __get_osType(self): - """ - Private method to get OS ID and Type - Args : - None - Return : - osID : OS ID - osType: OS Type - """ - osID = None - osType = None - os_info = self.__read_yaml_file(OS_INFO_FILE_PATH) - try: - if self.os_type and os_info: - for os_id , os_type in os_info.iteritems(): - if self.os_type.lower() == os_type.lower(): - osID = os_id - osType = os_type - break - except Exception as exp: - error_msg = "ERROR:Error occurred while getting OS details : {}".format(exp) - self.logger.error(error_msg) - print(error_msg) - - return osID, osType - - - def __get_diskcontroller(self): - """ - Private method to get details of Disk Controller - Args : - None - Return : - disk_controller : dict with details of Disk Controller - """ - disk_controller = {} - scsi_subtype = None - if self.disk_controller.lower() in ["paravirtual", "lsilogic", "buslogic", "lsilogicsas"]: - scsi_subtype = self.disk_controller - self.disk_controller = "SCSI" - - disk_controller_info = self.__read_yaml_file(DISK_CONTROLLER_INFO_FILE_PATH) - try: - if self.disk_controller and disk_controller_info: - for key , value in disk_controller_info.iteritems(): - if self.disk_controller.lower() in key.lower(): - disk_controller['controllerName'] = key - disk_controller['resourceType'] = str(value["ResourceType"]) - resourceSubTypes = value["ResourceSubTypes"] if "ResourceSubTypes" in value else None - if key == "SATA Controller": - disk_controller["resourceSubType"] = resourceSubTypes[0] - elif key == "SCSI Controller": - if scsi_subtype: - if scsi_subtype.lower() == "paravirtual": - scsi_subtype = "VirtualSCSI" - for subtype in resourceSubTypes: - if scsi_subtype.lower() == subtype.lower(): - disk_controller["resourceSubType"] = subtype - break - else: - error_msg = "ERROR: Invalid inputs can not "\ - "find SCSI subtype {}".format(scsi_subtype) - self.__raise_exception(error_msg) - - except KeyError as exp: - error_msg = "ERROR:Error occurred while getting Disk Controller details : {}".format(exp) - self.logger.error(error_msg) - print(error_msg) - - return disk_controller - - def __read_yaml_file(self, file_path): - """ - Private method to execute command - Args : - command : command to execute - Return : - Dict of yaml data - """ - with open(file_path) as data_file: - data = yaml.load(data_file) - return data - - def __raise_exception(self, error_msg , exception_type="Generic"): - """ - Private method to execute command - Args : - command : command to execute - Return : - None - """ - if error_msg: - self.logger.debug(error_msg) - print(error_msg) - if exception_type == "Generic": - raise Exception(error_msg) - elif exception_type == "IO": - raise Exception(error_msg) - - def __execute_command(self, command): - """ - Private method to execute command - Args : - command : command to execute - Return : - stdout : output of command - stderr: error occurred while executing command if any - returncode : return code of command execution - """ - try: - self.logger.info("Execute command: {} ".format(command)) - - proc = subprocess.Popen(command , stdout = subprocess.PIPE, stdin = subprocess.PIPE, - stderr = subprocess.PIPE,shell=True) - stdout, stderr = proc.communicate() - returncode = proc.returncode - - except Exception as exp: - self.logger.error("Error {} occurred while executing command {} ".format(exp,command)) - - return stdout, stderr , returncode - - - def create_ovf(self): - """ - Method to convert source image into OVF - Args : None - Return : True on success else False - """ - #check output format - if self.source_format == self.output_format: - self.logger.info("Source format is OVF. No need to convert: {} ") - return self.source_img_path - - #Get source img properties - img_info = self.__get_image_info() - if img_info: - - #Create vmdk disk image - disk_img = self.__convert_image() - if disk_img: - - #Edit OVF tempalte - ovf_path = self.__edit_ovf_template() - return ovf_path - else: - self.logger.error("Error in getting image information cannot convert image") - raise Exception("Error in getting image information cannot convert image") - return False - diff --git a/tools/OVF_converter/format_converter/ovf_converter_cli.py b/tools/OVF_converter/format_converter/ovf_converter_cli.py deleted file mode 100644 index a505bfee..00000000 --- a/tools/OVF_converter/format_converter/ovf_converter_cli.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -## -# Copyright 2016-2017 VMware Inc. -# This file is part of ETSI OSM -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# For those usages not covered by the Apache License, Version 2.0 please -# contact: osslegalrouting@vmware.com -## - -import argparse -from converter import OVFConverter , get_version - -def execute_cli(): - - """ - Method to parse CLI arguments and execute commands accordingly - Args : None - Return : None - """ - parser = argparse.ArgumentParser(description='OVF converter to convert .qcow2 or raw image into OVF') - - parser.add_argument("-v","--version", action="version", version=str(get_version()), - help="shows version of OVF Converter tool") - - parser.add_argument("path", action="store", - help=" absolute path to source image which will get convert into ovf") - - parser.add_argument("-o", "--output_location", action="store", - help="location where created OVF will be kept. This location "\ - "should have write access. If not given file will get "\ - "created at source location (optional)") - - parser.add_argument("-n","--ovf_name", action="store", - help="name of output ovf file. If not given source image name will "\ - " be used (optional)") - - parser.add_argument("-m","--memory", action="store", - help="required memory for VM in MB (default 1 GB)(optional)") - - parser.add_argument("-c","--cpu", action="store", - help="required number of virtual cpus for VM (default 1 cpu) (optional)") - - parser.add_argument("-d","--disk", action="store", - help="required size of disk for VM in GB "\ - "(default as in source disk img) (optional)") - - parser.add_argument("-s","--osType", action="store", - help="required operating system type as specified "\ - "in user document (default os type other 32 bit) (optional)") - - parser.add_argument("-dc","--disk_Controller", action="store", - help="required disk controller type "\ - " (default controller SCSI with lsilogicsas) "\ - "(SATA, IDE, Paravirtual, Buslogic, Lsilogic, Lsilogicsas) (optional)") - - parser.add_argument("--cdrom", action="store_true", - help="whether to include a cd/dvd device (optional)") - - args = parser.parse_args() - - if args.path: - con = OVFConverter(args.path, - output_location=args.output_location, - output_ovf_name=args.ovf_name, - memory=args.memory, - cpu=args.cpu, - disk=args.disk, - os_type=args.osType, - disk_controller=args.disk_Controller, - cdrom=args.cdrom, - ) - - print("#### Start OVF conversion ####") - con.create_ovf() - print("#### Completed OVF conversion ####") - - -if __name__ == "__main__": - execute_cli() - diff --git a/tools/OVF_converter/install.sh b/tools/OVF_converter/install.sh index 7b184c80..d25c1f63 100755 --- a/tools/OVF_converter/install.sh +++ b/tools/OVF_converter/install.sh @@ -22,22 +22,23 @@ ## echo ' - ################################################################# - ##### Installing Require Packages ##### - #################################################################' + ################################################################ + ##### Installing Required Packages ##### + ################################################################' # Paths to copy application files Install_dir="/usr/local/bin" App_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -App_CLI_path="${App_dir}/format_converter/ovf_converter_cli.py" -Install_CLI_path="${Install_dir}/OVF_converter/format_converter/ovf_converter_cli.py" +Converter_App_CLI_path="${App_dir}/ovf_converter_cli.py" +Converter_Install_CLI_path="${Install_dir}/OVF_converter/ovf_converter_cli.py" +Uploader_App_CLI_path="${App_dir}/ovf_uploader_cli.py" +Uploader_Install_CLI_path="${Install_dir}/OVF_converter/ovf_uploader_cli.py" Logs_Folder="${Install_dir}/OVF_converter/logs" -Log_Path="${Install_dir}/OVF_converter/logs/ovf_converter.log" #Function to install packages using apt-get function install_packages(){ [ -x /usr/bin/apt-get ] && apt-get install -y $* - + #check properly installed for PACKAGE in $* do @@ -50,7 +51,7 @@ function install_packages(){ fi done } - + apt-get update # To get the latest package lists install_packages "libxml2-dev libxslt-dev python-dev python-pip python-lxml python-yaml" install_packages "qemu-utils" @@ -62,18 +63,21 @@ cp -R "${App_dir}" "${Install_dir}" #Create logs folder and file mkdir "${Logs_Folder}" -touch "${Log_Path}" #Change permission -chmod -R 777 $Install_CLI_path -chmod -R 777 $Log_Path +chmod -R 777 ${Converter_Install_CLI_path} +chmod -R 777 ${Uploader_Install_CLI_path} +chmod -R 777 ${Logs_Folder} touch "${Install_dir}/ovf_converter" echo "#!/bin/sh" > "${Install_dir}/ovf_converter" -echo "python ${Install_CLI_path} \"\$@\"" >> "${Install_dir}/ovf_converter" +echo "python3 ${Converter_Install_CLI_path} \"\$@\"" >> "${Install_dir}/ovf_converter" chmod a+x "${Install_dir}/ovf_converter" -echo ' - ################################################################# - ##### Done ##### - #################################################################' +touch "${Install_dir}/ovf_uploader" +echo "#!/bin/sh" > "${Install_dir}/ovf_uploader" +echo "python3 ${Uploader_Install_CLI_path} \"\$@\"" >> "${Install_dir}/ovf_uploader" +chmod a+x "${Install_dir}/ovf_uploader" + +echo "Installation complete. More information can be found at:" +echo " ${Install_dir}/OVF_converter/Usage.txt" diff --git a/tools/OVF_converter/ovf_converter_cli.py b/tools/OVF_converter/ovf_converter_cli.py new file mode 100644 index 00000000..afc70379 --- /dev/null +++ b/tools/OVF_converter/ovf_converter_cli.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +## +# Copyright 2016-2017 VMware Inc. +# This file is part of ETSI OSM +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact: osslegalrouting@vmware.com +## + +import argparse +import yaml + +from converter import OVFConverter, get_version, OS_INFO_FILE_PATH + + +def execute_cli(): + + """ + Method to parse CLI arguments and execute commands accordingly + Args : None + Return : None + """ + + with open(OS_INFO_FILE_PATH) as data_file: + os_info = yaml.load(data_file, Loader=yaml.SafeLoader) + + valid_os_strings = "Valid values for osType are:\n" + for os_name in os_info.values(): + valid_os_strings += os_name + ", " + + valid_os_strings = valid_os_strings[:-2] + + parser = argparse.ArgumentParser(description='OVF converter to convert .qcow2 or raw image into OVF') + + parser.add_argument("-v", "--version", action="version", version=str(get_version()), + help="shows version of OVF Converter tool") + + parser.add_argument("path", action="store", + help="absolute path to source image which will get converted into ovf") + + parser.add_argument("-o", "--output_location", action="store", + help="location where created OVF will be kept. This location " + "should have write access. If not given file will get " + "created at source location (optional)") + + parser.add_argument("-n", "--ovf_name", action="store", + help="name of output ovf file. If not given source image name will " + " be used (optional)") + + parser.add_argument("-m", "--memory", action="store", + help="required memory for VM in MB (default 1 GB)(optional)") + + parser.add_argument("-c", "--cpu", action="store", + help="required number of virtual cpus for VM (default 1 cpu) (optional)") + + parser.add_argument("-d", "--disk", action="store", + help="required size of disk for VM in GB " + "(default as in source disk img) (optional)") + + parser.add_argument("-s", "--osType", action="store", + help="required operating system type as specified " + "in user document (default os type other 32 bit) (optional) " + valid_os_strings) + + parser.add_argument("-dc", "--disk_Controller", action="store", + help="required disk controller type " + "(default controller SCSI with lsilogicsas) " + "(SATA, IDE, Paravirtual, Buslogic, Lsilogic, Lsilogicsas) (optional)") + + parser.add_argument("--cdrom", action="store_true", default=True, + help="whether to include a cd/dvd device (optional)") + + parser.add_argument("-hw", "--hwversion", action="store", default=14, + help="Virtual hardware version (default 14)") + + args = parser.parse_args() + + if args.path: + con = OVFConverter(args.path, + output_location=args.output_location, + output_ovf_name=args.ovf_name, + memory=args.memory, + cpu=args.cpu, + disk=args.disk, + os_type=args.osType, + disk_controller=args.disk_Controller, + cdrom=args.cdrom, + hwversion=args.hwversion, + ) + + con.create_ovf() + + +if __name__ == "__main__": + execute_cli() diff --git a/tools/OVF_converter/ovf_template/ide_cdrom.xml b/tools/OVF_converter/ovf_template/ide_cdrom.xml index ffab3928..431f38a9 100644 --- a/tools/OVF_converter/ovf_template/ide_cdrom.xml +++ b/tools/OVF_converter/ovf_template/ide_cdrom.xml @@ -1,3 +1,23 @@ + 0 diff --git a/tools/OVF_converter/ovf_template/template.xml b/tools/OVF_converter/ovf_template/template.xml index 5cbb3280..a963582f 100644 --- a/tools/OVF_converter/ovf_template/template.xml +++ b/tools/OVF_converter/ovf_template/template.xml @@ -1,87 +1,107 @@ - - - - - - - Virtual disk information - - - - - List of logical networks used in the package - - - - A virtual machine - Virtual machine name - - - The kind of installed guest operating system - Other (32-bit) - - - - Virtual hardware requirements - - Virtual Hardware Family - 0 - Test1 - vmx-7 - - - - hertz * 10^6 - Number of Virtual CPUs - 1 virtual CPU(s) - 1 - 3 - 1 - - - byte * 2^20 - Memory Size - 1GB of memory - 2 - 4 - 1024 - - - 0 - SCSI Controller - scsiController0 - 3 - lsilogicsas - 6 - - - 0 - disk0 - ovf:/disk/vmdisk1 - 4 - 3 - 17 - - - false - video - 5 - 24 - - - false - vmci - 6 - vmware.vmci - 1 - - - - - - - - - - - \ No newline at end of file + + + + + + + + Virtual disk information + + + + + List of logical networks used in the package + + + + A virtual machine + Virtual machine name + + + The kind of installed guest operating system + Other (32-bit) + + + + Virtual hardware requirements + + Virtual Hardware Family + 0 + Test1 + vmx-14 + + + + hertz * 10^6 + Number of Virtual CPUs + 1 virtual CPU(s) + 1 + 3 + 1 + + + byte * 2^20 + Memory Size + 1GB of memory + 2 + 4 + 1024 + + + 0 + SCSI Controller + scsiController0 + 3 + lsilogicsas + 6 + + + 0 + disk0 + ovf:/disk/vmdisk1 + 4 + 3 + 17 + + + false + video + 5 + 24 + + + false + vmci + 6 + vmware.vmci + 1 + + + + + + + + + + + diff --git a/tools/OVF_converter/ovf_uploader_cli.py b/tools/OVF_converter/ovf_uploader_cli.py new file mode 100644 index 00000000..e487877e --- /dev/null +++ b/tools/OVF_converter/ovf_uploader_cli.py @@ -0,0 +1,75 @@ +## +# Copyright 2019 VMware Inc. +# This file is part of ETSI OSM +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact: osslegalrouting@vmware.com +## + +import argparse +from uploader import OVFUploader, get_version + + +def execute_cli(): + + """ + Method to parse CLI arguments and execute commands accordingly + Args : None + Return : None + """ + parser = argparse.ArgumentParser(description='Utility to upload an OVF package to vCD') + + parser.add_argument("-v", "--version", action="version", version=str(get_version()), + help="shows version of vCD Uploader tool") + + parser.add_argument("ovf_file", action="store", + help="filename of OVF file to upload to vCD") + + parser.add_argument("-l", "--vcd_url", action="store", + required=True, + help="URL for vCD login (ie: https://vcd.local/") + + parser.add_argument("-u", "--username", action="store", + required=True, + help="Username for vCD login") + + parser.add_argument("-p", "--password", action="store", + required=True, + help="Password for vCD login") + + parser.add_argument("-o", "--orgname", action="store", + required=True, + help="Organization name for vCD login") + + args = parser.parse_args() + + if args.ovf_file: + try: + uploader = OVFUploader(args.ovf_file, + vcd_url=args.vcd_url, + username=args.username, + password=args.password, + orgname=args.orgname) + uploader.make_catalog() + uploader.upload_ovf() + uploader.wait_for_task_completion() + except Exception as exp: + print(exp) + exit(1) + + +if __name__ == "__main__": + execute_cli() diff --git a/tools/OVF_converter/readme.txt b/tools/OVF_converter/readme.txt index f74acb1e..1804a1de 100644 --- a/tools/OVF_converter/readme.txt +++ b/tools/OVF_converter/readme.txt @@ -1,5 +1,5 @@ ## -# Copyright 2016-2017 VMware Inc. +# Copyright 2016-2019 VMware Inc. # This file is part of ETSI OSM # All Rights Reserved. # @@ -19,6 +19,7 @@ # contact: osslegalrouting@vmware.com ## + This README file explains how to use OVF Converter Tool to generate OVF file from .qcow2, .img (raw) disk images. CONTENTS OF THIS FILE @@ -35,9 +36,14 @@ o Release History Introduction ================== -OVF Converter Tool is a Command Line Interface (CLI) basically designed to generate OVF files from disk image such as .qcow2, and .img (raw format) which are other than .vmdk. -Given a path to .qcow2 image this tool can generate .vmdk and .ovf files. -User can optionally specify name and location of ovf file. Also, one can use command line options to give disk size (GB), memory (MB), number of cpus, OS type, disk controller required for VM which will get deployed using generated OVF. +OVF Converter Tool is a Command Line Interface (CLI) basically designed to generate OVF files from disk image such as +.qcow2, and .img (raw format) which are other than .vmdk. Given a path to .qcow2 image this tool can generate .vmdk +and .ovf files. + +The user can optionally specify name and location of ovf file. Also, one can use command line options to give disk +size (GB), memory (MB), number of cpus, OS type, disk controller required for VM which will get deployed using +generated OVF. + Generated OVF file can be used to deploy VM in Vmware vSphere or vCloud Director. Note- Currently this tool supports only Ubuntu platform. @@ -47,17 +53,16 @@ Tool Requirements ================== This tool requires the following software package: -o apt-get package manager -o qemu-utils -o python 2.7 -o python-lxml -o libxml2-dev -o libxslt-dev -o python-dev -o python-pip - -Install.sh script in this folder will install all of these -software packages. +o apt-get package manager +o qemu-utils +o python 2.7 +o python-lxml +o libxml2-dev +o libxslt-dev +o python-dev +o python-pip + +Install.sh script in this folder will install all of these software packages. Installation @@ -68,35 +73,35 @@ Follow below setups for installation of OVF Converter Tool. chmod a+x install.sh 2. Run install.sh script as: ./install.sh - + Sample output - - ################################################################# - ##### Installing Require Packages ##### - ################################################################# - Hit:1 http://us.archive.ubuntu.com/ubuntu xenial InRelease - Get:2 http://us.archive.ubuntu.com/ubuntu xenial-updates InRelease [102 kB] - Get:3 http://security.ubuntu.com/ubuntu xenial-security InRelease [102 kB] - Get:4 http://us.archive.ubuntu.com/ubuntu xenial-backports InRelease [102 kB] - Fetched 306 kB in 1s (286 kB/s) - Reading package lists... Done - Reading package lists... Done - Building dependency tree - Reading state information... Done - Note, selecting 'libxslt1-dev' instead of 'libxslt-dev' - libxslt1-dev is already the newest version (1.1.28-2.1). - python-dev is already the newest version (2.7.11-1). - python-lxml is already the newest version (3.5.0-1build1). - libxml2-dev is already the newest version (2.9.3+dfsg1-1ubuntu0.1). - - t version (1:2.5+dfsg-5ubuntu10.6). - 0 upgraded, 0 newly installed, 0 to remove and 72 not upgraded. - - ################################################################# - ##### Done ##### - ################################################################# - root@ubuntu:/home/vmware/OVF_converter# - root@ubuntu:/home/vmware/OVF_converter# - + ################################################################# + ##### Installing Require Packages ##### + ################################################################# + Hit:1 http://us.archive.ubuntu.com/ubuntu xenial InRelease + Get:2 http://us.archive.ubuntu.com/ubuntu xenial-updates InRelease [102 kB] + Get:3 http://security.ubuntu.com/ubuntu xenial-security InRelease [102 kB] + Get:4 http://us.archive.ubuntu.com/ubuntu xenial-backports InRelease [102 kB] + Fetched 306 kB in 1s (286 kB/s) + Reading package lists... Done + Reading package lists... Done + Building dependency tree + Reading state information... Done + Note, selecting 'libxslt1-dev' instead of 'libxslt-dev' + libxslt1-dev is already the newest version (1.1.28-2.1). + python-dev is already the newest version (2.7.11-1). + python-lxml is already the newest version (3.5.0-1build1). + libxml2-dev is already the newest version (2.9.3+dfsg1-1ubuntu0.1). + + t version (1:2.5+dfsg-5ubuntu10.6). + 0 upgraded, 0 newly installed, 0 to remove and 72 not upgraded. + + ################################################################# + ##### Done ##### + ################################################################# + root@ubuntu:/home/vmware/OVF_converter# + root@ubuntu:/home/vmware/OVF_converter# + 3. Now tool is installed at /usr/local/bin and available for all users 4. User can access commands of OVF Generator Tool as: ovf_converter path [output location] [ovf name][memory][cpu][disk][ostype][disk_controller] @@ -104,67 +109,69 @@ Follow below setups for installation of OVF Converter Tool. Usage =================== -o Get version of tool as: - - Command - ovf_converter -v - Sample output - - root@ubuntu:/home/vmware/OVF_converter# ovf_converter -v - 1.0 - Initial Realse - root@ubuntu:/home/vmware/OVF_converter# - -o See all command line options of tool as: - Command - ovf_converter -h - Sample output - - root@ubuntu:/home/vmware/OVF_converter# - root@ubuntu:/home/vmware/OVF_converter# ovf_converter -h - Usage: ovf_converter_cli.py [-h] [-v] [-o OUTPUT_LOCATION] [-n OVF_NAME] +o Get version of tool as: + + Command - ovf_converter -v + Sample output - + root@ubuntu:/home/vmware/OVF_converter# ovf_converter -v + 1.0 - Initial Realse + root@ubuntu:/home/vmware/OVF_converter# + +o See all command line options of tool as: + Command - ovf_converter -h + Sample output - + root@ubuntu:/home/vmware/OVF_converter# + root@ubuntu:/home/vmware/OVF_converter# ovf_converter -h + Usage: ovf_converter_cli.py [-h] [-v] [-o OUTPUT_LOCATION] [-n OVF_NAME] [-m MEMORY] [-c CPU] [-d DISK] [-s OSTYPE] - [-dc DISK_CONTROLLER] [--cdrom] + [-dc DISK_CONTROLLER] [--cdrom] [-w 14] path - OVF converter to convert .qcow2 or raw image into OVF - - positional arguments: - path absolute path to source image which will get convert - into ovf - - optional arguments: - -h, --help show this help message and exit - -v, --version shows version of OVF Converter tool - -o OUTPUT_LOCATION, --output_location OUTPUT_LOCATION - location where created OVF will be kept. This location - should have write access. If not given file will get - created at source location (optional) - -n OVF_NAME, --ovf_name OVF_NAME - name of output ovf file. If not given source image - name will be used (optional) - -m MEMORY, --memory MEMORY - required memory for VM in MB (default 1 GB)(optional) - -c CPU, --cpu CPU required number of virtual cpus for VM (default 1 cpu) - (optional) - -d DISK, --disk DISK required size of disk for VM in GB (default as - in source disk img) (optional) - -s OSTYPE, --osType OSTYPE - required operating system type as specified in user - document (default os type other 32 bit) (optional) - -dc DISK_CONTROLLER, --disk_Controller DISK_CONTROLLER - required disk controller type (default controller SCSI - with lsilogicsas) (SATA, IDE, Paravirtual, Buslogic, - lsilogic, lsilogicsas) (optional) - --cdrom whether to include a cd/dvd device (optional) - - -o Create OVF file from qcow2 or raw disk image as: - Command - ovf_converter /home/vmware/centos_ovf/CentOS-7-x86_64-GenericCloud-1503.qcow2 -n centos_qcow2.ovf -m 2048 -c 4 -d 10 -s "Centos 32-bit" -dc "SATA" - Sample output - - root@ubuntu:/home/vmware/OVF_converter# ovf_converter /home/vmware/centos_ovf/CentOS-7-x86_64-GenericCloud-1503.qcow2 -n centos_qcow2.ovf -m 2048 -c 4 -d 10 - #### Start OVF conversion #### - Getting source image information - Converting source disk image to .vmdk - ..................................................... - Creating OVF - Output OVF is at: /home/vmware/centos_ovf/centos_qcow2.ovf - #### Completed OVF conversion #### + OVF converter to convert .qcow2 or raw image into OVF + + positional arguments: + path absolute path to source image which will get convert + into ovf + + optional arguments: + -h, --help show this help message and exit + -v, --version shows version of OVF Converter tool + -o OUTPUT_LOCATION, --output_location OUTPUT_LOCATION + location where created OVF will be kept. This location + should have write access. If not given file will get + created at source location (optional) + -n OVF_NAME, --ovf_name OVF_NAME + name of output ovf file. If not given source image + name will be used (optional) + -m MEMORY, --memory MEMORY + required memory for VM in MB (default 1 GB)(optional) + -c CPU, --cpu CPU required number of virtual cpus for VM (default 1 cpu) + (optional) + -d DISK, --disk DISK required size of disk for VM in GB (default as + in source disk img) (optional) + -s OSTYPE, --osType OSTYPE + required operating system type as specified in user + document (default os type other 32 bit) (optional) + -dc DISK_CONTROLLER, --disk_Controller DISK_CONTROLLER + required disk controller type (default controller SCSI + with lsilogicsas) (SATA, IDE, Paravirtual, Buslogic, + lsilogic, lsilogicsas) (optional) + --cdrom whether to include a cd/dvd device (optional) + -hw VERSION, --hwversion VERSION + VMware ESXi hardware family version (optional) + + +o Create OVF file from qcow2 or raw disk image as: + Command - ovf_converter /home/vmware/centos_ovf/CentOS-7-x86_64-GenericCloud-1503.qcow2 -n centos_qcow2.ovf -m 2048 -c 4 -d 10 -s "Centos 32-bit" -dc "SATA" + Sample output - + root@ubuntu:/home/vmware/OVF_converter# ovf_converter /home/vmware/centos_ovf/CentOS-7-x86_64-GenericCloud-1503.qcow2 -n centos_qcow2.ovf -m 2048 -c 4 -d 10 + #### Start OVF conversion #### + Getting source image information + Converting source disk image to .vmdk + ..................................................... + Creating OVF + Output OVF is at: /home/vmware/centos_ovf/centos_qcow2.ovf + #### Completed OVF conversion #### Trouble shooting @@ -179,4 +186,15 @@ Release History Version 1.0 ------------- -Initial release +Initial release + +Version 1.1 +------------- + +Adds support for cdrom + +Version 1.2 +------------- + +Adds support for hardware version +Improves progress indicator diff --git a/tools/OVF_converter/uploader.py b/tools/OVF_converter/uploader.py new file mode 100644 index 00000000..3702d382 --- /dev/null +++ b/tools/OVF_converter/uploader.py @@ -0,0 +1,220 @@ +# # +# Copyright 2019 VMware Inc. +# This file is part of ETSI OSM +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact: osslegalrouting@vmware.com +# # + +import logging +from lxml import etree +import os +from pyvcloud.vcd.client import BasicLoginCredentials, Client, QueryResultFormat, ResourceType, TaskStatus +from pyvcloud.vcd.exceptions import EntityNotFoundException, InternalServerException +from pyvcloud.vcd.org import Org +import sys +import tarfile +import time + +MODULE_DIR = os.path.dirname(__file__) + +# Set logger +LOG_FILE = os.path.join(MODULE_DIR, "logs/ovf_uploader.log") +os.makedirs(os.path.dirname(LOG_FILE), exist_ok=True) +logger = logging.getLogger(__name__) +file_handler = logging.FileHandler(LOG_FILE) +file_handler.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +file_handler.setFormatter(formatter) +logger.addHandler(file_handler) +stdout_handler = logging.StreamHandler(sys.stdout) +stdout_handler.setLevel(logging.INFO) +logger.addHandler(stdout_handler) +logger.setLevel(10) +logging.captureWarnings(True) + +__version__ = "1.0" +__description__ = "Initial Release" + + +def get_version(): + """ get version of this application""" + version = str(__version__) + " - " + str(__description__) + return version + + +def report_progress(bytes_written, total_size): + percent_complete = int((bytes_written * 100) / total_size) + print("{}% complete \r".format(percent_complete), end='') + + +class OVFUploader(object): + """ Class to convert input image into OVF format """ + + def __init__(self, ovf_file, vcd_url=None, username=None, password=None, orgname=None): + self.ovf_file = os.path.abspath(ovf_file) + self.vcd_url = vcd_url + self.username = username + self.password = password + self.orgname = orgname + try: + client = Client(self.vcd_url, verify_ssl_certs=False) + client.set_highest_supported_version() + client.set_credentials(BasicLoginCredentials(self.username, self.orgname, + self.password)) + logger.info("Logged into {} using version {}".format(self.vcd_url, client.get_api_version())) + self.client = client + self.org = Org(self.client, resource=self.client.get_org()) + + except Exception as exp: + problem = Exception("Failed to connect to vCD at {}, org {}, username {}:\n{}".format( + self.vcd_url, self.orgname, self.username, exp)) + logger.error(problem) + raise problem + + try: + # Retrieve the VM name from the OVF. We will use this as both the image and catalog name + OVF_tree = etree.parse(self.ovf_file) + root = OVF_tree.getroot() + nsmap = {k: v for k, v in root.nsmap.items() if k} + nsmap["xmlns"] = "http://schemas.dmtf.org/ovf/envelope/1" + + virtuasystem = root.find('xmlns:VirtualSystem', nsmap) + name_tag = virtuasystem.find('xmlns:Name', nsmap) + self.image_name = name_tag.text + info_tag = virtuasystem.find('xmlns:Info', nsmap) + self.image_description = info_tag.text + + references = root.find('xmlns:References', nsmap) + file = references.find('xmlns:File', nsmap) + self.vmdk_file = "{}/{}".format( + os.path.dirname(self.ovf_file), + file.attrib['{http://schemas.dmtf.org/ovf/envelope/1}href']) + logger.info("Loaded VM {}: {}".format(self.image_name, self.image_description)) + + except Exception as exp: + problem = Exception("Failed to fetch VirtualSystem Name element from OVF {}:\n{}".format( + self.ovf_file, exp)) + logger.error(problem) + raise problem + + def make_catalog(self): + try: + try: + catalog = self.org.get_catalog(self.image_name) + self.catalog_id = catalog.attrib['id'].split(':')[-1] + except EntityNotFoundException: + logger.info("Creating a new catalog entry {} in vCD".format(self.image_name)) + result = self.org.create_catalog(self.image_name, self.image_description) + if result is None: + raise Exception("Failed to create new catalog entry") + self.catalog_id = result.attrib['id'].split(':')[-1] + self.org.reload() + + logger.debug("Using catalog {}, id {}".format(self.image_name, self.catalog_id)) + + except Exception as exp: + problem = Exception("Failed to fetch catalog for {}:\n{} ".format(self.image_name, exp)) + logger.error(problem) + raise problem + + def upload_ovf(self): + + try: + # Check if the content already exists: + items = self.org.list_catalog_items(self.image_name) + for item in items: + if item['name'] == self.image_name: + logger.info("Removing old version from catalog") + try: + self.org.delete_catalog_item(self.image_name, self.image_name) + except InternalServerException as exp: + problem = Exception( + "Cannot delete vAppTemplate {}. Please check in vCD if " + "the content is still being imported into the catalog".format( + self.image_name)) + raise problem + + # Create a single OVA bundle + ova_tarfilename, _ = os.path.splitext(self.ovf_file) + ova_tarfilename += '.ova' + ova = tarfile.open(name=ova_tarfilename, + mode='w') + ova.add(self.ovf_file, arcname=os.path.basename(self.ovf_file)) + ova.add(self.vmdk_file, arcname=os.path.basename(self.vmdk_file)) + ova.close() + logger.info("Uploading content to vCD") + self.org.upload_ovf(self.image_name, + ova_tarfilename, + item_name=self.image_name, + description=self.image_description, + callback=report_progress) + except Exception as exp: + problem = Exception("Failed to upload OVF {}:\n{} ".format(self.ovf_file, exp)) + logger.error(problem) + raise problem + finally: + if os.path.exists(ova_tarfilename): + os.remove(ova_tarfilename) + + def wait_for_task_completion(self): + + logger.info("Importing content to vCD") + try: + + query = self.client.get_typed_query( + query_type_name=ResourceType.TASK.value, + qfilter='ownerName==' + self.username + ';(status==queued,status==preRunning,status==running)', + query_result_format=QueryResultFormat.REFERENCES) + + upload_task = None + tasks = list(query.execute()) + for task in tasks: + if task.get('name') == 'VDC_UPLOAD_OVF_CONTENTS': + upload_task = self.client.get_resource(task.get('href')) + break + + bad_statuses = [ + TaskStatus.ABORTED, + TaskStatus.CANCELED, + TaskStatus.ERROR + ] + + except Exception as exp: + problem = Exception("Failed to import OVF {}:\n{} ".format(self.ovf_file, exp)) + logger.error(problem) + raise problem + + while(True): + task_status = upload_task.get('status').lower() + if(hasattr(upload_task, 'Progress')): + print("{}% complete \r".format(upload_task.Progress), end='') + + for status in bad_statuses: + if task_status == status.value.lower(): + problem = Exception( + "vCD failed to import OVF {}:\n{}: {} ".format(self.ovf_file, + task_status, + upload_task.Error.get('Message'))) + logger.error(problem) + raise problem + if task_status == str(TaskStatus.SUCCESS.value).lower(): + break + + time.sleep(2) + upload_task = self.client.get_resource(upload_task.get('href')) + + logger.info("OVF upload and import complete, content is ready to use") diff --git a/tools/vmware_ovf_upload.py b/tools/vmware_ovf_upload.py index 4f3f0e7a..834068de 100755 --- a/tools/vmware_ovf_upload.py +++ b/tools/vmware_ovf_upload.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -## +# # # Copyright 2016-2017 VMware Inc. # This file is part of ETSI OSM # All Rights Reserved. @@ -19,26 +19,25 @@ # # For those usages not covered by the Apache License, Version 2.0 please # contact: osslegalrouting@vmware.com -## +# # - -from xml.etree import ElementTree as XmlElementTree -from pyvcloud.vcd.client import BasicLoginCredentials,Client -from pyvcloud.vcd.vdc import VDC -from pyvcloud.vcd.org import Org -import sys,os import logging +import os +from progressbar import Percentage, Bar, ETA, FileTransferSpeed, ProgressBar +from pyvcloud.vcd.client import BasicLoginCredentials, Client +from pyvcloud.vcd.org import Org +import re import requests +import sys import time -import re -import hashlib -from progressbar import Percentage, Bar, ETA, FileTransferSpeed, ProgressBar - +from xml.etree import ElementTree as XmlElementTree API_VERSION = '5.6' + class vCloudconfig(object): - def __init__(self, host=None, user=None, password=None,orgname=None, logger=None): + + def __init__(self, host=None, user=None, password=None, orgname=None, logger=None): self.url = host self.user = user self.password = password @@ -55,12 +54,12 @@ class vCloudconfig(object): try: self.logger.debug("Logging in to a vcd {} as user {}".format(self.org, - self.user)) + self.user)) client = Client(self.url, verify_ssl_certs=False) client.set_credentials(BasicLoginCredentials(self.user, self.org, self.password)) - except: + except Exception: raise Exception("Can't connect to a vCloud director org: " - "{} as user: {}".format(self.org, self.user)) + "{} as user: {}".format(self.org, self.user)) return client @@ -84,19 +83,19 @@ class vCloudconfig(object): self.logger.debug("get_catalog_id_from_path() client requesting {} ".format(path)) - dirpath, filename = os.path.split(path) - flname, file_extension = os.path.splitext(path) + _, filename = os.path.split(path) + _, file_extension = os.path.splitext(path) if file_extension != '.ovf': self.logger.debug("Wrong file extension {} connector support only OVF container.".format(file_extension)) raise Exception("Wrong container. vCloud director supports only OVF.") self.logger.debug("File name {} Catalog Name {} file path {} ".format(filename, - catalog_name, - path)) + catalog_name, + path)) try: client = self.connect() if not client: - raise Exception("Failed to connect vCD") + raise Exception("Failed to connect vCD") org = Org(client, resource=client.get_org()) catalogs = org.list_catalogs() except Exception as exp: @@ -109,7 +108,7 @@ class vCloudconfig(object): if result is None: raise Exception("Failed to create new catalog {} ".format(catalog_name)) result = self.upload_ovf(org=org, catalog_name=catalog_name, image_name=filename.split(".")[0], - media_file_name=path, description='medial_file_name', progress=progress) + media_file_name=path, description='medial_file_name', progress=progress) if not result: raise Exception("Failed to create vApp template for catalog {} ".format(catalog_name)) return self.get_catalogid(catalog_name, catalogs) @@ -118,18 +117,18 @@ class vCloudconfig(object): # search for existing catalog if we find same name we return ID if catalog['name'] == catalog_name: self.logger.debug("Found existing catalog entry for {} " - "catalog id {}".format(catalog_name, - self.get_catalogid(catalog_name, catalogs))) + "catalog id {}".format(catalog_name, + self.get_catalogid(catalog_name, catalogs))) return self.get_catalogid(catalog_name, catalogs) # if we didn't find existing catalog we create a new one and upload image. self.logger.debug("Creating new catalog entry {} - {}".format(catalog_name, catalog_name)) - result = org.create_catalog(catalog_name, catalog_name) + result = org.create_catalog(catalog_name, catalog_name) if result is None: raise Exception("Failed to create new catalog {} ".format(catalog_name)) result = self.upload_ovf(org=org, catalog_name=catalog_name, image_name=filename.split(".")[0], - media_file_name=path, description='medial_file_name', progress=progress) + media_file_name=path, description='medial_file_name', progress=progress) if not result: raise Exception("Failed create vApp template for catalog {} ".format(catalog_name)) @@ -176,21 +175,23 @@ class vCloudconfig(object): continue catalog_href = "{}/api/catalog/{}/action/upload".format(self.url, catalog['id']) data = """ - {} vApp Template + + {} vApp Template """.format(catalog_name, description) if client: - headers = {'Accept':'application/*+xml;version=' + API_VERSION, - 'x-vcloud-authorization': client._session.headers['x-vcloud-authorization']} + headers = {'Accept': 'application/*+xml;version=' + API_VERSION, + 'x-vcloud-authorization': client._session.headers['x-vcloud-authorization']} headers['Content-Type'] = 'application/vnd.vmware.vcloud.uploadVAppTemplateParams+xml' response = requests.post(url=catalog_href, - headers=headers, - data=data, - verify=False) + headers=headers, + data=data, + verify=False) if response.status_code != 201: self.logger.debug("Failed to create vApp template") - raise Exception("Failed to create vApp template") + raise Exception("Failed to create vApp template") if response.status_code == requests.codes.created: catalogItem = XmlElementTree.fromstring(response.content) @@ -200,45 +201,48 @@ class vCloudconfig(object): template = href response = requests.get(url=href, - headers=headers, - verify=False) + headers=headers, + verify=False) if response.status_code == requests.codes.ok: headers['Content-Type'] = 'Content-Type text/xml' - result = re.search('rel="upload:default"\shref="(.*?\/descriptor.ovf)"',response.content) + result = re.search('rel="upload:default"\shref="(.*?\/descriptor.ovf)"', response.content) if result: transfer_href = result.group(1) response = requests.put(url=transfer_href, headers=headers, - data=open(media_file_name, 'rb'), - verify=False) + data=open(media_file_name, 'rb'), + verify=False) if response.status_code != requests.codes.ok: self.logger.debug( - "Failed create vApp template for catalog name {} and image {}".format(catalog_name, - media_file_name)) + "Failed create vApp template for catalog name {} and image {}".format( + catalog_name, + media_file_name)) return False # TODO fix this with aync block time.sleep(5) - self.logger.debug("vApp template for catalog name {} and image {}".format(catalog_name, media_file_name)) + self.logger.debug("vApp template for catalog name {} and image {}".format( + catalog_name, + media_file_name)) # uploading VMDK file # check status of OVF upload and upload remaining files. response = requests.get(url=template, - headers=headers, + headers=headers, verify=False) if response.status_code == requests.codes.ok: - result = re.search('rel="upload:default"\s*href="(.*?vmdk)"',response.content) + result = re.search('rel="upload:default"\s*href="(.*?vmdk)"', response.content) if result: link_href = result.group(1) # we skip ovf since it already uploaded. if 'ovf' in link_href: continue # The OVF file and VMDK must be in a same directory - head, tail = os.path.split(media_file_name) + head, _ = os.path.split(media_file_name) file_vmdk = head + '/' + link_href.split("/")[-1] if not os.path.isfile(file_vmdk): return False @@ -248,7 +252,7 @@ class vCloudconfig(object): hrefvmdk = link_href if progress: widgets = ['Uploading file: ', Percentage(), ' ', Bar(), ' ', ETA(), ' ', - FileTransferSpeed()] + FileTransferSpeed()] progress_bar = ProgressBar(widgets=widgets, maxval=statinfo.st_size).start() bytes_transferred = 0 @@ -260,9 +264,9 @@ class vCloudconfig(object): bytes_transferred, len(my_bytes) - 1, statinfo.st_size) headers['Content-Length'] = str(len(my_bytes)) response = requests.put(url=hrefvmdk, - headers=headers, - data=my_bytes, - verify=False) + headers=headers, + data=my_bytes, + verify=False) if response.status_code == requests.codes.ok: bytes_transferred += len(my_bytes) if progress: @@ -270,7 +274,7 @@ class vCloudconfig(object): else: self.logger.debug( 'file upload failed with error: [%s] %s' % (response.status_code, - response.content)) + response.content)) f.close() return False @@ -282,18 +286,21 @@ class vCloudconfig(object): else: self.logger.debug("Failed retrieve vApp template for catalog name {} for OVF {}". format(catalog_name, media_file_name)) - return False + return False except Exception as exp: self.logger.debug("Failed while uploading OVF to catalog {} for OVF file {} with Exception {}" - .format(catalog_name,media_file_name, exp)) + .format(catalog_name, media_file_name, exp)) raise Exception( "Failed while uploading OVF to catalog {} for OVF file {} with Exception {}" - .format(catalog_name,media_file_name, exp)) + .format(catalog_name, media_file_name, exp)) self.logger.debug("Failed to retrieve catalog name {} for OVF file {}".format(catalog_name, media_file_name)) - return False + return False + if __name__ == "__main__": + print("This file is deprecated. Please use ovf_uplader_cli instead.") + # vmware vcloud director credentials vcd_hostname = sys.argv[1] vcd_username = sys.argv[2] @@ -302,17 +309,7 @@ if __name__ == "__main__": # OVF image path to be upload to vCD ovf_file_path = sys.argv[5] - # changing virtual system type in ovf file - fh = open(ovf_file_path,'r') - content = fh.read() - content = content.replace('vmx-7','vmx-07') - fh.close() - fh1 = open(ovf_file_path,'w') - fh1.write(content) - fh1.close() - - - logging.basicConfig(filename='ovf_upload.log',level=logging.DEBUG) + logging.basicConfig(filename='ovf_upload.log', level=logging.DEBUG) logger = logging.getLogger(__name__) obj = vCloudconfig(vcd_hostname, vcd_username, vcd_password, orgname, logger) @@ -322,10 +319,11 @@ if __name__ == "__main__": # Get image name from cirros vnfd cirros_yaml = '../descriptor-packages/vnfd/cirros_vnf/src/cirros_vnfd.yaml' - rh = open(cirros_yaml,'r') - match = re.search("image:\s'(.*?)'\n",rh.read()) - if match: catalog = match.group(1) + rh = open(cirros_yaml, 'r') + match = re.search("image:\s'(.*?)'\n", rh.read()) + if match: + catalog = match.group(1) if file_extension == '.ovf': obj.get_catalog_id_from_path(catalog_name=catalog, path=ovf_file_path, - progress=True) + progress=True)