tools/OVF_converter/converter.py

   1 #!/usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3
   4 # #
   5 # Copyright 2016-2017 VMware Inc.
   6 # This file is part of ETSI OSM
   7 # All Rights Reserved.
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  10 # not use this file except in compliance with the License. You may obtain
  11 # a copy of the License at
  12 #
  13 #         http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  18 # License for the specific language governing permissions and limitations
  19 # under the License.
  20 #
  21 # For those usages not covered by the Apache License, Version 2.0 please
  22 # contact:  osslegalrouting@vmware.com
  23 # #
  24
  25 import logging
  26 import os
  27 import subprocess
  28 import yaml
  29 from lxml import etree as ET
  30
  31 # file paths
  32 MODULE_DIR = os.path.dirname(__file__)
  33 OVF_TEMPLATE_PATH = os.path.join(MODULE_DIR,
  34                                  "ovf_template/template.xml")
  35 IDE_CDROM_XML_PATH = os.path.join(MODULE_DIR,
  36                                   "ovf_template/ide_cdrom.xml")
  37 OS_INFO_FILE_PATH = os.path.join(MODULE_DIR,
  38                                  "config/os_type.yaml")
  39 DISK_CONTROLLER_INFO_FILE_PATH = os.path.join(MODULE_DIR,
  40                                               "config/disk_controller.yaml")
  41
  42 # Set logger
  43 LOG_FILE = os.path.join(MODULE_DIR, "logs/ovf_converter.log")
  44 os.makedirs(os.path.dirname(LOG_FILE), exist_ok=True)
  45 logger = logging.getLogger(__name__)
  46 hdlr = logging.FileHandler(LOG_FILE)
  47 formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
  48 hdlr.setFormatter(formatter)
  49 logger.addHandler(hdlr)
  50 logger.setLevel(10)
  51
  52 __version__ = "1.2"
  53 __description__ = "OVF Hardware Version 14 compatible"
  54
  55
  56 def get_version(*args, **kwargs):
  57     """ get version of this application"""
  58     version = str(__version__) + " - " + str(__description__)
  59     return version
  60
  61
  62 # converter class
  63 class OVFConverter(object):
  64     """ Class to convert input image into OVF format """
  65
  66     def __init__(self, source_img_path, output_location=None, output_ovf_name=None,
  67                  memory=None, cpu=None, disk=None, os_type=None,
  68                  disk_controller=None, cdrom=None, hwversion=14):
  69         """
  70             Constructor to initialize object of class OVFConverter
  71             Args:
  72             source_img_path - absolute path to source image which will get convert into ovf
  73             output_location - location where created OVF will be kept. This location
  74                               should have write access. If not given file will get
  75                               created at source location  (optional)
  76             output_ovf_name - name of output ovf.If not given source image name will
  77                               be used (optional)
  78             memory -          required memory for VM in MB (optional)
  79             cpu -             required number of virtual cpus for VM (optional)
  80             disk -            required size of disk for VM in GB (optional)
  81             os_type-          required operating system type as specified in user document
  82                               (default os type other 32 bit) (optional)
  83             disk_controller - required disk controller type
  84                               (default controller SCSI with lsilogicsas)
  85                               (SATA, IDE, Paravirtual, Buslogic, Lsilogic, Lsilogicsas) (optional)
  86             hwversion -       VMware ESXi hardware family version (optional)
  87
  88             Returns:
  89                 Nothing.
  90         """
  91         self.logger = logger
  92         self.ovf_template_path = OVF_TEMPLATE_PATH
  93
  94         self.source_img_path = source_img_path
  95         self.source_img_filename, file_extension = os.path.splitext(os.path.basename(self.source_img_path))
  96         self.source_img_location = os.path.dirname(self.source_img_path)
  97         self.source_format = file_extension[1:]
  98
  99         self.output_format = "ovf"
 100         self.output_ovf_name = output_ovf_name.split('.')[0] if output_ovf_name else self.source_img_filename
 101         self.output_location = output_location if output_location else "."
 102         self.output_ovf_name_ext = self.output_ovf_name + "." + self.output_format
 103         self.output_path = os.path.join(self.output_location, self.output_ovf_name_ext)
 104
 105         self.output_diskimage_format = "vmdk"
 106         self.output_diskimage_name = self.source_img_filename + "." + self.output_diskimage_format
 107         self.output_diskimage_path = os.path.join(self.output_location, self.output_diskimage_name)
 108
 109         self.logger.info("Input parameters to Converter: \n ovf_template_path = {}, \n source_img_path = {}, \n"
 110                          "source_img_location ={} , \n source_format = {}, \n source_img_filename = {}".format(
 111                              self.ovf_template_path,
 112                              self.source_img_path, self.source_img_location,
 113                              self.source_format, self.source_img_filename))
 114
 115         self.logger.info("Output parameters to Converter: \n output_format = {}, \n output_ovf_name = {}, \n"
 116                          "output_location ={} , \n output_path = {}, \n output_diskimage_name = {} , \n"
 117                          " output_diskimage_path = {} ".format(self.output_format, self.output_ovf_name,
 118                                                                self.output_location, self.output_path,
 119                                                                self.output_diskimage_name, self.output_diskimage_path))
 120
 121         self.disk_capacity = 1
 122         self.disk_populated_size = 0
 123
 124         self.vm_name = self.output_ovf_name
 125         self.memory = str(memory) if memory is not None else None
 126         self.cpu = str(cpu) if cpu is not None else None
 127         self.os_type = str(os_type).strip() if os_type else None
 128         self.cdrom = cdrom
 129         self.hwversion = hwversion
 130
 131         if self.os_type:
 132             self.osID, self.osType = self.__get_osType()
 133             if self.osID is None or self.osType is None:
 134                 error_msg = "ERROR: Invalid input can not find OS type {} ".format(self.os_type)
 135                 self.__raise_exception(error_msg)
 136
 137         self.disk_controller = str(disk_controller).strip() if disk_controller else None
 138
 139         if self.disk_controller:
 140             self.disk_controller_info = self.__get_diskcontroller()
 141
 142             if not self.disk_controller_info:
 143                 error_msg = "ERROR: Invalid input can not find Disk Controller {} ".format(self.disk_controller)
 144                 self.__raise_exception(error_msg)
 145
 146         if disk is not None:
 147             # convert disk size from GB to bytes
 148             self.disk_size = int(disk) * 1024 * 1024 * 1024
 149         else:
 150             self.disk_size = None
 151
 152         self.logger.info("Other input parameters to Converter: \n vm_name = {}, \n memory = {}, \n"
 153                          "disk_size ={} \n os type = {} \n disk controller = {}".format(
 154                              self.vm_name, self.memory, self.disk_size, self.os_type, self.disk_controller))
 155
 156         # check access for read input location and write output location return none if no access
 157         if not os.access(self.source_img_path, os.F_OK):
 158             error_msg = "ERROR: Source image file {} not present".format(self.source_img_path)
 159             self.__raise_exception(error_msg, exception_type="IO")
 160
 161         elif not os.access(self.source_img_path, os.R_OK):
 162             error_msg = "ERROR: Cannot read source image file {}".format(self.source_img_path)
 163             self.__raise_exception(error_msg, exception_type="IO")
 164
 165         if not os.access(self.output_location, os.W_OK):
 166             error_msg = "ERROR: No write access to location {} to write output OVF ".format(self.output_location)
 167             self.__raise_exception(error_msg, exception_type="IO")
 168
 169     def __get_image_info(self):
 170         """
 171             Private method to get information about source imager.
 172             Args  : None
 173             Return : True on success else False
 174         """
 175         try:
 176             print("Getting source image information")
 177             command = "qemu-img info \t " + self.source_img_path
 178             output, error, returncode = self.__execute_command(command)
 179
 180             if error or returncode:
 181                 self.logger.error("ERROR: Error occurred while getting information about source image : {} \n "
 182                                   "return code : {} ".format(error, returncode))
 183                 return False
 184
 185             elif output:
 186                 self.logger.info("Get Image Info Output : {} \n ".format(output))
 187                 split_output = output.decode().split("\n")
 188                 for line in split_output:
 189                     line = line.strip()
 190                     if "virtual size" in line:
 191                         virtual_size_info = line.split(":")[1].split()
 192                         if len(virtual_size_info) == 3 and virtual_size_info[2].strip(")") == "bytes":
 193                             self.disk_capacity = int(virtual_size_info[1].strip("("))
 194                         else:
 195                             self.disk_capacity = self.__convert_size(virtual_size_info[0])
 196
 197                     elif "disk size" in line:
 198                         size = line.split(":")[1].split()[0]
 199                         self.disk_populated_size = self.__convert_size(size)
 200                     elif "file format" in line:
 201                         self.source_format = line.split(":")[1]
 202
 203                 self.logger.info("Updated source image virtual disk capacity : {} ,"
 204                                  "Updated source image populated size: {}".format(self.disk_capacity,
 205                                                                                   self.disk_populated_size))
 206                 return True
 207         except Exception as exp:
 208             error_msg = "ERROR: Error occurred while getting information about source image : {}".format(exp)
 209             self.logger.error(error_msg)
 210             print(error_msg)
 211             return False
 212
 213     def __convert_image(self):
 214         """
 215             Private method to convert source disk image into .vmdk disk image.
 216             Args  : None
 217             Return : True on success else False
 218         """
 219
 220         print("Converting source disk image to .vmdk ")
 221
 222         command = "qemu-img convert -p -f " + self.source_format + " -O " + self.output_diskimage_format + \
 223             " -o subformat=streamOptimized " + self.source_img_path + " " + self.output_diskimage_path
 224
 225         _, error, returncode = self.__execute_command(command, show_output=True)
 226
 227         if error or returncode:
 228             error_msg = "ERROR: Error occurred while converting source disk image into vmdk: {}\n" + \
 229                 "return code : {} ".format(error, returncode)
 230             self.logger.error(error_msg)
 231             print(error_msg)
 232             return False
 233         else:
 234             if os.path.isfile(self.output_diskimage_path):
 235                 self.logger.info("Successfully converted source image {} into {} \n "
 236                                  "return code : {} ".format(self.source_img_path,
 237                                                             self.output_diskimage_path,
 238                                                             returncode))
 239                 result = self.__make_image_bootable()
 240                 if result:
 241                     self.logger.info("Made {} bootable".format(self.output_diskimage_path))
 242                     print("Output VMDK is at: {}".format(self.output_diskimage_path))
 243                     return True
 244                 else:
 245                     self.logger.error("Cannot make {} bootable".format(self.output_diskimage_path))
 246                     print("ERROR: Fail to convert source image into .vmdk")
 247                     return False
 248             else:
 249                 self.logger.error("Converted vmdk disk file {} is not present \n ".format(
 250                     self.output_diskimage_path))
 251                 print("Fail to convert source image into .vmdk")
 252                 return False
 253
 254     def __make_image_bootable(self):
 255         """
 256             Private method to make source disk image bootable.
 257             Args  : None
 258             Return : True on success else False
 259         """
 260         command = "printf '\x03' | dd conv=notrunc of=" + self.output_diskimage_path + "\t bs=1 seek=$((0x4))"
 261         output, error, returncode = self.__execute_command(command)
 262
 263         if error and returncode:
 264             error_msg = "ERROR:Error occurred while making source disk image bootable : {} \n "\
 265                 "return code : {} ".format(error, returncode)
 266             self.logger.error(error_msg)
 267             print(error_msg)
 268             return False
 269         else:
 270             self.logger.info("Make Image Bootable Output : {} ".format(output))
 271             return True
 272
 273     def __edit_ovf_template(self):
 274         """
 275             Private method to create new OVF file by editing OVF template
 276             Args  : None
 277             Return : True on success else False
 278         """
 279         try:
 280             print("Creating OVF")
 281             # Read OVF template file
 282             OVF_tree = ET.parse(self.ovf_template_path)
 283             root = OVF_tree.getroot()
 284
 285             # Collect namespaces
 286             nsmap = {k: v for k, v in root.nsmap.items() if k}
 287             nsmap["xmlns"] = "http://schemas.dmtf.org/ovf/envelope/1"
 288
 289             # Edit OVF template
 290             references = root.find('xmlns:References', nsmap)
 291             if references is not None:
 292                 file_tag = references.find('xmlns:File', nsmap)
 293                 if file_tag is not None:
 294                     file_tag.attrib['{' + nsmap['ovf'] + '}href'] = self.output_diskimage_name
 295
 296             disksection = root.find('xmlns:DiskSection', nsmap)
 297             if disksection is not None:
 298                 diak_tag = disksection.find('xmlns:Disk', nsmap)
 299                 if diak_tag is not None:
 300                     if self.disk_size and self.disk_size > self.disk_capacity:
 301                         self.disk_capacity = self.disk_size
 302
 303                     diak_tag.attrib['{' + nsmap['ovf'] + '}capacity'] = str(self.disk_capacity)
 304                     diak_tag.attrib['{' + nsmap['ovf'] + '}populatedSize'] = str(self.disk_populated_size)
 305
 306             virtuasystem = root.find('xmlns:VirtualSystem', nsmap)
 307             if virtuasystem is not None:
 308                 name_tag = virtuasystem.find('xmlns:Name', nsmap)
 309                 if name_tag is not None:
 310                     name_tag.text = self.vm_name
 311
 312                 if self.os_type is not None:
 313                     operatingSystemSection = virtuasystem.find('xmlns:OperatingSystemSection', nsmap)
 314                     if self.osID and self.osType:
 315                         operatingSystemSection.attrib['{' + nsmap['ovf'] + '}id'] = self.osID
 316                         os_discription_tag = operatingSystemSection.find('xmlns:Description', nsmap)
 317                         os_discription_tag.text = self.osType
 318
 319                 virtualHardwareSection = virtuasystem.find('xmlns:VirtualHardwareSection', nsmap)
 320                 system = virtualHardwareSection.find('xmlns:System', nsmap)
 321                 virtualSystemIdentifier = system.find('vssd:VirtualSystemIdentifier', nsmap)
 322                 if virtualSystemIdentifier is not None:
 323                     virtualSystemIdentifier.text = self.vm_name
 324                 VirtualSystemType = system.find('vssd:VirtualSystemType', nsmap)
 325                 if VirtualSystemType is not None:
 326                     VirtualSystemType.text = "vmx-{}".format(self.hwversion)
 327
 328                 if self.memory is not None or self.cpu is not None or self.disk_controller is not None:
 329                     for item in virtualHardwareSection.iterfind('xmlns:Item', nsmap):
 330                         description = item.find("rasd:Description", nsmap)
 331
 332                         if self.cpu is not None:
 333                             if description is not None and description.text == "Number of Virtual CPUs":
 334                                 cpu_item = item.find("rasd:VirtualQuantity", nsmap)
 335                                 name_item = item.find("rasd:ElementName", nsmap)
 336                                 if cpu_item is not None:
 337                                     cpu_item.text = self.cpu
 338                                     name_item.text = self.cpu + " virtual CPU(s)"
 339
 340                         if self.memory is not None:
 341                             if description is not None and description.text == "Memory Size":
 342                                 mem_item = item.find("rasd:VirtualQuantity", nsmap)
 343                                 name_item = item.find("rasd:ElementName", nsmap)
 344                                 if mem_item is not None:
 345                                     mem_item.text = self.memory
 346                                     name_item.text = self.memory + " MB of memory"
 347
 348                         if self.disk_controller is not None:
 349                             if description is not None and description.text == "SCSI Controller":
 350                                 if self.disk_controller_info is not None:
 351                                     name_item = item.find("rasd:ElementName", nsmap)
 352                                     name_item.text = str(self.disk_controller_info["controllerName"]) + "0"
 353
 354                                     resource_type = item.find("rasd:ResourceType", nsmap)
 355                                     resource_type.text = self.disk_controller_info["resourceType"]
 356
 357                                     description.text = self.disk_controller_info["controllerName"]
 358                                     resource_subtype = item.find("rasd:ResourceSubType", nsmap)
 359                                     if self.disk_controller_info["controllerName"] == "IDE Controller":
 360                                         # Remove resource subtype item
 361                                         resource_subtype.getparent().remove(resource_subtype)
 362                                     if "resourceSubType" in self.disk_controller_info:
 363                                         resource_subtype.text = self.disk_controller_info["resourceSubType"]
 364                 if self.cdrom:
 365                     last_item = list(virtualHardwareSection.iterfind('xmlns:Item', nsmap))[-1]
 366                     ide_cdrom_items_etree = ET.parse(IDE_CDROM_XML_PATH)
 367                     ide_cdrom_items = list(ide_cdrom_items_etree.iterfind('Item'))
 368                     for item in ide_cdrom_items:
 369                         last_item.addnext(item)
 370
 371             # Save output OVF
 372             OVF_tree.write(self.output_path, xml_declaration=True, encoding='utf-8',
 373                            method="xml")
 374
 375             if os.path.isfile(self.output_path):
 376                 logger.info("Successfully written output OVF at {}".format(self.output_path))
 377                 print("Output OVF is at: {}".format(self.output_path))
 378                 return self.output_path
 379             else:
 380                 error_msg = "ERROR: Error occurred while creating OVF file"
 381                 print(error_msg)
 382                 return False
 383
 384         except Exception as exp:
 385             error_msg = "ERROR: Error occurred while editing OVF template : {}".format(exp)
 386             self.logger.error(error_msg)
 387             print(error_msg)
 388             return False
 389
 390     def __convert_size(self, size):
 391         """
 392             Private method to convert disk size from GB,MB to bytes.
 393             Args :
 394                 size  : disk size with prefix 'G' for GB and 'M' for MB
 395             Return :  disk size in bytes
 396         """
 397         byte_size = 0
 398         try:
 399             if not size:
 400                 self.logger.error("No size {} to convert in bytes".format(size))
 401             else:
 402                 size = str(size)
 403                 disk_size = float(size[:-1])
 404                 input_type = size[-1].strip()
 405
 406                 self.logger.info("Disk size : {} , size type : {} ".format(disk_size, input_type))
 407
 408                 if input_type == "G":
 409                     byte_size = disk_size * 1024 * 1024 * 1024
 410                 elif input_type == "M":
 411                     byte_size = disk_size * 1024 * 1024
 412
 413                 self.logger.info("Disk size in bytes: {} ".format(byte_size))
 414
 415             return int(byte_size)
 416
 417         except Exception as exp:
 418             error_msg = "ERROR:Error occurred while converting disk size in bytes : {}".format(exp)
 419             self.logger.error(error_msg)
 420             print(error_msg)
 421             return False
 422
 423     def __get_osType(self):
 424         """
 425             Private method to get OS ID and Type
 426             Args :
 427                 None
 428             Return :
 429                 osID : OS ID
 430                 osType: OS Type
 431         """
 432         osID = None
 433         osType = None
 434         os_info = self.__read_yaml_file(OS_INFO_FILE_PATH)
 435
 436         try:
 437             if self.os_type and os_info:
 438                 for os_id, os_type in os_info.items():
 439                     if self.os_type.lower() == os_type.lower():
 440                         osID = os_id
 441                         osType = os_type
 442                         break
 443         except Exception as exp:
 444             error_msg = "ERROR:Error occurred while getting OS details : {}".format(exp)
 445             self.logger.error(error_msg)
 446             print(error_msg)
 447
 448         return osID, osType
 449
 450     def __get_diskcontroller(self):
 451         """
 452             Private method to get details of Disk Controller
 453             Args :
 454                 None
 455             Return :
 456                 disk_controller : dict with details of Disk Controller
 457         """
 458         disk_controller = {}
 459         scsi_subtype = None
 460         if self.disk_controller.lower() in ["paravirtual", "lsilogic", "buslogic", "lsilogicsas"]:
 461             scsi_subtype = self.disk_controller
 462             self.disk_controller = "SCSI"
 463
 464         disk_controller_info = self.__read_yaml_file(DISK_CONTROLLER_INFO_FILE_PATH)
 465         try:
 466             if self.disk_controller and disk_controller_info:
 467                 for key, value in disk_controller_info.iteritems():
 468                     if self.disk_controller.lower() in key.lower():
 469                         disk_controller['controllerName'] = key
 470                         disk_controller['resourceType'] = str(value["ResourceType"])
 471                         resourceSubTypes = value["ResourceSubTypes"] if "ResourceSubTypes" in value else None
 472                         if key == "SATA Controller":
 473                             disk_controller["resourceSubType"] = resourceSubTypes[0]
 474                         elif key == "SCSI Controller":
 475                             if scsi_subtype:
 476                                 if scsi_subtype.lower() == "paravirtual":
 477                                     scsi_subtype = "VirtualSCSI"
 478                                 for subtype in resourceSubTypes:
 479                                     if scsi_subtype.lower() == subtype.lower():
 480                                         disk_controller["resourceSubType"] = subtype
 481                                         break
 482                                 else:
 483                                     error_msg = "ERROR: Invalid inputs can not "\
 484                                         "find SCSI subtype {}".format(scsi_subtype)
 485                                     self.__raise_exception(error_msg)
 486
 487         except KeyError as exp:
 488             error_msg = "ERROR:Error occurred while getting Disk Controller details : {}".format(exp)
 489             self.logger.error(error_msg)
 490             print(error_msg)
 491
 492         return disk_controller
 493
 494     def __read_yaml_file(self, file_path):
 495         """
 496             Private method to execute command
 497             Args :
 498                 command  : command to execute
 499             Return :
 500                 Dict of yaml data
 501         """
 502         with open(file_path) as data_file:
 503             data = yaml.load(data_file, Loader=yaml.SafeLoader)
 504         return data
 505
 506     def __raise_exception(self, error_msg, exception_type="Generic"):
 507         """
 508             Private method to execute command
 509             Args :
 510                 command  : command to execute
 511             Return :
 512                None
 513         """
 514         if error_msg:
 515             self.logger.debug(error_msg)
 516             print(error_msg)
 517             if exception_type == "Generic":
 518                 raise Exception(error_msg)
 519             elif exception_type == "IO":
 520                 raise Exception(error_msg)
 521
 522     def __execute_command(self, command, show_output=False):
 523         """
 524             Private method to execute command
 525             Args :
 526                 command  : command to execute
 527             Return :
 528                 stdout : output of command
 529                 stderr: error occurred while executing command if any
 530                 returncode : return code of command execution
 531         """
 532         try:
 533             self.logger.info("Execute command: {} ".format(command))
 534
 535             proc = subprocess.Popen(command, stdout=subprocess.PIPE, stdin=subprocess.PIPE,
 536                                     stderr=subprocess.PIPE, shell=True, bufsize=1)
 537
 538             stdout = b''
 539             stderr = b''
 540
 541             while True:
 542                 output = proc.stdout.read(1)
 543                 stdout += output
 544                 if show_output:
 545                     print(output.decode(), end='')
 546                 returncode = proc.poll()
 547                 if returncode is not None:
 548                     for output in proc.stdout.readlines():
 549                         stdout += output
 550                         if show_output:
 551                             print(output.decode(), end='')
 552                     break
 553
 554             for output in proc.stderr.readlines():
 555                 stderr += output
 556
 557         except Exception as exp:
 558             self.logger.error("Error {} occurred while executing command {} ".format(exp, command))
 559
 560         return stdout, stderr, returncode
 561
 562     def create_ovf(self):
 563         """
 564             Method to convert source image into OVF
 565             Args : None
 566             Return : True on success else False
 567         """
 568         # check output format
 569         if self.source_format == self.output_format:
 570             self.logger.info("Source format is OVF. No need to convert: {} ")
 571             return self.source_img_path
 572
 573         # Get source img properties
 574         img_info = self.__get_image_info()
 575         if img_info:
 576
 577             # Create vmdk disk image
 578             disk_img = self.__convert_image()
 579             if disk_img:
 580
 581                 # Edit OVF tempalte
 582                 ovf_path = self.__edit_ovf_template()
 583                 return ovf_path
 584         else:
 585             self.logger.error("Error in getting image information cannot convert image")
 586             raise Exception("Error in getting image information cannot convert image")
 587             return False