tools/OVF_converter/format_converter/converter.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2016-2017 VMware Inc.
   6 # This file is part of ETSI OSM
   7 # All Rights Reserved.
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  10 # not use this file except in compliance with the License. You may obtain
  11 # a copy of the License at
  12 #
  13 #         http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  18 # License for the specific language governing permissions and limitations
  19 # under the License.
  20 #
  21 # For those usages not covered by the Apache License, Version 2.0 please
  22 # contact:  osslegalrouting@vmware.com
  23 ##
  24
  25 import logging
  26 import os
  27 import subprocess
  28 import yaml
  29 from lxml import etree as ET
  30 from command_progress import CommandProgressbar
  31
  32 #file paths
  33 MODULE_DIR = os.path.dirname(__file__)
  34 OVF_TEMPLATE_PATH = os.path.join(os.path.dirname(MODULE_DIR),
  35                                 "ovf_template/template.xml")
  36 IDE_CDROM_XML_PATH = os.path.join(os.path.dirname(MODULE_DIR),
  37                                 "ovf_template/ide_cdrom.xml")
  38 OS_INFO_FILE_PATH = os.path.join(os.path.dirname(MODULE_DIR),
  39                                 "config/os_type.yaml")
  40 DISK_CONTROLLER_INFO_FILE_PATH = os.path.join(os.path.dirname(MODULE_DIR),
  41                                               "config/disk_controller.yaml")
  42
  43
  44 #Set logger
  45 LOG_FILE = os.path.join(os.path.dirname(MODULE_DIR),"logs/ovf_converter.log")
  46 logger = logging.getLogger(__name__)
  47 hdlr = logging.FileHandler(LOG_FILE)
  48 formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
  49 hdlr.setFormatter(formatter)
  50 logger.addHandler(hdlr)
  51 logger.setLevel(10)
  52
  53 __version__ = "1.0"
  54 __description__ = "initial Release"
  55
  56 def get_version(*args, **kwargs):
  57     """ get version of this application"""
  58     version = str(__version__ ) +" - "+ str( __description__ )
  59     return version
  60
  61 #converter class
  62 class OVFConverter(object):
  63     """ Class to convert input image into OVF format """
  64
  65     def __init__(self, source_img_path, output_location=None, output_ovf_name=None,
  66                     memory=None, cpu=None, disk=None, os_type=None,
  67                     disk_controller=None, cdrom=None,
  68                     options={'subformat':'streamOptimized'}):
  69         """
  70             Constructor to initialize object of class OVFConverter
  71             Args:
  72             source_img_path - absolute path to source image which will get convert into ovf
  73             output_location - location where created OVF will be kept. This location
  74                               should have write access. If not given file will get
  75                               created at source location  (optional)
  76             output_ovf_name - name of output ovf.If not given source image name will
  77                               be used (optional)
  78             memory -          required memory for VM in MB (optional)
  79             cpu -             required number of virtual cpus for VM (optional)
  80             disk -            required size of disk for VM in GB (optional)
  81             os_type-          required operating system type as specified in user document
  82                                 (default os type other 32 bit) (optional)
  83             disk_controller - required disk controller type
  84                                 (default controller SCSI with lsilogicsas)
  85                                 (SATA, IDE, Paravirtual, Buslogic, Lsilogic, Lsilogicsas) (optional)
  86             options - subformat option for OVF  (optional)
  87
  88             Returns:
  89                 Nothing.
  90         """
  91         self.logger = logger
  92         self.ovf_template_path = OVF_TEMPLATE_PATH
  93
  94         self.source_img_path = source_img_path
  95         file_location, file_extension = os.path.splitext(self.source_img_path)
  96         self.source_img_location = os.path.dirname(self.source_img_path)
  97         self.source_format = file_extension[1:]
  98         self.source_img_filename = os.path.basename(self.source_img_path).split('.')[0]
  99
 100         self.output_format = "ovf"
 101         self.output_ovf_name = output_ovf_name.split('.')[0] if output_ovf_name else self.source_img_filename
 102         self.output_location = output_location if output_location else self.source_img_location
 103         self.output_ovf_name_ext = self.output_ovf_name + "." + self.output_format
 104         self.output_path = os.path.join(self.output_location , self.output_ovf_name_ext )
 105
 106         self.output_diskimage_format = "vmdk"
 107         self.output_diskimage_name = self.source_img_filename + "."+ self.output_diskimage_format
 108         self.output_diskimage_path = os.path.join(self.output_location,  self.output_diskimage_name)
 109
 110
 111         self.logger.info("Input parameters to Converter: \n ovf_template_path = {}, \n source_img_path = {}, \n"\
 112                     "source_img_location ={} , \n source_format = {}, \n source_img_filename = {}".format(
 113                                                         self.ovf_template_path,
 114                                                         self.source_img_path, self.source_img_location,
 115                                                         self.source_format, self.source_img_filename ))
 116
 117         self.logger.info("Output parameters to Converter: \n output_format = {}, \n output_ovf_name = {}, \n"\
 118                     "output_location ={} , \n output_path = {}, \n output_diskimage_name = {} , \n"\
 119                     " output_diskimage_path = {} ".format(self.output_format, self.output_ovf_name,
 120                                                     self.output_location, self.output_path,
 121                                                     self.output_diskimage_name,self.output_diskimage_path ))
 122
 123
 124         self.disk_capacity = 1
 125         self.disk_populated_size = 0
 126
 127         self.vm_name = self.output_ovf_name
 128         self.memory = str(memory) if memory is not None else None
 129         self.cpu = str(cpu) if cpu is not None else None
 130         self.os_type=str(os_type).strip() if os_type else None
 131         self.cdrom = cdrom
 132
 133         if self.os_type:
 134             self.osID , self.osType = self.__get_osType()
 135             if self.osID is None or self.osType is None:
 136                error_msg = "ERROR: Invalid input can not find OS type {} ".format(self.os_type)
 137                self.__raise_exception(error_msg)
 138
 139         self.disk_controller = str(disk_controller).strip() if disk_controller else None
 140
 141         if self.disk_controller:
 142             self.disk_controller_info = self.__get_diskcontroller()
 143
 144             if not self.disk_controller_info:
 145                 error_msg = "ERROR: Invalid input can not find Disk Controller {} ".format(self.disk_controller)
 146                 self.__raise_exception(error_msg)
 147
 148         if disk is not None:
 149             #convert disk size from GB to bytes
 150             self.disk_size = int(disk) * 1024 * 1024 * 1024
 151         else:
 152             self.disk_size = None
 153
 154         self.logger.info("Other input parameters to Converter: \n vm_name = {}, \n memory = {}, \n"\
 155                     "disk_size ={} \n os type = {} \n disk controller = {}".format(
 156                                 self.vm_name, self.memory, self.disk_size, self.os_type, self.disk_controller
 157                                 ))
 158
 159         #check access for read input location and write output location return none if no access
 160         if not os.access(self.source_img_path, os.F_OK):
 161             error_msg = "ERROR: Source image file {} not present".format(self.source_img_path)
 162             self.__raise_exception(error_msg, exception_type="IO")
 163
 164         elif not os.access(self.source_img_path, os.R_OK):
 165             error_msg = "ERROR: Cannot read source image file {}".format(self.source_img_path)
 166             self.__raise_exception(error_msg, exception_type="IO")
 167
 168         if not os.access(self.output_location, os.W_OK):
 169             error_msg = "ERROR: Not have write access to location {} to write output OVF ".format(self.source_img_path)
 170             self.__raise_exception(error_msg, exception_type="IO")
 171
 172     def __get_image_info(self):
 173         """
 174             Private method to get information about source imager.
 175             Args  : None
 176             Return : True on success else False
 177         """
 178         try:
 179             print("Getting source image information")
 180             command = "qemu-img info \t " + self.source_img_path
 181             output, error, returncode= self.__execute_command(command)
 182
 183             if error or returncode:
 184                 self.logger.error("ERROR: Error occurred while getting information about source image : {} \n "\
 185                                   "return code : {} ".format(error, returncode))
 186                 return False
 187
 188             elif output:
 189                 self.logger.info("Get Image Info Output : {} \n ".format(output))
 190                 split_output = output.split("\n")
 191                 for line in split_output:
 192                     line = line.strip()
 193                     if "virtual size" in line:
 194                         virtual_size_info = line.split(":")[1].split()
 195                         if len(virtual_size_info) == 3 and virtual_size_info[2].strip(")") == "bytes":
 196                             self.disk_capacity  = int(virtual_size_info[1].strip("("))
 197                         else:
 198                             self.disk_capacity  = self.__convert_size(virtual_size_info[0])
 199
 200                     elif "disk size" in line:
 201                         size = line.split(":")[1].split()[0]
 202                         self.disk_populated_size = self.__convert_size(size)
 203                     elif "file format" in line:
 204                         self.source_format = line.split(":")[1]
 205
 206                 self.logger.info("Updated source image virtual disk capacity : {} ,"\
 207                                  "Updated source image populated size: {}".format(self.disk_capacity,
 208                                                                     self.disk_populated_size))
 209                 return True
 210         except Exception as exp:
 211             error_msg = "ERROR: Error occurred while getting information about source image : {}".format(exp)
 212             self.logger.error(error_msg)
 213             print(error_msg)
 214             return False
 215
 216     def __convert_image(self):
 217         """
 218             Private method to convert source disk image into .vmdk disk image.
 219             Args  : None
 220             Return : True on success else False
 221         """
 222
 223         print("Converting source disk image to .vmdk ")
 224
 225         progress = CommandProgressbar()
 226         progress.start_progressbar()
 227
 228         command = "qemu-img convert -f "+ self.source_format +" -O " + self.output_diskimage_format + \
 229                 " -o subformat=streamOptimized " + self.source_img_path + "\t" + self.output_diskimage_path
 230
 231         output, error , returncode = self.__execute_command(command)
 232
 233         progress.stop_progressbar()
 234
 235         if error or returncode :
 236             error_msg = "ERROR: Error occurred while converting source disk image into vmdk : {} \n "\
 237                                   "return code : {} ".format(error, returncode)
 238             self.logger.error(error_msg)
 239             print(error_msg)
 240             return False
 241         else:
 242             if os.path.isfile(self.output_diskimage_path):
 243                 self.logger.info("Successfully converted source image {} into {} \n "\
 244                                   "return code : {} ".format(self.source_img_path,
 245                                                         self.output_diskimage_path,
 246                                                         returncode))
 247                 result = self.__make_image_bootable()
 248                 if result:
 249                     self.logger.info("Made {} bootable".format(self.output_diskimage_path))
 250                     return True
 251                 else:
 252                     self.logger.error("Cannot make {} bootable".format(self.output_diskimage_path))
 253                     print("ERROR: Fail to convert source image into .vmdk")
 254                     return False
 255             else:
 256                 self.logger.error("Converted vmdk disk file {} is not present \n ".format(
 257                                                     self.output_diskimage_path))
 258                 print("Fail to convert source image into .vmdk")
 259                 return False
 260
 261     def __make_image_bootable(self):
 262         """
 263             Private method to make source disk image bootable.
 264             Args  : None
 265             Return : True on success else False
 266         """
 267         command = "printf '\x03' | dd conv=notrunc of="+ self.output_diskimage_path + "\t bs=1 seek=$((0x4))"
 268         output, error, returncode = self.__execute_command(command)
 269
 270         if error and returncode :
 271             error_msg = "ERROR:Error occurred while making source disk image bootable : {} \n "\
 272                                   "return code : {} ".format(error, returncode)
 273             self.logger.error(error_msg)
 274             print(error_msg)
 275             return False
 276         else:
 277             self.logger.info("Make Image Bootable Output : {} ".format(output))
 278             return True
 279
 280
 281     def __edit_ovf_template(self):
 282         """
 283             Private method to create new OVF file by editing OVF template
 284             Args  : None
 285             Return : True on success else False
 286         """
 287         try:
 288             print("\nCreating OVF")
 289             #Read OVF template file
 290             OVF_tree = ET.parse(self.ovf_template_path)
 291             root = OVF_tree.getroot()
 292
 293             #Collect namespaces
 294             nsmap = {k:v for k,v in root.nsmap.iteritems() if k}
 295             nsmap["xmlns"]= "http://schemas.dmtf.org/ovf/envelope/1"
 296
 297             #Edit OVF template
 298             references = root.find('xmlns:References',nsmap)
 299             if references is not None:
 300                 file_tag = references.find('xmlns:File', nsmap)
 301                 if file_tag is not None:
 302                     file_tag.attrib['{'+nsmap['ovf']+'}href'] = self.output_diskimage_name
 303
 304             disksection = root.find('xmlns:DiskSection',nsmap)
 305             if disksection is not None:
 306                 diak_tag = disksection.find('xmlns:Disk', nsmap)
 307                 if diak_tag is not None:
 308                     if self.disk_size and self.disk_size > self.disk_capacity:
 309                         self.disk_capacity = self.disk_size
 310
 311                     diak_tag.attrib['{'+nsmap['ovf']+'}capacity'] = str(self.disk_capacity)
 312                     diak_tag.attrib['{'+nsmap['ovf']+'}populatedSize'] = str(self.disk_populated_size)
 313
 314             virtuasystem = root.find('xmlns:VirtualSystem',nsmap)
 315             if virtuasystem is not None:
 316                 name_tag = virtuasystem.find('xmlns:Name', nsmap)
 317                 if name_tag is not None:
 318                     name_tag.text = self.vm_name
 319
 320                 if self.os_type is not None:
 321                     operatingSystemSection = virtuasystem.find('xmlns:OperatingSystemSection', nsmap)
 322                     if self.osID and self.osType:
 323                         operatingSystemSection.attrib['{'+nsmap['ovf']+'}id'] = self.osID
 324                         os_discription_tag = operatingSystemSection.find('xmlns:Description', nsmap)
 325                         os_discription_tag.text = self.osType
 326
 327                 virtualHardwareSection = virtuasystem.find('xmlns:VirtualHardwareSection', nsmap)
 328                 system = virtualHardwareSection.find('xmlns:System', nsmap)
 329                 virtualSystemIdentifier = system.find('vssd:VirtualSystemIdentifier', nsmap)
 330                 if virtualSystemIdentifier is not None:
 331                     virtualSystemIdentifier.text = self.vm_name
 332
 333                 if self.memory is not None or self.cpu is not None or self.disk_controller is not None:
 334                     for item in virtualHardwareSection.iterfind('xmlns:Item',nsmap):
 335                         description = item.find("rasd:Description",nsmap)
 336
 337                         if self.cpu is not None:
 338                             if description is not None and description.text == "Number of Virtual CPUs":
 339                                 cpu_item = item.find("rasd:VirtualQuantity", nsmap)
 340                                 name_item = item.find("rasd:ElementName", nsmap)
 341                                 if cpu_item is not None:
 342                                     cpu_item.text = self.cpu
 343                                     name_item.text = self.cpu+" virtual CPU(s)"
 344
 345                         if self.memory is not None:
 346                             if description is not None and description.text == "Memory Size":
 347                                 mem_item = item.find("rasd:VirtualQuantity", nsmap)
 348                                 name_item = item.find("rasd:ElementName", nsmap)
 349                                 if mem_item is not None:
 350                                     mem_item.text = self.memory
 351                                     name_item.text = self.memory + " MB of memory"
 352
 353                         if self.disk_controller is not None:
 354                             if description is not None and description.text == "SCSI Controller":
 355                                 if self.disk_controller_info is not None:
 356                                     name_item = item.find("rasd:ElementName", nsmap)
 357                                     name_item.text = str(self.disk_controller_info["controllerName"])+"0"
 358
 359                                     resource_type = item.find("rasd:ResourceType", nsmap)
 360                                     resource_type.text = self.disk_controller_info["resourceType"]
 361
 362                                     description.text = self.disk_controller_info["controllerName"]
 363                                     resource_subtype = item.find("rasd:ResourceSubType", nsmap)
 364                                     if self.disk_controller_info["controllerName"] == "IDE Controller":
 365                                         #Remove resource subtype item
 366                                         resource_subtype.getparent().remove(resource_subtype)
 367                                     if "resourceSubType" in  self.disk_controller_info:
 368                                         resource_subtype.text = self.disk_controller_info["resourceSubType"]
 369                 if self.cdrom:
 370                     last_item = list(virtualHardwareSection.iterfind('xmlns:Item',nsmap))[-1]
 371                     ide_cdrom_items_etree = ET.parse(IDE_CDROM_XML_PATH)
 372                     ide_cdrom_items = list(ide_cdrom_items_etree.iterfind('Item'))
 373                     for item in ide_cdrom_items:
 374                         last_item.addnext(item)
 375
 376             # Save output OVF
 377             OVF_tree.write(self.output_path, xml_declaration=True,encoding='utf-8',
 378                method="xml" )
 379
 380             if os.path.isfile(self.output_path):
 381                 logger.info("Successfully written output OVF at {}".format(self.output_path))
 382                 print("Output OVF is at :  {}".format(self.output_path))
 383                 return self.output_path
 384             else:
 385                 error_msg = "ERROR: Error occurred while creating OVF file"
 386                 print(error_msg)
 387                 return False
 388
 389         except Exception as exp:
 390             error_msg = "ERROR: Error occurred while editing OVF template : {}".format(exp)
 391             self.logger.error(error_msg)
 392             print(error_msg)
 393             return False
 394
 395
 396     def __convert_size(self,size):
 397         """
 398             Private method to convert disk size from GB,MB to bytes.
 399             Args :
 400                 size  : disk size with prefix 'G' for GB and 'M' for MB
 401             Return :  disk size in bytes
 402         """
 403         byte_size= 0
 404         try:
 405             if not size:
 406                 self.logger.error("No size {} to convert in bytes".format(size))
 407             else:
 408                 size = str(size)
 409                 disk_size = float(size[:-1])
 410                 input_type = size[-1].strip()
 411
 412                 self.logger.info("Disk size : {} , size type : {} ".format(disk_size,input_type))
 413
 414                 if input_type == "G":
 415                     byte_size = disk_size * 1024 * 1024 *1024
 416                 elif input_type == "M":
 417                     byte_size = disk_size * 1024 * 1024
 418
 419                 self.logger.info("Disk size in bytes: {} ".format(byte_size))
 420
 421             return int(byte_size)
 422
 423         except Exception as exp:
 424             error_msg = "ERROR:Error occurred while converting disk size in bytes : {}".format(exp)
 425             self.logger.error(error_msg)
 426             print(error_msg)
 427             return False
 428
 429     def __get_osType(self):
 430         """
 431             Private method to get OS ID and Type
 432             Args :
 433                 None
 434             Return :
 435                 osID : OS ID
 436                 osType: OS Type
 437         """
 438         osID = None
 439         osType = None
 440         os_info = self.__read_yaml_file(OS_INFO_FILE_PATH)
 441         try:
 442             if self.os_type and os_info:
 443                 for os_id , os_type in os_info.iteritems():
 444                     if self.os_type.lower() == os_type.lower():
 445                         osID = os_id
 446                         osType = os_type
 447                         break
 448         except Exception as exp:
 449             error_msg = "ERROR:Error occurred while getting OS details : {}".format(exp)
 450             self.logger.error(error_msg)
 451             print(error_msg)
 452
 453         return osID, osType
 454
 455
 456     def __get_diskcontroller(self):
 457         """
 458             Private method to get details of Disk Controller
 459             Args :
 460                 None
 461             Return :
 462                 disk_controller : dict with details of Disk Controller
 463         """
 464         disk_controller = {}
 465         scsi_subtype = None
 466         if self.disk_controller.lower() in ["paravirtual", "lsilogic", "buslogic", "lsilogicsas"]:
 467             scsi_subtype = self.disk_controller
 468             self.disk_controller = "SCSI"
 469
 470         disk_controller_info = self.__read_yaml_file(DISK_CONTROLLER_INFO_FILE_PATH)
 471         try:
 472             if self.disk_controller and disk_controller_info:
 473                 for key , value in disk_controller_info.iteritems():
 474                     if self.disk_controller.lower() in key.lower():
 475                         disk_controller['controllerName'] = key
 476                         disk_controller['resourceType'] = str(value["ResourceType"])
 477                         resourceSubTypes = value["ResourceSubTypes"] if "ResourceSubTypes" in value else None
 478                         if key == "SATA Controller":
 479                             disk_controller["resourceSubType"] = resourceSubTypes[0]
 480                         elif key == "SCSI Controller":
 481                             if scsi_subtype:
 482                                 if scsi_subtype.lower() == "paravirtual":
 483                                     scsi_subtype = "VirtualSCSI"
 484                                 for subtype in resourceSubTypes:
 485                                     if scsi_subtype.lower() == subtype.lower():
 486                                         disk_controller["resourceSubType"] = subtype
 487                                         break
 488                                 else:
 489                                     error_msg = "ERROR: Invalid inputs can not "\
 490                                     "find SCSI subtype {}".format(scsi_subtype)
 491                                     self.__raise_exception(error_msg)
 492
 493         except KeyError as exp:
 494             error_msg = "ERROR:Error occurred while getting Disk Controller details : {}".format(exp)
 495             self.logger.error(error_msg)
 496             print(error_msg)
 497
 498         return disk_controller
 499
 500     def __read_yaml_file(self, file_path):
 501         """
 502             Private method to execute command
 503             Args :
 504                 command  : command to execute
 505             Return :
 506                 Dict of yaml data
 507         """
 508         with open(file_path) as data_file:
 509             data = yaml.load(data_file)
 510         return data
 511
 512     def __raise_exception(self, error_msg , exception_type="Generic"):
 513         """
 514             Private method to execute command
 515             Args :
 516                 command  : command to execute
 517             Return :
 518                None
 519         """
 520         if error_msg:
 521             self.logger.debug(error_msg)
 522             print(error_msg)
 523             if exception_type == "Generic":
 524                 raise Exception(error_msg)
 525             elif exception_type == "IO":
 526                 raise Exception(error_msg)
 527
 528     def __execute_command(self, command):
 529         """
 530             Private method to execute command
 531             Args :
 532                 command  : command to execute
 533             Return :
 534                 stdout : output of command
 535                 stderr: error occurred while executing command if any
 536                 returncode : return code of command execution
 537         """
 538         try:
 539             self.logger.info("Execute command: {} ".format(command))
 540
 541             proc = subprocess.Popen(command , stdout = subprocess.PIPE, stdin = subprocess.PIPE,
 542                                              stderr = subprocess.PIPE,shell=True)
 543             stdout, stderr = proc.communicate()
 544             returncode = proc.returncode
 545
 546         except Exception as exp:
 547             self.logger.error("Error {} occurred while executing command {} ".format(exp,command))
 548
 549         return  stdout, stderr , returncode
 550
 551
 552     def create_ovf(self):
 553         """
 554             Method to convert source image into OVF
 555             Args : None
 556             Return : True on success else False
 557         """
 558         #check output format
 559         if self.source_format == self.output_format:
 560             self.logger.info("Source format is OVF. No need to convert: {} ")
 561             return self.source_img_path
 562
 563         #Get source img properties
 564         img_info = self.__get_image_info()
 565         if img_info:
 566
 567             #Create vmdk disk image
 568             disk_img = self.__convert_image()
 569             if disk_img:
 570
 571                 #Edit OVF tempalte
 572                 ovf_path = self.__edit_ovf_template()
 573                 return ovf_path
 574         else:
 575             self.logger.error("Error in getting image information cannot convert image")
 576             raise Exception("Error in getting image information cannot convert image")
 577             return False
 578