osm_ro/vmware_utils/OVF_converter/format_converter/converter.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2016-2017 VMware Inc.
   6 # This file is part of ETSI OSM
   7 # All Rights Reserved.
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  10 # not use this file except in compliance with the License. You may obtain
  11 # a copy of the License at
  12 #
  13 #         http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  18 # License for the specific language governing permissions and limitations
  19 # under the License.
  20 #
  21 # For those usages not covered by the Apache License, Version 2.0 please
  22 # contact:  osslegalrouting@vmware.com
  23 ##
  24
  25 import logging
  26 import os
  27 import subprocess
  28 import yaml
  29 from lxml import etree as ET
  30 from command_progress import CommandProgressbar
  31
  32 #file paths
  33 MODULE_DIR = os.path.dirname(__file__)
  34 OVF_TEMPLATE_PATH = os.path.join(os.path.dirname(MODULE_DIR),
  35                                 "ovf_template/template.xml")
  36 OS_INFO_FILE_PATH = os.path.join(os.path.dirname(MODULE_DIR),
  37                                 "config/os_type.yaml")
  38 DISK_CONTROLLER_INFO_FILE_PATH = os.path.join(os.path.dirname(MODULE_DIR),
  39                                               "config/disk_controller.yaml")
  40
  41
  42 #Set logger
  43 LOG_FILE = os.path.join(os.path.dirname(MODULE_DIR),"logs/ovf_converter.log")
  44 logger = logging.getLogger(__name__)
  45 hdlr = logging.FileHandler(LOG_FILE)
  46 formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
  47 hdlr.setFormatter(formatter)
  48 logger.addHandler(hdlr)
  49 logger.setLevel(10)
  50
  51 __version__ = "1.0"
  52 __description__ = "initial Release"
  53
  54 def get_version(*args, **kwargs):
  55     """ get version of this application"""
  56     version = str(__version__ ) +" - "+ str( __description__ )
  57     return version
  58
  59 #converter class
  60 class OVFConverter(object):
  61     """ Class to convert input image into OVF format """
  62
  63     def __init__(self, source_img_path, output_location=None, output_ovf_name=None,
  64                     memory=None, cpu=None, disk=None, os_type=None,
  65                     disk_controller=None,
  66                     options={'subformat':'streamOptimized'}):
  67         """
  68             Constructor to initialize object of class OVFConverter
  69             Args:
  70             source_img_path - absolute path to source image which will get convert into ovf
  71             output_location - location where created OVF will be kept. This location
  72                               should have write access. If not given file will get
  73                               created at source location  (optional)
  74             output_ovf_name - name of output ovf.If not given source image name will
  75                               be used (optional)
  76             memory -          required memory for VM in MB (optional)
  77             cpu -             required number of virtual cpus for VM (optional)
  78             disk -            required size of disk for VM in GB (optional)
  79             os_type-          required operating system type as specified in user document
  80                                 (default os type other 32 bit) (optional)
  81             disk_controller - required disk controller type
  82                                 (default controller SCSI with lsilogicsas)
  83                                 (SATA, IDE, Paravirtual, Buslogic, Lsilogic, Lsilogicsas) (optional)
  84             options - subformat option for OVF  (optional)
  85
  86             Returns:
  87                 Nothing.
  88         """
  89         self.logger = logger
  90         self.ovf_template_path = OVF_TEMPLATE_PATH
  91
  92         self.source_img_path = source_img_path
  93         file_location, file_extension = os.path.splitext(self.source_img_path)
  94         self.source_img_location = os.path.dirname(self.source_img_path)
  95         self.source_format = file_extension[1:]
  96         self.source_img_filename = os.path.basename(self.source_img_path).split('.')[0]
  97
  98         self.output_format = "ovf"
  99         self.output_ovf_name = output_ovf_name.split('.')[0] if output_ovf_name else self.source_img_filename
 100         self.output_location = output_location if output_location else self.source_img_location
 101         self.output_ovf_name_ext = self.output_ovf_name + "." + self.output_format
 102         self.output_path = os.path.join(self.output_location , self.output_ovf_name_ext )
 103
 104         self.output_diskimage_format = "vmdk"
 105         self.output_diskimage_name = self.source_img_filename + "."+ self.output_diskimage_format
 106         self.output_diskimage_path = os.path.join(self.output_location,  self.output_diskimage_name)
 107
 108
 109         self.logger.info("Input parameters to Converter: \n ovf_template_path = {}, \n source_img_path = {}, \n"\
 110                     "source_img_location ={} , \n source_format = {}, \n source_img_filename = {}".format(
 111                                                         self.ovf_template_path,
 112                                                         self.source_img_path, self.source_img_location,
 113                                                         self.source_format, self.source_img_filename ))
 114
 115         self.logger.info("Output parameters to Converter: \n output_format = {}, \n output_ovf_name = {}, \n"\
 116                     "output_location ={} , \n output_path = {}, \n output_diskimage_name = {} , \n"\
 117                     " output_diskimage_path = {} ".format(self.output_format, self.output_ovf_name,
 118                                                     self.output_location, self.output_path,
 119                                                     self.output_diskimage_name,self.output_diskimage_path ))
 120
 121
 122         self.disk_capacity = 1
 123         self.disk_populated_size = 0
 124
 125         self.vm_name = self.output_ovf_name
 126         self.memory = str(memory) if memory is not None else None
 127         self.cpu = str(cpu) if cpu is not None else None
 128         self.os_type=str(os_type).strip() if os_type else None
 129
 130         if self.os_type:
 131             self.osID , self.osType = self.__get_osType()
 132             if self.osID is None or self.osType is None:
 133                error_msg = "ERROR: Invalid input can not find OS type {} ".format(self.os_type)
 134                self.__raise_exception(error_msg)
 135
 136         self.disk_controller = str(disk_controller).strip() if disk_controller else None
 137
 138         if self.disk_controller:
 139             self.disk_controller_info = self.__get_diskcontroller()
 140
 141             if not self.disk_controller_info:
 142                 error_msg = "ERROR: Invalid input can not find Disk Controller {} ".format(self.disk_controller)
 143                 self.__raise_exception(error_msg)
 144
 145         if disk is not None:
 146             #convert disk size from GB to bytes
 147             self.disk_size = int(disk) * 1024 * 1024 * 1024
 148         else:
 149             self.disk_size = None
 150
 151         self.logger.info("Other input parameters to Converter: \n vm_name = {}, \n memory = {}, \n"\
 152                     "disk_size ={} \n os type = {} \n disk controller = {}".format(
 153                                 self.vm_name, self.memory, self.disk_size, self.os_type, self.disk_controller
 154                                 ))
 155
 156         #check access for read input location and write output location return none if no access
 157         if not os.access(self.source_img_path, os.F_OK):
 158             error_msg = "ERROR: Source image file {} not present".format(self.source_img_path)
 159             self.__raise_exception(error_msg, exception_type="IO")
 160
 161         elif not os.access(self.source_img_path, os.R_OK):
 162             error_msg = "ERROR: Cannot read source image file {}".format(self.source_img_path)
 163             self.__raise_exception(error_msg, exception_type="IO")
 164
 165         if not os.access(self.output_location, os.W_OK):
 166             error_msg = "ERROR: Not have write access to location {} to write output OVF ".format(self.source_img_path)
 167             self.__raise_exception(error_msg, exception_type="IO")
 168
 169     def __get_image_info(self):
 170         """
 171             Private method to get information about source imager.
 172             Args  : None
 173             Return : True on success else False
 174         """
 175         try:
 176             print("Getting source image information")
 177             command = "qemu-img info \t " + self.source_img_path
 178             output, error, returncode= self.__execute_command(command)
 179
 180             if error or returncode:
 181                 self.logger.error("ERROR: Error occurred while getting information about source image : {} \n "\
 182                                   "return code : {} ".format(error, returncode))
 183                 return False
 184
 185             elif output:
 186                 self.logger.info("Get Image Info Output : {} \n ".format(output))
 187                 split_output = output.split("\n")
 188                 for line in split_output:
 189                     line = line.strip()
 190                     if "virtual size" in line:
 191                         virtual_size_info = line.split(":")[1].split()
 192                         if len(virtual_size_info) == 3 and virtual_size_info[2].strip(")") == "bytes":
 193                             self.disk_capacity  = int(virtual_size_info[1].strip("("))
 194                         else:
 195                             self.disk_capacity  = self.__convert_size(virtual_size_info[0])
 196
 197                     elif "disk size" in line:
 198                         size = line.split(":")[1].split()[0]
 199                         self.disk_populated_size = self.__convert_size(size)
 200                     elif "file format" in line:
 201                         self.source_format = line.split(":")[1]
 202
 203                 self.logger.info("Updated source image virtual disk capacity : {} ,"\
 204                                  "Updated source image populated size: {}".format(self.disk_capacity,
 205                                                                     self.disk_populated_size))
 206                 return True
 207         except Exception as exp:
 208             error_msg = "ERROR: Error occurred while getting information about source image : {}".format(exp)
 209             self.logger.error(error_msg)
 210             print(error_msg)
 211             return False
 212
 213     def __convert_image(self):
 214         """
 215             Private method to convert source disk image into .vmdk disk image.
 216             Args  : None
 217             Return : True on success else False
 218         """
 219
 220         print("Converting source disk image to .vmdk ")
 221
 222         progress = CommandProgressbar()
 223         progress.start_progressbar()
 224
 225         command = "qemu-img convert -f "+ self.source_format +" -O " + self.output_diskimage_format + \
 226                 " -o subformat=streamOptimized " + self.source_img_path + "\t" + self.output_diskimage_path
 227
 228         output, error , returncode = self.__execute_command(command)
 229
 230         progress.stop_progressbar()
 231
 232         if error or returncode :
 233             error_msg = "ERROR: Error occurred while converting source disk image into vmdk : {} \n "\
 234                                   "return code : {} ".format(error, returncode)
 235             self.logger.error(error_msg)
 236             print(error_msg)
 237             return False
 238         else:
 239             if os.path.isfile(self.output_diskimage_path):
 240                 self.logger.info("Successfully converted source image {} into {} \n "\
 241                                   "return code : {} ".format(self.source_img_path,
 242                                                         self.output_diskimage_path,
 243                                                         returncode))
 244                 result = self.__make_image_bootable()
 245                 if result:
 246                     self.logger.info("Made {} bootable".format(self.output_diskimage_path))
 247                     return True
 248                 else:
 249                     self.logger.error("Cannot make {} bootable".format(self.output_diskimage_path))
 250                     print("ERROR: Fail to convert source image into .vmdk")
 251                     return False
 252             else:
 253                 self.logger.error("Converted vmdk disk file {} is not present \n ".format(
 254                                                     self.output_diskimage_path))
 255                 print("Fail to convert source image into .vmdk")
 256                 return False
 257
 258     def __make_image_bootable(self):
 259         """
 260             Private method to make source disk image bootable.
 261             Args  : None
 262             Return : True on success else False
 263         """
 264         command = "printf '\x03' | dd conv=notrunc of="+ self.output_diskimage_path + "\t bs=1 seek=$((0x4))"
 265         output, error, returncode = self.__execute_command(command)
 266
 267         if error and returncode :
 268             error_msg = "ERROR:Error occurred while making source disk image bootable : {} \n "\
 269                                   "return code : {} ".format(error, returncode)
 270             self.logger.error(error_msg)
 271             print(error_msg)
 272             return False
 273         else:
 274             self.logger.info("Make Image Bootable Output : {} ".format(output))
 275             return True
 276
 277
 278     def __edit_ovf_template(self):
 279         """
 280             Private method to create new OVF file by editing OVF template
 281             Args  : None
 282             Return : True on success else False
 283         """
 284         try:
 285             print("\nCreating OVF")
 286             #Read OVF template file
 287             OVF_tree = ET.parse(self.ovf_template_path)
 288             root = OVF_tree.getroot()
 289
 290             #Collect namespaces
 291             nsmap = {k:v for k,v in root.nsmap.iteritems() if k}
 292             nsmap["xmlns"]= "http://schemas.dmtf.org/ovf/envelope/1"
 293
 294             #Edit OVF template
 295             references = root.find('xmlns:References',nsmap)
 296             if references is not None:
 297                 file_tag = references.find('xmlns:File', nsmap)
 298                 if file_tag is not None:
 299                     file_tag.attrib['{'+nsmap['ovf']+'}href'] = self.output_diskimage_name
 300
 301             disksection = root.find('xmlns:DiskSection',nsmap)
 302             if disksection is not None:
 303                 diak_tag = disksection.find('xmlns:Disk', nsmap)
 304                 if diak_tag is not None:
 305                     if self.disk_size and self.disk_size > self.disk_capacity:
 306                         self.disk_capacity = self.disk_size
 307
 308                     diak_tag.attrib['{'+nsmap['ovf']+'}capacity'] = str(self.disk_capacity)
 309                     diak_tag.attrib['{'+nsmap['ovf']+'}populatedSize'] = str(self.disk_populated_size)
 310
 311             virtuasystem = root.find('xmlns:VirtualSystem',nsmap)
 312             if virtuasystem is not None:
 313                 name_tag = virtuasystem.find('xmlns:Name', nsmap)
 314                 if name_tag is not None:
 315                     name_tag.text = self.vm_name
 316
 317                 if self.os_type is not None:
 318                     operatingSystemSection = virtuasystem.find('xmlns:OperatingSystemSection', nsmap)
 319                     if self.osID and self.osType:
 320                         operatingSystemSection.attrib['{'+nsmap['ovf']+'}id'] = self.osID
 321                         os_discription_tag = operatingSystemSection.find('xmlns:Description', nsmap)
 322                         os_discription_tag.text = self.osType
 323
 324                 virtualHardwareSection = virtuasystem.find('xmlns:VirtualHardwareSection', nsmap)
 325                 system = virtualHardwareSection.find('xmlns:System', nsmap)
 326                 virtualSystemIdentifier = system.find('vssd:VirtualSystemIdentifier', nsmap)
 327                 if virtualSystemIdentifier is not None:
 328                     virtualSystemIdentifier.text = self.vm_name
 329
 330                 if self.memory is not None or self.cpu is not None or self.disk_controller is not None:
 331                     for item in virtualHardwareSection.iterfind('xmlns:Item',nsmap):
 332                         description = item.find("rasd:Description",nsmap)
 333
 334                         if self.cpu is not None:
 335                             if description is not None and description.text == "Number of Virtual CPUs":
 336                                 cpu_item = item.find("rasd:VirtualQuantity", nsmap)
 337                                 name_item = item.find("rasd:ElementName", nsmap)
 338                                 if cpu_item is not None:
 339                                     cpu_item.text = self.cpu
 340                                     name_item.text = self.cpu+" virtual CPU(s)"
 341
 342                         if self.memory is not None:
 343                             if description is not None and description.text == "Memory Size":
 344                                 mem_item = item.find("rasd:VirtualQuantity", nsmap)
 345                                 name_item = item.find("rasd:ElementName", nsmap)
 346                                 if mem_item is not None:
 347                                     mem_item.text = self.memory
 348                                     name_item.text = self.memory + " MB of memory"
 349
 350                         if self.disk_controller is not None:
 351                             if description is not None and description.text == "SCSI Controller":
 352                                 if self.disk_controller_info is not None:
 353                                     name_item = item.find("rasd:ElementName", nsmap)
 354                                     name_item.text = str(self.disk_controller_info["controllerName"])+"0"
 355
 356                                     resource_type = item.find("rasd:ResourceType", nsmap)
 357                                     resource_type.text = self.disk_controller_info["resourceType"]
 358
 359                                     description.text = self.disk_controller_info["controllerName"]
 360                                     resource_subtype = item.find("rasd:ResourceSubType", nsmap)
 361                                     if self.disk_controller_info["controllerName"] == "IDE Controller":
 362                                         #Remove resource subtype item
 363                                         resource_subtype.getparent().remove(resource_subtype)
 364                                     if "resourceSubType" in  self.disk_controller_info:
 365                                         resource_subtype.text = self.disk_controller_info["resourceSubType"]
 366
 367                 #Save output OVF
 368             OVF_tree.write(self.output_path, xml_declaration=True,encoding='utf-8',
 369                method="xml" )
 370
 371             if os.path.isfile(self.output_path):
 372                 logger.info("Successfully written output OVF at {}".format(self.output_path))
 373                 print("Output OVF is at :  {}".format(self.output_path))
 374                 return self.output_path
 375             else:
 376                 error_msg = "ERROR: Error occurred while creating OVF file"
 377                 print(error_msg)
 378                 return False
 379
 380         except Exception as exp:
 381             error_msg = "ERROR: Error occurred while editing OVF template : {}".format(exp)
 382             self.logger.error(error_msg)
 383             print(error_msg)
 384             return False
 385
 386
 387     def __convert_size(self,size):
 388         """
 389             Private method to convert disk size from GB,MB to bytes.
 390             Args :
 391                 size  : disk size with prefix 'G' for GB and 'M' for MB
 392             Return :  disk size in bytes
 393         """
 394         byte_size= 0
 395         try:
 396             if not size:
 397                 self.logger.error("No size {} to convert in bytes".format(size))
 398             else:
 399                 size = str(size)
 400                 disk_size = float(size[:-1])
 401                 input_type = size[-1].strip()
 402
 403                 self.logger.info("Disk size : {} , size type : {} ".format(disk_size,input_type))
 404
 405                 if input_type == "G":
 406                     byte_size = disk_size * 1024 * 1024 *1024
 407                 elif input_type == "M":
 408                     byte_size = disk_size * 1024 * 1024
 409
 410                 self.logger.info("Disk size in bytes: {} ".format(byte_size))
 411
 412             return int(byte_size)
 413
 414         except Exception as exp:
 415             error_msg = "ERROR:Error occurred while converting disk size in bytes : {}".format(exp)
 416             self.logger.error(error_msg)
 417             print(error_msg)
 418             return False
 419
 420     def __get_osType(self):
 421         """
 422             Private method to get OS ID and Type
 423             Args :
 424                 None
 425             Return :
 426                 osID : OS ID
 427                 osType: OS Type
 428         """
 429         osID = None
 430         osType = None
 431         os_info = self.__read_yaml_file(OS_INFO_FILE_PATH)
 432         try:
 433             if self.os_type and os_info:
 434                 for os_id , os_type in os_info.iteritems():
 435                     if self.os_type.lower() == os_type.lower():
 436                         osID = os_id
 437                         osType = os_type
 438                         break
 439         except Exception as exp:
 440             error_msg = "ERROR:Error occurred while getting OS details : {}".format(exp)
 441             self.logger.error(error_msg)
 442             print(error_msg)
 443
 444         return osID, osType
 445
 446
 447     def __get_diskcontroller(self):
 448         """
 449             Private method to get details of Disk Controller
 450             Args :
 451                 None
 452             Return :
 453                 disk_controller : dict with details of Disk Controller
 454         """
 455         disk_controller = {}
 456         scsi_subtype = None
 457         if self.disk_controller.lower() in ["paravirtual", "lsilogic", "buslogic", "lsilogicsas"]:
 458             scsi_subtype = self.disk_controller
 459             self.disk_controller = "SCSI"
 460
 461         disk_controller_info = self.__read_yaml_file(DISK_CONTROLLER_INFO_FILE_PATH)
 462         try:
 463             if self.disk_controller and disk_controller_info:
 464                 for key , value in disk_controller_info.iteritems():
 465                     if self.disk_controller.lower() in key.lower():
 466                         disk_controller['controllerName'] = key
 467                         disk_controller['resourceType'] = str(value["ResourceType"])
 468                         resourceSubTypes = value["ResourceSubTypes"] if "ResourceSubTypes" in value else None
 469                         if key == "SATA Controller":
 470                             disk_controller["resourceSubType"] = resourceSubTypes[0]
 471                         elif key == "SCSI Controller":
 472                             if scsi_subtype:
 473                                 if scsi_subtype.lower() == "paravirtual":
 474                                     scsi_subtype = "VirtualSCSI"
 475                                 for subtype in resourceSubTypes:
 476                                     if scsi_subtype.lower() == subtype.lower():
 477                                         disk_controller["resourceSubType"] = subtype
 478                                         break
 479                                 else:
 480                                     error_msg = "ERROR: Invalid inputs can not "\
 481                                     "find SCSI subtype {}".format(scsi_subtype)
 482                                     self.__raise_exception(error_msg)
 483
 484         except KeyError as exp:
 485             error_msg = "ERROR:Error occurred while getting Disk Controller details : {}".format(exp)
 486             self.logger.error(error_msg)
 487             print(error_msg)
 488
 489         return disk_controller
 490
 491     def __read_yaml_file(self, file_path):
 492         """
 493             Private method to execute command
 494             Args :
 495                 command  : command to execute
 496             Return :
 497                 Dict of yaml data
 498         """
 499         with open(file_path) as data_file:
 500             data = yaml.load(data_file)
 501         return data
 502
 503     def __raise_exception(self, error_msg , exception_type="Generic"):
 504         """
 505             Private method to execute command
 506             Args :
 507                 command  : command to execute
 508             Return :
 509                None
 510         """
 511         if error_msg:
 512             self.logger.debug(error_msg)
 513             print(error_msg)
 514             if exception_type == "Generic":
 515                 raise Exception(error_msg)
 516             elif exception_type == "IO":
 517                 raise Exception(error_msg)
 518
 519     def __execute_command(self, command):
 520         """
 521             Private method to execute command
 522             Args :
 523                 command  : command to execute
 524             Return :
 525                 stdout : output of command
 526                 stderr: error occurred while executing command if any
 527                 returncode : return code of command execution
 528         """
 529         try:
 530             self.logger.info("Execute command: {} ".format(command))
 531
 532             proc = subprocess.Popen(command , stdout = subprocess.PIPE, stdin = subprocess.PIPE,
 533                                              stderr = subprocess.PIPE,shell=True)
 534             stdout, stderr = proc.communicate()
 535             returncode = proc.returncode
 536
 537         except Exception as exp:
 538             self.logger.error("Error {} occurred while executing command {} ".format(exp,command))
 539
 540         return  stdout, stderr , returncode
 541
 542
 543     def create_ovf(self):
 544         """
 545             Method to convert source image into OVF
 546             Args : None
 547             Return : True on success else False
 548         """
 549         #check output format
 550         if self.source_format == self.output_format:
 551             self.logger.info("Source format is OVF. No need to convert: {} ")
 552             return self.source_img_path
 553
 554         #Get source img properties
 555         img_info = self.__get_image_info()
 556         if img_info:
 557
 558             #Create vmdk disk image
 559             disk_img = self.__convert_image()
 560             if disk_img:
 561
 562                 #Edit OVF tempalte
 563                 ovf_path = self.__edit_ovf_template()
 564                 return ovf_path
 565         else:
 566             self.logger.error("Error in getting image information cannot convert image")
 567             raise Exception("Error in getting image information cannot convert image")
 568             return False
 569