tools/OVF_converter/uploader.py

   1 # #
   2 # Copyright 2019 VMware Inc.
   3 # This file is part of ETSI OSM
   4 # All Rights Reserved.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 #
  18 # For those usages not covered by the Apache License, Version 2.0 please
  19 # contact:  osslegalrouting@vmware.com
  20 # #
  21
  22 import logging
  23 from lxml import etree
  24 import os
  25 from pyvcloud.vcd.client import BasicLoginCredentials, Client, QueryResultFormat, ResourceType, TaskStatus
  26 from pyvcloud.vcd.exceptions import EntityNotFoundException, InternalServerException
  27 from pyvcloud.vcd.org import Org
  28 import sys
  29 import tarfile
  30 import time
  31
  32 MODULE_DIR = os.path.dirname(__file__)
  33
  34 # Set logger
  35 LOG_FILE = os.path.join(MODULE_DIR, "logs/ovf_uploader.log")
  36 os.makedirs(os.path.dirname(LOG_FILE), exist_ok=True)
  37 logger = logging.getLogger(__name__)
  38 file_handler = logging.FileHandler(LOG_FILE)
  39 file_handler.setLevel(logging.DEBUG)
  40 formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
  41 file_handler.setFormatter(formatter)
  42 logger.addHandler(file_handler)
  43 stdout_handler = logging.StreamHandler(sys.stdout)
  44 stdout_handler.setLevel(logging.INFO)
  45 logger.addHandler(stdout_handler)
  46 logger.setLevel(10)
  47 logging.captureWarnings(True)
  48
  49 __version__ = "1.0"
  50 __description__ = "Initial Release"
  51
  52
  53 def get_version():
  54     """ get version of this application"""
  55     version = str(__version__) + " - " + str(__description__)
  56     return version
  57
  58
  59 def report_progress(bytes_written, total_size):
  60     percent_complete = int((bytes_written * 100) / total_size)
  61     print("{}% complete  \r".format(percent_complete), end='')
  62
  63
  64 class OVFUploader(object):
  65     """ Class to convert input image into OVF format """
  66
  67     def __init__(self, ovf_file, vcd_url=None, username=None, password=None, orgname=None):
  68         self.ovf_file = os.path.abspath(ovf_file)
  69         self.vcd_url = vcd_url
  70         self.username = username
  71         self.password = password
  72         self.orgname = orgname
  73         try:
  74             client = Client(self.vcd_url, verify_ssl_certs=False)
  75             client.set_highest_supported_version()
  76             client.set_credentials(BasicLoginCredentials(self.username, self.orgname,
  77                                                          self.password))
  78             logger.info("Logged into {} using version {}".format(self.vcd_url, client.get_api_version()))
  79             self.client = client
  80             self.org = Org(self.client, resource=self.client.get_org())
  81
  82         except Exception as exp:
  83             problem = Exception("Failed to connect to vCD at {}, org {}, username {}:\n{}".format(
  84                 self.vcd_url, self.orgname, self.username, exp))
  85             logger.error(problem)
  86             raise problem
  87
  88         try:
  89             # Retrieve the VM name from the OVF.  We will use this as both the image and catalog name
  90             OVF_tree = etree.parse(self.ovf_file)
  91             root = OVF_tree.getroot()
  92             nsmap = {k: v for k, v in root.nsmap.items() if k}
  93             nsmap["xmlns"] = "http://schemas.dmtf.org/ovf/envelope/1"
  94
  95             virtuasystem = root.find('xmlns:VirtualSystem', nsmap)
  96             name_tag = virtuasystem.find('xmlns:Name', nsmap)
  97             self.image_name = name_tag.text
  98             info_tag = virtuasystem.find('xmlns:Info', nsmap)
  99             self.image_description = info_tag.text
 100
 101             references = root.find('xmlns:References', nsmap)
 102             file = references.find('xmlns:File', nsmap)
 103             self.vmdk_file = "{}/{}".format(
 104                 os.path.dirname(self.ovf_file),
 105                 file.attrib['{http://schemas.dmtf.org/ovf/envelope/1}href'])
 106             logger.info("Loaded VM {}: {}".format(self.image_name, self.image_description))
 107
 108         except Exception as exp:
 109             problem = Exception("Failed to fetch VirtualSystem Name element from OVF {}:\n{}".format(
 110                 self.ovf_file, exp))
 111             logger.error(problem)
 112             raise problem
 113
 114     def make_catalog(self):
 115         try:
 116             try:
 117                 catalog = self.org.get_catalog(self.image_name)
 118                 self.catalog_id = catalog.attrib['id'].split(':')[-1]
 119             except EntityNotFoundException:
 120                 logger.info("Creating a new catalog entry {} in vCD".format(self.image_name))
 121                 result = self.org.create_catalog(self.image_name, self.image_description)
 122                 if result is None:
 123                     raise Exception("Failed to create new catalog entry")
 124                 self.catalog_id = result.attrib['id'].split(':')[-1]
 125                 self.org.reload()
 126
 127             logger.debug("Using catalog {}, id {}".format(self.image_name, self.catalog_id))
 128
 129         except Exception as exp:
 130             problem = Exception("Failed to fetch catalog for {}:\n{} ".format(self.image_name, exp))
 131             logger.error(problem)
 132             raise problem
 133
 134     def upload_ovf(self):
 135
 136         try:
 137             # Check if the content already exists:
 138             items = self.org.list_catalog_items(self.image_name)
 139             for item in items:
 140                 if item['name'] == self.image_name:
 141                     logger.info("Removing old version from catalog")
 142                     try:
 143                         self.org.delete_catalog_item(self.image_name, self.image_name)
 144                     except InternalServerException as exp:
 145                         problem = Exception(
 146                             "Cannot delete vAppTemplate {}. Please check in vCD if "
 147                             "the content is still being imported into the catalog".format(
 148                                 self.image_name))
 149                         raise problem
 150
 151             # Create a single OVA bundle
 152             ova_tarfilename, _ = os.path.splitext(self.ovf_file)
 153             ova_tarfilename += '.ova'
 154             ova = tarfile.open(name=ova_tarfilename,
 155                                mode='w')
 156             ova.add(self.ovf_file, arcname=os.path.basename(self.ovf_file))
 157             ova.add(self.vmdk_file, arcname=os.path.basename(self.vmdk_file))
 158             ova.close()
 159             logger.info("Uploading content to vCD")
 160             self.org.upload_ovf(self.image_name,
 161                                 ova_tarfilename,
 162                                 item_name=self.image_name,
 163                                 description=self.image_description,
 164                                 callback=report_progress)
 165         except Exception as exp:
 166             problem = Exception("Failed to upload OVF {}:\n{} ".format(self.ovf_file, exp))
 167             logger.error(problem)
 168             raise problem
 169         finally:
 170             if os.path.exists(ova_tarfilename):
 171                 os.remove(ova_tarfilename)
 172
 173     def wait_for_task_completion(self):
 174
 175         logger.info("Importing content to vCD")
 176         try:
 177
 178             query = self.client.get_typed_query(
 179                 query_type_name=ResourceType.TASK.value,
 180                 qfilter='ownerName==' + self.username + ';(status==queued,status==preRunning,status==running)',
 181                 query_result_format=QueryResultFormat.REFERENCES)
 182
 183             upload_task = None
 184             tasks = list(query.execute())
 185             for task in tasks:
 186                 if task.get('name') == 'VDC_UPLOAD_OVF_CONTENTS':
 187                     upload_task = self.client.get_resource(task.get('href'))
 188                     break
 189
 190             bad_statuses = [
 191                 TaskStatus.ABORTED,
 192                 TaskStatus.CANCELED,
 193                 TaskStatus.ERROR
 194             ]
 195
 196         except Exception as exp:
 197             problem = Exception("Failed to import OVF {}:\n{} ".format(self.ovf_file, exp))
 198             logger.error(problem)
 199             raise problem
 200
 201         while(True):
 202             task_status = upload_task.get('status').lower()
 203             if(hasattr(upload_task, 'Progress')):
 204                 print("{}% complete  \r".format(upload_task.Progress), end='')
 205
 206             for status in bad_statuses:
 207                 if task_status == status.value.lower():
 208                     problem = Exception(
 209                         "vCD failed to import OVF {}:\n{}: {} ".format(self.ovf_file,
 210                                                                        task_status,
 211                                                                        upload_task.Error.get('Message')))
 212                     logger.error(problem)
 213                     raise problem
 214             if task_status == str(TaskStatus.SUCCESS.value).lower():
 215                 break
 216
 217             time.sleep(2)
 218             upload_task = self.client.get_resource(upload_task.get('href'))
 219
 220         logger.info("OVF upload and import complete, content is ready to use")