tools/OVF_converter/uploader.py

   1 # #
   2 # Copyright 2019 VMware Inc.
   3 # This file is part of ETSI OSM
   4 # All Rights Reserved.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 #
  18 # For those usages not covered by the Apache License, Version 2.0 please
  19 # contact:  osslegalrouting@vmware.com
  20 # #
  21
  22 import logging
  23 from lxml import etree
  24 import os
  25 from pyvcloud.vcd.client import BasicLoginCredentials, Client, QueryResultFormat, ResourceType, TaskStatus, ApiVersion
  26 from pyvcloud.vcd.exceptions import EntityNotFoundException, InternalServerException
  27 from pyvcloud.vcd.org import Org
  28 import sys
  29 import tarfile
  30 import time
  31
  32 MODULE_DIR = os.path.dirname(__file__)
  33
  34 # Set logger
  35 LOG_FILE = os.path.join(MODULE_DIR, "logs/ovf_uploader.log")
  36 os.makedirs(os.path.dirname(LOG_FILE), exist_ok=True)
  37 logger = logging.getLogger(__name__)
  38 file_handler = logging.FileHandler(LOG_FILE)
  39 file_handler.setLevel(logging.DEBUG)
  40 formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
  41 file_handler.setFormatter(formatter)
  42 logger.addHandler(file_handler)
  43 stdout_handler = logging.StreamHandler(sys.stdout)
  44 stdout_handler.setLevel(logging.INFO)
  45 logger.addHandler(stdout_handler)
  46 logger.setLevel(10)
  47 logging.captureWarnings(True)
  48
  49 __version__ = "1.0"
  50 __description__ = "Initial Release"
  51
  52
  53 def get_version():
  54     """ get version of this application"""
  55     version = str(__version__) + " - " + str(__description__)
  56     return version
  57
  58
  59 def report_progress(bytes_written, total_size):
  60     percent_complete = int((bytes_written * 100) / total_size)
  61     print("{}% complete  \r".format(percent_complete), end='')
  62
  63
  64 class OVFUploader(object):
  65     """ Class to convert input image into OVF format """
  66
  67     def __init__(self, ovf_file, vcd_url=None, username=None, password=None, orgname=None):
  68         self.ovf_file = os.path.abspath(ovf_file)
  69         self.vcd_url = vcd_url
  70         self.username = username
  71         self.password = password
  72         self.orgname = orgname
  73         try:
  74             client = Client(self.vcd_url, verify_ssl_certs=False, api_version=ApiVersion.VERSION_32.value,
  75                             log_requests=True,
  76                             log_headers=True,
  77                             log_bodies=True,
  78                             log_file=LOG_FILE)
  79             # sclient.set_highest_supported_version()
  80             client.set_credentials(BasicLoginCredentials(self.username, self.orgname,
  81                                                          self.password))
  82             logger.info("Logged into {} using version {}".format(self.vcd_url, client.get_api_version()))
  83             self.client = client
  84             self.org = Org(self.client, resource=self.client.get_org())
  85
  86         except Exception as exp:
  87             problem = Exception("Failed to connect to vCD at {}, org {}, username {}:\n{}".format(
  88                 self.vcd_url, self.orgname, self.username, exp))
  89             logger.error(problem)
  90             raise problem
  91
  92         try:
  93             # Retrieve the VM name from the OVF.  We will use this as both the image and catalog name
  94             OVF_tree = etree.parse(self.ovf_file)
  95             root = OVF_tree.getroot()
  96             nsmap = {k: v for k, v in root.nsmap.items() if k}
  97             nsmap["xmlns"] = "http://schemas.dmtf.org/ovf/envelope/1"
  98
  99             virtuasystem = root.find('xmlns:VirtualSystem', nsmap)
 100             name_tag = virtuasystem.find('xmlns:Name', nsmap)
 101             self.image_name = name_tag.text
 102             info_tag = virtuasystem.find('xmlns:Info', nsmap)
 103             self.image_description = info_tag.text
 104
 105             references = root.find('xmlns:References', nsmap)
 106             file = references.find('xmlns:File', nsmap)
 107             self.vmdk_file = "{}/{}".format(
 108                 os.path.dirname(self.ovf_file),
 109                 file.attrib['{http://schemas.dmtf.org/ovf/envelope/1}href'])
 110             logger.info("Loaded VM {}: {}".format(self.image_name, self.image_description))
 111
 112         except Exception as exp:
 113             problem = Exception("Failed to fetch VirtualSystem Name element from OVF {}:\n{}".format(
 114                 self.ovf_file, exp))
 115             logger.error(problem)
 116             raise problem
 117
 118     def make_catalog(self):
 119         self.catalog_id = None
 120         try:
 121             for catalog in self.org.list_catalogs():
 122                 if catalog['name'] == self.image_name:
 123                     self.catalog_id = catalog['id']
 124             if self.catalog_id is None:
 125                 logger.info("Creating a new catalog entry {} in vCD".format(self.image_name))
 126                 result = self.org.create_catalog(self.image_name, self.image_description)
 127                 if result is None:
 128                     raise Exception("Failed to create new catalog entry")
 129                 self.catalog_id = result.attrib['id'].split(':')[-1]
 130                 self.org.reload()
 131
 132             logger.debug("Using catalog {}, id {}".format(self.image_name, self.catalog_id))
 133
 134         except Exception as exp:
 135             problem = Exception("Failed to fetch catalog for {}:\n{} ".format(self.image_name, exp))
 136             logger.error(problem)
 137             raise problem
 138
 139     def upload_ovf(self):
 140
 141         ova_tarfilename, _ = os.path.splitext(self.ovf_file)
 142         ova_tarfilename += '.ova'
 143         try:
 144             # Check if the content already exists:
 145             resource_type = ResourceType.CATALOG_ITEM.value
 146             q = self.client.get_typed_query(
 147                 resource_type,
 148                 query_result_format=QueryResultFormat.ID_RECORDS,
 149                 equality_filter=('catalogName', self.image_name))
 150             for item in list(q.execute()):
 151                 if item.get('name') == self.image_name:
 152                     logger.info("Removing old version from catalog")
 153                     try:
 154                         self.org.delete_catalog_item(self.image_name, self.image_name)
 155                     except InternalServerException as exp:
 156                         problem = Exception(
 157                             "Cannot delete vAppTemplate {}. Please check in vCD if "
 158                             "the content is still being imported into the catalog".format(
 159                                 self.image_name))
 160                         raise problem
 161
 162             # Create a single OVA bundle
 163             ova = tarfile.open(name=ova_tarfilename,
 164                                mode='w')
 165             ova.add(self.ovf_file, arcname=os.path.basename(self.ovf_file))
 166             ova.add(self.vmdk_file, arcname=os.path.basename(self.vmdk_file))
 167             ova.close()
 168             logger.info("Uploading content to vCD")
 169             self.org.upload_ovf(self.image_name,
 170                                 ova_tarfilename,
 171                                 item_name=self.image_name,
 172                                 description=self.image_description,
 173                                 callback=report_progress)
 174         except Exception as exp:
 175             problem = Exception("Failed to upload OVF {}:\n{} ".format(self.ovf_file, exp))
 176             logger.error(problem)
 177             raise problem
 178         finally:
 179             if os.path.exists(ova_tarfilename):
 180                 os.remove(ova_tarfilename)
 181
 182     def wait_for_task_completion(self):
 183
 184         logger.info("Importing content to vCD")
 185         try:
 186
 187             query = self.client.get_typed_query(
 188                 query_type_name=ResourceType.TASK.value,
 189                 qfilter='ownerName==' + self.username + ';(status==queued,status==preRunning,status==running)',
 190                 query_result_format=QueryResultFormat.REFERENCES)
 191
 192             upload_task = None
 193             tasks = list(query.execute())
 194             for task in tasks:
 195                 if task.get('name') == 'VDC_UPLOAD_OVF_CONTENTS':
 196                     upload_task = self.client.get_resource(task.get('href'))
 197                     break
 198
 199             bad_statuses = [
 200                 TaskStatus.ABORTED,
 201                 TaskStatus.CANCELED,
 202                 TaskStatus.ERROR
 203             ]
 204
 205         except Exception as exp:
 206             problem = Exception("Failed to import OVF {}:\n{} ".format(self.ovf_file, exp))
 207             logger.error(problem)
 208             raise problem
 209
 210         while(True):
 211             task_status = upload_task.get('status').lower()
 212             if(hasattr(upload_task, 'Progress')):
 213                 print("{}% complete  \r".format(upload_task.Progress), end='')
 214
 215             for status in bad_statuses:
 216                 if task_status == status.value.lower():
 217                     problem = Exception(
 218                         "vCD failed to import OVF {}:\n{}: {} ".format(self.ovf_file,
 219                                                                        task_status,
 220                                                                        upload_task.Error.get('Message')))
 221                     logger.error(problem)
 222                     raise problem
 223             if task_status == str(TaskStatus.SUCCESS.value).lower():
 224                 break
 225
 226             time.sleep(2)
 227             upload_task = self.client.get_resource(upload_task.get('href'))
 228
 229         logger.info("OVF upload and import complete, content is ready to use")