From: marchettim Date: Tue, 20 Nov 2018 20:41:07 +0000 (+0100) Subject: Merge "cirros test code changes for VCD" X-Git-Tag: v5.0.0~20 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=95c0173af060c3f475fdac79c4480d6ba38bf39b;hp=1c0180cdcb669271beb919ad414389c692272229;p=osm%2Fdevops.git Merge "cirros test code changes for VCD" --- diff --git a/systest/Makefile b/systest/Makefile index 75ab5563..a5b63039 100644 --- a/systest/Makefile +++ b/systest/Makefile @@ -193,7 +193,7 @@ endif %.openstack: check_openstack_env $(Q)$(MAKE) $* -%s.vcd: check_vcd_env +%.vcd: check_vcd_env $(Q)$(MAKE) $* cirros: check_OSM_HOSTNAME \ diff --git a/systest/lib/vim/vim.py b/systest/lib/vim/vim.py index f3d17539..0ac95254 100644 --- a/systest/lib/vim/vim.py +++ b/systest/lib/vim/vim.py @@ -23,4 +23,7 @@ class Vim(): try: osm.get_api().vim.get(self.vim_name) except ClientException: - osm.get_api().vim.create(self.vim_name,openstack.get_access()) + if vmware._os_access['vim-url'] and vmware._os_access['vim-type'] == 'vmware': + osm.get_api().vim.create(self.vim_name,vmware.get_access()) + else: + osm.get_api().vim.create(self.vim_name,openstack.get_access()) diff --git a/tools/vmware_ovf_upload.py b/tools/vmware_ovf_upload.py index 47374153..4f3f0e7a 100755 --- a/tools/vmware_ovf_upload.py +++ b/tools/vmware_ovf_upload.py @@ -22,19 +22,20 @@ ## -from pyvcloud.vcloudair import VCA -from pyvcloud import Http from xml.etree import ElementTree as XmlElementTree -from pyvcloud.schema.vcd.v1_5.schemas.vcloud import mediaType +from pyvcloud.vcd.client import BasicLoginCredentials,Client +from pyvcloud.vcd.vdc import VDC +from pyvcloud.vcd.org import Org import sys,os import logging import requests import time import re +import hashlib +from progressbar import Percentage, Bar, ETA, FileTransferSpeed, ProgressBar -STANDALONE = 'standalone' -VCAVERSION = '5.9' +API_VERSION = '5.6' class vCloudconfig(object): def __init__(self, host=None, user=None, password=None,orgname=None, logger=None): @@ -42,7 +43,8 @@ class vCloudconfig(object): self.user = user self.password = password self.org = orgname - self.logger = logger + self.logger = logging.getLogger('vmware_ovf_upload') + self.logger.setLevel(10) def connect(self): """ Method connect as normal user to vCloud director. @@ -52,87 +54,164 @@ class vCloudconfig(object): """ try: - self.logger.debug("Logging in to a vca {} as {} to datacenter {}.".format(self.org, - self.user, - self.org)) - vca = VCA(host=self.url, - username=self.user, - service_type=STANDALONE, - version=VCAVERSION, - verify=False, - log=False) - - result = vca.login(password=self.password, org=self.org) - if not result: - raise vimconn.vimconnConnectionException("Can't connect to a vCloud director as: {}".format(self.user)) - result = vca.login(token=vca.token, org=self.org, org_url=vca.vcloud_session.org_url) - if result is True: - self.logger.info( - "Successfully logged to a vcloud direct org: {} as user: {}".format(self.org, self.user)) - + self.logger.debug("Logging in to a vcd {} as user {}".format(self.org, + self.user)) + client = Client(self.url, verify_ssl_certs=False) + client.set_credentials(BasicLoginCredentials(self.user, self.org, self.password)) except: - raise vimconn.vimconnConnectionException("Can't connect to a vCloud director org: " + raise Exception("Can't connect to a vCloud director org: " "{} as user: {}".format(self.org, self.user)) - return vca + return client + + def get_catalog_id_from_path(self, catalog_name=None, path=None, progress=False): + """ + Args + catalog - catalog name to be created + path: - valid path to OVF file. + progress - boolean progress bar show progress bar. + + Return: if image uploaded correct method will provide image catalog UUID. + """ + if not path: + raise Exception("Image path can't be None.") + + if not os.path.isfile(path): + raise Exception("Can't read file. File not found.") + + if not os.access(path, os.R_OK): + raise Exception("Can't read file. Check file permission to read.") + + self.logger.debug("get_catalog_id_from_path() client requesting {} ".format(path)) - def upload_ovf(self, catalog_name=None, image_name=None, media_file_name=None, + dirpath, filename = os.path.split(path) + flname, file_extension = os.path.splitext(path) + if file_extension != '.ovf': + self.logger.debug("Wrong file extension {} connector support only OVF container.".format(file_extension)) + raise Exception("Wrong container. vCloud director supports only OVF.") + + self.logger.debug("File name {} Catalog Name {} file path {} ".format(filename, + catalog_name, + path)) + try: + client = self.connect() + if not client: + raise Exception("Failed to connect vCD") + org = Org(client, resource=client.get_org()) + catalogs = org.list_catalogs() + except Exception as exp: + self.logger.debug("Failed get catalogs() with Exception {} ".format(exp)) + raise Exception("Failed get catalogs() with Exception {} ".format(exp)) + + if len(catalogs) == 0: + self.logger.info("Creating a new catalog entry {} in vcloud director".format(catalog_name)) + result = org.create_catalog(catalog_name, catalog_name) + if result is None: + raise Exception("Failed to create new catalog {} ".format(catalog_name)) + result = self.upload_ovf(org=org, catalog_name=catalog_name, image_name=filename.split(".")[0], + media_file_name=path, description='medial_file_name', progress=progress) + if not result: + raise Exception("Failed to create vApp template for catalog {} ".format(catalog_name)) + return self.get_catalogid(catalog_name, catalogs) + else: + for catalog in catalogs: + # search for existing catalog if we find same name we return ID + if catalog['name'] == catalog_name: + self.logger.debug("Found existing catalog entry for {} " + "catalog id {}".format(catalog_name, + self.get_catalogid(catalog_name, catalogs))) + return self.get_catalogid(catalog_name, catalogs) + + # if we didn't find existing catalog we create a new one and upload image. + self.logger.debug("Creating new catalog entry {} - {}".format(catalog_name, catalog_name)) + result = org.create_catalog(catalog_name, catalog_name) + if result is None: + raise Exception("Failed to create new catalog {} ".format(catalog_name)) + + result = self.upload_ovf(org=org, catalog_name=catalog_name, image_name=filename.split(".")[0], + media_file_name=path, description='medial_file_name', progress=progress) + if not result: + raise Exception("Failed create vApp template for catalog {} ".format(catalog_name)) + + def get_catalogid(self, catalog_name=None, catalogs=None): + """ Method check catalog and return catalog ID in UUID format. + + Args + catalog_name: catalog name as string + catalogs: list of catalogs. + + Return: catalogs uuid + """ + + for catalog in catalogs: + if catalog['name'] == catalog_name: + catalog_id = catalog['id'] + return catalog_id + return None + + def upload_ovf(self, org=None, catalog_name=None, image_name=None, media_file_name=None, description='', progress=False, chunk_bytes=128 * 1024): """ Uploads a OVF file to a vCloud catalog + org : organization object catalog_name: (str): The name of the catalog to upload the media. media_file_name: (str): The name of the local media file to upload. return: (bool) True if the media file was successfully uploaded, false otherwise. """ - vca = self.connect() - - # Creating new catalog in vCD - task = vca.create_catalog(catalog_name, catalog_name) - result = vca.block_until_completed(task) - if not result: - return False + client = self.connect() + if not client: + raise Exception("Failed to connect vCD!") os.path.isfile(media_file_name) statinfo = os.stat(media_file_name) - # find a catalog entry where we upload OVF. # create vApp Template and check the status if vCD able to read OVF it will respond with appropirate # status change. # if VCD can parse OVF we upload VMDK file try: - for catalog in vca.get_catalogs(): - if catalog_name != catalog.name: + for catalog in org.list_catalogs(): + if catalog_name != catalog['name']: continue - link = filter(lambda link: link.get_type() == "application/vnd.vmware.vcloud.media+xml" and - link.get_rel() == 'add', catalog.get_Link()) - assert len(link) == 1 + catalog_href = "{}/api/catalog/{}/action/upload".format(self.url, catalog['id']) data = """ {} vApp Template - """.format(catalog_name, catalog_name) - headers = vca.vcloud_session.get_vcloud_headers() - headers['Content-Type'] = 'application/vnd.vmware.vcloud.uploadVAppTemplateParams+xml' - response = Http.post(link[0].get_href(), headers=headers, data=data, verify=vca.verify, logger=self.logger) + """.format(catalog_name, description) + + if client: + headers = {'Accept':'application/*+xml;version=' + API_VERSION, + 'x-vcloud-authorization': client._session.headers['x-vcloud-authorization']} + headers['Content-Type'] = 'application/vnd.vmware.vcloud.uploadVAppTemplateParams+xml' + + response = requests.post(url=catalog_href, + headers=headers, + data=data, + verify=False) + if response.status_code != 201: + self.logger.debug("Failed to create vApp template") + raise Exception("Failed to create vApp template") + if response.status_code == requests.codes.created: catalogItem = XmlElementTree.fromstring(response.content) entity = [child for child in catalogItem if child.get("type") == "application/vnd.vmware.vcloud.vAppTemplate+xml"][0] href = entity.get('href') template = href - response = Http.get(href, headers=vca.vcloud_session.get_vcloud_headers(), - verify=vca.verify, logger=self.logger) + response = requests.get(url=href, + headers=headers, + verify=False) if response.status_code == requests.codes.ok: - media = mediaType.parseString(response.content, True) - link = filter(lambda link: link.get_rel() == 'upload:default', - media.get_Files().get_File()[0].get_Link())[0] - headers = vca.vcloud_session.get_vcloud_headers() headers['Content-Type'] = 'Content-Type text/xml' - response = Http.put(link.get_href(), - data=open(media_file_name, 'rb'), - headers=headers, - verify=vca.verify, logger=self.logger) + result = re.search('rel="upload:default"\shref="(.*?\/descriptor.ovf)"',response.content) + if result: + transfer_href = result.group(1) + + response = requests.put(url=transfer_href, headers=headers, + data=open(media_file_name, 'rb'), + verify=False) + if response.status_code != requests.codes.ok: self.logger.debug( "Failed create vApp template for catalog name {} and image {}".format(catalog_name, @@ -146,80 +225,72 @@ class vCloudconfig(object): # uploading VMDK file # check status of OVF upload and upload remaining files. - response = Http.get(template, - headers=vca.vcloud_session.get_vcloud_headers(), - verify=vca.verify, - logger=self.logger) + + response = requests.get(url=template, + headers=headers, + verify=False) if response.status_code == requests.codes.ok: - media = mediaType.parseString(response.content, True) - number_of_files = len(media.get_Files().get_File()) - for index in xrange(0, number_of_files): - links_list = filter(lambda link: link.get_rel() == 'upload:default', - media.get_Files().get_File()[index].get_Link()) - for link in links_list: - # we skip ovf since it already uploaded. - if 'ovf' in link.get_href(): - continue - # The OVF file and VMDK must be in a same directory - head, tail = os.path.split(media_file_name) - file_vmdk = head + '/' + link.get_href().split("/")[-1] - if not os.path.isfile(file_vmdk): - return False - statinfo = os.stat(file_vmdk) - if statinfo.st_size == 0: + result = re.search('rel="upload:default"\s*href="(.*?vmdk)"',response.content) + if result: + link_href = result.group(1) + # we skip ovf since it already uploaded. + if 'ovf' in link_href: + continue + # The OVF file and VMDK must be in a same directory + head, tail = os.path.split(media_file_name) + file_vmdk = head + '/' + link_href.split("/")[-1] + if not os.path.isfile(file_vmdk): + return False + statinfo = os.stat(file_vmdk) + if statinfo.st_size == 0: + return False + hrefvmdk = link_href + if progress: + widgets = ['Uploading file: ', Percentage(), ' ', Bar(), ' ', ETA(), ' ', + FileTransferSpeed()] + progress_bar = ProgressBar(widgets=widgets, maxval=statinfo.st_size).start() + + bytes_transferred = 0 + f = open(file_vmdk, 'rb') + while bytes_transferred < statinfo.st_size: + my_bytes = f.read(chunk_bytes) + if len(my_bytes) <= chunk_bytes: + headers['Content-Range'] = 'bytes %s-%s/%s' % ( + bytes_transferred, len(my_bytes) - 1, statinfo.st_size) + headers['Content-Length'] = str(len(my_bytes)) + response = requests.put(url=hrefvmdk, + headers=headers, + data=my_bytes, + verify=False) + if response.status_code == requests.codes.ok: + bytes_transferred += len(my_bytes) + if progress: + progress_bar.update(bytes_transferred) + else: + self.logger.debug( + 'file upload failed with error: [%s] %s' % (response.status_code, + response.content)) + + f.close() return False - hrefvmdk = link.get_href() - - if progress: - print("Uploading file: {}".format(file_vmdk)) - if progress: - widgets = ['Uploading file: ', Percentage(), ' ', Bar(), ' ', ETA(), ' ', - FileTransferSpeed()] - progress_bar = ProgressBar(widgets=widgets, maxval=statinfo.st_size).start() - - bytes_transferred = 0 - f = open(file_vmdk, 'rb') - while bytes_transferred < statinfo.st_size: - my_bytes = f.read(chunk_bytes) - if len(my_bytes) <= chunk_bytes: - headers = vca.vcloud_session.get_vcloud_headers() - headers['Content-Range'] = 'bytes %s-%s/%s' % ( - bytes_transferred, len(my_bytes) - 1, statinfo.st_size) - headers['Content-Length'] = str(len(my_bytes)) - response = Http.put(hrefvmdk, - headers=headers, - data=my_bytes, - verify=vca.verify, - logger=None) - - if response.status_code == requests.codes.ok: - bytes_transferred += len(my_bytes) - if progress: - progress_bar.update(bytes_transferred) - else: - self.logger.debug( - 'file upload failed with error: [%s] %s' % (response.status_code, - response.content)) - - f.close() - return False - f.close() - if progress: - progress_bar.finish() - time.sleep(15) - self.logger.debug("OVF image sucessfully uploaded to the VMware vCloud Director") - return True - else: - self.logger.debug("Failed retrieve vApp template for catalog name {} for OVF {}". - format(catalog_name, media_file_name)) - return False + f.close() + if progress: + progress_bar.finish() + time.sleep(60) + return True + else: + self.logger.debug("Failed retrieve vApp template for catalog name {} for OVF {}". + format(catalog_name, media_file_name)) + return False except Exception as exp: self.logger.debug("Failed while uploading OVF to catalog {} for OVF file {} with Exception {}" .format(catalog_name,media_file_name, exp)) - raise Exception("Failed while uploading OVF to catalog {} for OVF file {} with Exception {}" \ - .format(catalog_name,media_file_name, exp)) - + raise Exception( + "Failed while uploading OVF to catalog {} for OVF file {} with Exception {}" + .format(catalog_name,media_file_name, exp)) + self.logger.debug("Failed to retrieve catalog name {} for OVF file {}".format(catalog_name, media_file_name)) + return False if __name__ == "__main__": @@ -255,10 +326,6 @@ if __name__ == "__main__": match = re.search("image:\s'(.*?)'\n",rh.read()) if match: catalog = match.group(1) - if file_extension == '.ovf': - result = obj.upload_ovf(catalog_name=catalog, image_name='linux', - media_file_name=ovf_file_path, - description='', progress=False, - chunk_bytes=128 * 1024) - + obj.get_catalog_id_from_path(catalog_name=catalog, path=ovf_file_path, + progress=True)