From 12aa2cf9b6f165943bf1411f4da9c5c69b9ff904 Mon Sep 17 00:00:00 2001 From: kasar Date: Fri, 2 Feb 2018 02:51:07 -0800 Subject: [PATCH] Makefile code changes for cirros test for VMware vCD connector Change-Id: If04f5d78de0435405403344d9ab17710cb355f6d Signed-off-by: kasar --- systest/Makefile | 46 ++++- systest/lib/vim/fixtures.py | 4 +- systest/lib/vim/vim.py | 2 +- systest/lib/vmware/fixtures.py | 5 +- systest/testcases/vnfs/test_vnfs.py | 16 +- tools/vmware_ovf_upload.py | 264 ++++++++++++++++++++++++++++ 6 files changed, 321 insertions(+), 16 deletions(-) create mode 100755 tools/vmware_ovf_upload.py diff --git a/systest/Makefile b/systest/Makefile index 929896f3..36ed0a57 100644 --- a/systest/Makefile +++ b/systest/Makefile @@ -29,6 +29,11 @@ OS_USERNAME ?= OS_PASSWORD_NAME ?= OS_PROJECT_NAME ?= VIM_CONFIG ?= +VCD_AUTH_URL ?= +VCD_USERNAME ?= +VCD_PASSWORD ?= +VCD_TENANT_NAME ?= +VCD_ORGANIZATION ?= TOPDIR=$(shell readlink -f .|sed -e 's/systest.*//') @@ -59,9 +64,25 @@ endif ifdef VIM_CONFIG OPTION_VIM_CONFIG=--vim-config "$(VIM_CONFIG)" -else endif +ifdef VCD_AUTH_URL + OPTION_VCD_AUTH_URL=--vcd-url $(VCD_AUTH_URL) +endif +ifdef VCD_USERNAME + OPTION_VCD_USERNAME=--vcd-username $(VCD_USERNAME) +endif +ifdef VCD_PASSWORD + OPTION_VCD_PASSWORD=--vcd-password $(VCD_PASSWORD) +endif +ifdef VCD_TENANT_NAME + OPTION_VCD_TENANT_NAME=--vcd-tenant-name $(VCD_TENANT_NAME) +endif +ifdef VCD_ORGANIZATION + OPTION_VCD_ORGANIZATION=--vcd-org $(VCD_ORGANIZATION) +endif + + DESCRIPTOR_DIR ?= $(TOPDIR)/descriptor-packages #TODO: Need to re-add this once charm application name length issue is resolved @@ -94,7 +115,14 @@ check_openstack_env: $(call check_env_var,OS_PASSWORD) $(call check_env_var,OS_PROJECT_NAME) -.PHONY: check_openstack_env check_OSM_HOSTNAME +check_vcd_env: + $(call check_env_var,VCD_AUTH_URL) + $(call check_env_var,VCD_USERNAME) + $(call check_env_var,VCD_PASSWORD) + $(call check_env_var,VCD_TENANT_NAME) + $(call check_env_var,VCD_ORGANIZATION) + +.PHONY: check_openstack_env check_vcd_env check_OSM_HOSTNAME report_dir: @mkdir -p reports @@ -108,6 +136,10 @@ _run_test: report_dir $(OPTION_OS_PASSWORD) \ $(OPTION_VIM_CONFIG) \ $(OPTION_OS_PROJECT_NAME) \ + $(OPTION_VCD_AUTH_URL) \ + $(OPTION_VCD_USERNAME) \ + $(OPTION_VCD_PASSWORD) \ + $(OPTION_VCD_TENANT_NAME) \ $(OPTION_TEST_VNFD_DESCRIPTORS) \ $(OPTION_TEST_NSD_DESCRIPTORS) \ $(OPTION_DESCRIPTOR_BUILD_DIR) \ @@ -135,17 +167,23 @@ images/cache/Fedora-x86_64-20-20131211.1-sda-pong.qcow2: # images are prefixed with 'osm/' to separate osm uploaded images from VIM installed images OSM_IMAGE_PREFIX ?= +sys_path ?= $(TOPDIR)systest ifdef OS_AUTH_URL images/%.qcow2 images/%.img: $(Q)openstack image show $(OSM_IMAGE_PREFIX)$(shell basename $@) || \ sh -c "make images/cache/$(shell basename $@) && openstack image create --file images/cache/$(shell basename $@) $(OSM_IMAGE_PREFIX)$(shell basename $@)" +endif +ifdef VCD_AUTH_URL +images/%.img: + ovf_converter images/cache/$(OSM_IMAGE_PREFIX)$(shell basename $@) -n cirros + python $(TOPDIR)tools/vmware_ovf_upload.py $(VCD_AUTH_URL) $(VCD_USERNAME) $(VCD_PASSWORD) $(VCD_ORGANIZATION) $(sys_path)/images/cache/cirros.ovf else images/%.img: echo "No method selected to upload image to VIM" endif -cirros: check_OSM_HOSTNAME check_openstack_env \ +cirros: check_OSM_HOSTNAME check_openstack_env check_vcd_env \ $(DESCRIPTOR_DIR)/vnfd/cirros_vnf/build/cirros_vnf.tar.gz \ $(DESCRIPTOR_DIR)/nsd/cirros_ns/build/cirros_ns.tar.gz \ images/cirros-0.3.5-x86_64-disk.img @@ -155,7 +193,7 @@ cirros: check_OSM_HOSTNAME check_openstack_env \ JUNITXML=pytest-$@.xml \ PYTEST_OPTIONS="$(PYTEST_OPTIONS) -m vnf" _run_test -ns_scale: check_OSM_HOSTNAME check_openstack_env \ +ns_scale: check_OSM_HOSTNAME check_openstack_env check_vcd_env \ $(DESCRIPTOR_DIR)/vnfd/cirros_vnf/build/cirros_vnf.tar.gz \ $(DESCRIPTOR_DIR)/nsd/cirros_ns/build/cirros_ns.tar.gz \ images/cirros-0.3.5-x86_64-disk.img diff --git a/systest/lib/vim/fixtures.py b/systest/lib/vim/fixtures.py index e552bbc8..6b4404d0 100644 --- a/systest/lib/vim/fixtures.py +++ b/systest/lib/vim/fixtures.py @@ -22,6 +22,6 @@ def vim_add_options(parser): pass @pytest.fixture -def vim(request,osm,openstack): +def vim(request,osm,openstack,vmware): from lib.vim import vim - return vim.Vim(osm,openstack) + return vim.Vim(osm,openstack,vmware) diff --git a/systest/lib/vim/vim.py b/systest/lib/vim/vim.py index 98bda387..f3d17539 100644 --- a/systest/lib/vim/vim.py +++ b/systest/lib/vim/vim.py @@ -18,7 +18,7 @@ from osmclient.common.exceptions import ClientException class Vim(): - def __init__(self,osm,openstack): + def __init__(self,osm,openstack,vmware): self.vim_name='pytest' try: osm.get_api().vim.get(self.vim_name) diff --git a/systest/lib/vmware/fixtures.py b/systest/lib/vmware/fixtures.py index 78ee80c3..cb6cc7b5 100644 --- a/systest/lib/vmware/fixtures.py +++ b/systest/lib/vmware/fixtures.py @@ -30,7 +30,7 @@ def vmware_add_options(parser): parser.addoption("--vcd-username", default="", help="VMware vCloud username") parser.addoption("--vcd-password", default="", help="VMware vCloud password") parser.addoption("--vcd-tenant-name", default="", help="VMware vCloud tenant name") - parser.addoption("--config", default="", help="VMware vCloud config paramters") + parser.addoption("--vcd-org", default="", help="VMware vCloud Organization name") @pytest.fixture def vmware(request): @@ -40,7 +40,8 @@ def vmware(request): access['vim-username'] = request.config.getoption("--vcd-username") access['vim-password'] = request.config.getoption("--vcd-password") access['vim-tenant-name'] = request.config.getoption("--vcd-tenant-name") - access['config'] = request.config.getoption("--config") + access['vcd-org'] = request.config.getoption("--vcd-org") + access['config'] = request.config.getoption("--vim-config") access['vim-type'] = 'vmware' access['description'] = 'pytest system test' diff --git a/systest/testcases/vnfs/test_vnfs.py b/systest/testcases/vnfs/test_vnfs.py index d987e931..79d9ae2a 100644 --- a/systest/testcases/vnfs/test_vnfs.py +++ b/systest/testcases/vnfs/test_vnfs.py @@ -78,7 +78,7 @@ class TestClass(object): # another way to check if the nsd is really ready via API? time.sleep(5) - def vnf_test(self,osm, openstack, vim, vnfd_file_list, nsd_file_list, ns_scale=False): + def vnf_test(self,osm, openstack, vim, vmware, vnfd_file_list, nsd_file_list, ns_scale=False): for file in nsd_file_list: nsd_desc = osm.get_api().package.get_key_val_from_pkg(file) @@ -89,7 +89,7 @@ class TestClass(object): assert utils.wait_for_value(lambda: osm.get_api().ns.get_field(ns_name,'operational-status'),result='vnf-init-phase') # make sure ns is running - assert utils.wait_for_value(lambda: osm.get_api().ns.get_field(ns_name,'operational-status'),result='running',wait_time=120) + assert utils.wait_for_value(lambda: osm.get_api().ns.get_field(ns_name,'operational-status'),result='running',wait_time=300) if ns_scale: # for each descriptor, scale it @@ -101,7 +101,7 @@ class TestClass(object): assert utils.wait_for_value(lambda: osm.get_api().ns.get_field(ns_name,'operational-status'),result='scaling-out',wait_time=120) # wait for ns to be in running-state - assert utils.wait_for_value(lambda: osm.get_api().ns.get_field(ns_name,'operational-status'),result='running',wait_time=120) + assert utils.wait_for_value(lambda: osm.get_api().ns.get_field(ns_name,'operational-status'),result='running',wait_time=300) time.sleep(10) @@ -115,18 +115,20 @@ class TestClass(object): @pytest.mark.openstack @pytest.mark.vnf - def test_vnf(self,osm, vim, openstack, cleanup_test_vnf): + @pytest.mark.vmware + def test_vnf(self,osm, vim, openstack, vmware, cleanup_test_vnf): vnfd_file_list = osm.vnfd_descriptors_list nsd_file_list = osm.nsd_descriptors_list self.vnf_upload_packages(osm, vnfd_file_list, nsd_file_list ) - self.vnf_test(osm,openstack, vim, vnfd_file_list, nsd_file_list) + self.vnf_test(osm,openstack, vim, vmware, vnfd_file_list, nsd_file_list) @pytest.mark.openstack @pytest.mark.ns_scale - def test_scale_vnf(self,osm, vim, openstack, cleanup_test_vnf): + @pytest.mark.vmware + def test_scale_vnf(self,osm, vim, openstack, vmware, cleanup_test_vnf): vnfd_file_list = osm.vnfd_descriptors_list nsd_file_list = osm.nsd_descriptors_list self.vnf_upload_packages(osm, vnfd_file_list, nsd_file_list ) - self.vnf_test(osm,openstack, vim, vnfd_file_list, nsd_file_list, ns_scale=True) + self.vnf_test(osm,openstack, vim, vmware, vnfd_file_list, nsd_file_list, ns_scale=True) diff --git a/tools/vmware_ovf_upload.py b/tools/vmware_ovf_upload.py new file mode 100755 index 00000000..47374153 --- /dev/null +++ b/tools/vmware_ovf_upload.py @@ -0,0 +1,264 @@ +# -*- coding: utf-8 -*- + +## +# Copyright 2016-2017 VMware Inc. +# This file is part of ETSI OSM +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact: osslegalrouting@vmware.com +## + + +from pyvcloud.vcloudair import VCA +from pyvcloud import Http +from xml.etree import ElementTree as XmlElementTree +from pyvcloud.schema.vcd.v1_5.schemas.vcloud import mediaType +import sys,os +import logging +import requests +import time +import re + + +STANDALONE = 'standalone' +VCAVERSION = '5.9' + +class vCloudconfig(object): + def __init__(self, host=None, user=None, password=None,orgname=None, logger=None): + self.url = host + self.user = user + self.password = password + self.org = orgname + self.logger = logger + + def connect(self): + """ Method connect as normal user to vCloud director. + + Returns: + The return vca object that letter can be used to connect to vCloud director as admin for VDC + """ + + try: + self.logger.debug("Logging in to a vca {} as {} to datacenter {}.".format(self.org, + self.user, + self.org)) + vca = VCA(host=self.url, + username=self.user, + service_type=STANDALONE, + version=VCAVERSION, + verify=False, + log=False) + + result = vca.login(password=self.password, org=self.org) + if not result: + raise vimconn.vimconnConnectionException("Can't connect to a vCloud director as: {}".format(self.user)) + result = vca.login(token=vca.token, org=self.org, org_url=vca.vcloud_session.org_url) + if result is True: + self.logger.info( + "Successfully logged to a vcloud direct org: {} as user: {}".format(self.org, self.user)) + + except: + raise vimconn.vimconnConnectionException("Can't connect to a vCloud director org: " + "{} as user: {}".format(self.org, self.user)) + + return vca + + def upload_ovf(self, catalog_name=None, image_name=None, media_file_name=None, + description='', progress=False, chunk_bytes=128 * 1024): + """ + Uploads a OVF file to a vCloud catalog + + catalog_name: (str): The name of the catalog to upload the media. + media_file_name: (str): The name of the local media file to upload. + return: (bool) True if the media file was successfully uploaded, false otherwise. + """ + vca = self.connect() + + # Creating new catalog in vCD + task = vca.create_catalog(catalog_name, catalog_name) + result = vca.block_until_completed(task) + if not result: + return False + + os.path.isfile(media_file_name) + statinfo = os.stat(media_file_name) + + + # find a catalog entry where we upload OVF. + # create vApp Template and check the status if vCD able to read OVF it will respond with appropirate + # status change. + # if VCD can parse OVF we upload VMDK file + try: + for catalog in vca.get_catalogs(): + if catalog_name != catalog.name: + continue + link = filter(lambda link: link.get_type() == "application/vnd.vmware.vcloud.media+xml" and + link.get_rel() == 'add', catalog.get_Link()) + assert len(link) == 1 + data = """ + {} vApp Template + """.format(catalog_name, catalog_name) + headers = vca.vcloud_session.get_vcloud_headers() + headers['Content-Type'] = 'application/vnd.vmware.vcloud.uploadVAppTemplateParams+xml' + response = Http.post(link[0].get_href(), headers=headers, data=data, verify=vca.verify, logger=self.logger) + if response.status_code == requests.codes.created: + catalogItem = XmlElementTree.fromstring(response.content) + entity = [child for child in catalogItem if + child.get("type") == "application/vnd.vmware.vcloud.vAppTemplate+xml"][0] + href = entity.get('href') + template = href + response = Http.get(href, headers=vca.vcloud_session.get_vcloud_headers(), + verify=vca.verify, logger=self.logger) + + if response.status_code == requests.codes.ok: + media = mediaType.parseString(response.content, True) + link = filter(lambda link: link.get_rel() == 'upload:default', + media.get_Files().get_File()[0].get_Link())[0] + headers = vca.vcloud_session.get_vcloud_headers() + headers['Content-Type'] = 'Content-Type text/xml' + response = Http.put(link.get_href(), + data=open(media_file_name, 'rb'), + headers=headers, + verify=vca.verify, logger=self.logger) + if response.status_code != requests.codes.ok: + self.logger.debug( + "Failed create vApp template for catalog name {} and image {}".format(catalog_name, + media_file_name)) + return False + + # TODO fix this with aync block + time.sleep(5) + + self.logger.debug("vApp template for catalog name {} and image {}".format(catalog_name, media_file_name)) + + # uploading VMDK file + # check status of OVF upload and upload remaining files. + response = Http.get(template, + headers=vca.vcloud_session.get_vcloud_headers(), + verify=vca.verify, + logger=self.logger) + + if response.status_code == requests.codes.ok: + media = mediaType.parseString(response.content, True) + number_of_files = len(media.get_Files().get_File()) + for index in xrange(0, number_of_files): + links_list = filter(lambda link: link.get_rel() == 'upload:default', + media.get_Files().get_File()[index].get_Link()) + for link in links_list: + # we skip ovf since it already uploaded. + if 'ovf' in link.get_href(): + continue + # The OVF file and VMDK must be in a same directory + head, tail = os.path.split(media_file_name) + file_vmdk = head + '/' + link.get_href().split("/")[-1] + if not os.path.isfile(file_vmdk): + return False + statinfo = os.stat(file_vmdk) + if statinfo.st_size == 0: + return False + hrefvmdk = link.get_href() + + if progress: + print("Uploading file: {}".format(file_vmdk)) + if progress: + widgets = ['Uploading file: ', Percentage(), ' ', Bar(), ' ', ETA(), ' ', + FileTransferSpeed()] + progress_bar = ProgressBar(widgets=widgets, maxval=statinfo.st_size).start() + + bytes_transferred = 0 + f = open(file_vmdk, 'rb') + while bytes_transferred < statinfo.st_size: + my_bytes = f.read(chunk_bytes) + if len(my_bytes) <= chunk_bytes: + headers = vca.vcloud_session.get_vcloud_headers() + headers['Content-Range'] = 'bytes %s-%s/%s' % ( + bytes_transferred, len(my_bytes) - 1, statinfo.st_size) + headers['Content-Length'] = str(len(my_bytes)) + response = Http.put(hrefvmdk, + headers=headers, + data=my_bytes, + verify=vca.verify, + logger=None) + + if response.status_code == requests.codes.ok: + bytes_transferred += len(my_bytes) + if progress: + progress_bar.update(bytes_transferred) + else: + self.logger.debug( + 'file upload failed with error: [%s] %s' % (response.status_code, + response.content)) + + f.close() + return False + f.close() + if progress: + progress_bar.finish() + time.sleep(15) + self.logger.debug("OVF image sucessfully uploaded to the VMware vCloud Director") + return True + else: + self.logger.debug("Failed retrieve vApp template for catalog name {} for OVF {}". + format(catalog_name, media_file_name)) + return False + except Exception as exp: + self.logger.debug("Failed while uploading OVF to catalog {} for OVF file {} with Exception {}" + .format(catalog_name,media_file_name, exp)) + raise Exception("Failed while uploading OVF to catalog {} for OVF file {} with Exception {}" \ + .format(catalog_name,media_file_name, exp)) + + +if __name__ == "__main__": + + # vmware vcloud director credentials + vcd_hostname = sys.argv[1] + vcd_username = sys.argv[2] + vcd_password = sys.argv[3] + orgname = sys.argv[4] + # OVF image path to be upload to vCD + ovf_file_path = sys.argv[5] + + # changing virtual system type in ovf file + fh = open(ovf_file_path,'r') + content = fh.read() + content = content.replace('vmx-7','vmx-07') + fh.close() + fh1 = open(ovf_file_path,'w') + fh1.write(content) + fh1.close() + + + logging.basicConfig(filename='ovf_upload.log',level=logging.DEBUG) + logger = logging.getLogger(__name__) + + obj = vCloudconfig(vcd_hostname, vcd_username, vcd_password, orgname, logger) + + dirpath, filename = os.path.split(ovf_file_path) + filename_name, file_extension = os.path.splitext(filename) + + # Get image name from cirros vnfd + cirros_yaml = '../descriptor-packages/vnfd/cirros_vnf/src/cirros_vnfd.yaml' + rh = open(cirros_yaml,'r') + match = re.search("image:\s'(.*?)'\n",rh.read()) + if match: catalog = match.group(1) + + + if file_extension == '.ovf': + result = obj.upload_ovf(catalog_name=catalog, image_name='linux', + media_file_name=ovf_file_path, + description='', progress=False, + chunk_bytes=128 * 1024) + -- 2.25.1