Makefile code changes for cirros test for VMware vCD connector

Change-Id: If04f5d78de0435405403344d9ab17710cb355f6d
Signed-off-by: kasar <pkasar@vmware.com>

systest/Makefile

@@ -29,6 +29,11 @@ OS_USERNAME ?=
 OS_PASSWORD_NAME ?=
 OS_PROJECT_NAME ?=
 VIM_CONFIG ?=
+VCD_AUTH_URL ?=
+VCD_USERNAME ?=
+VCD_PASSWORD ?=
+VCD_TENANT_NAME ?=
+VCD_ORGANIZATION ?=
 
 TOPDIR=$(shell readlink -f .|sed -e 's/systest.*//')
 
@@ -59,9 +64,25 @@ endif
 
 ifdef VIM_CONFIG
     OPTION_VIM_CONFIG=--vim-config "$(VIM_CONFIG)"
-else
 endif
 
+ifdef VCD_AUTH_URL
+    OPTION_VCD_AUTH_URL=--vcd-url $(VCD_AUTH_URL)
+endif
+ifdef VCD_USERNAME
+    OPTION_VCD_USERNAME=--vcd-username $(VCD_USERNAME)
+endif
+ifdef VCD_PASSWORD
+    OPTION_VCD_PASSWORD=--vcd-password $(VCD_PASSWORD)
+endif
+ifdef VCD_TENANT_NAME
+    OPTION_VCD_TENANT_NAME=--vcd-tenant-name $(VCD_TENANT_NAME)
+endif
+ifdef VCD_ORGANIZATION
+    OPTION_VCD_ORGANIZATION=--vcd-org $(VCD_ORGANIZATION)
+endif
+
+
 DESCRIPTOR_DIR ?= $(TOPDIR)/descriptor-packages
 
 #TODO: Need to re-add this once charm application name length issue is resolved
@@ -94,7 +115,14 @@ check_openstack_env:
 	$(call check_env_var,OS_PASSWORD)
 	$(call check_env_var,OS_PROJECT_NAME)
 
-.PHONY: check_openstack_env check_OSM_HOSTNAME
+check_vcd_env:
+	$(call check_env_var,VCD_AUTH_URL)
+	$(call check_env_var,VCD_USERNAME)
+	$(call check_env_var,VCD_PASSWORD)
+	$(call check_env_var,VCD_TENANT_NAME)
+	$(call check_env_var,VCD_ORGANIZATION)
+
+.PHONY: check_openstack_env check_vcd_env check_OSM_HOSTNAME
 
 report_dir:
 	@mkdir -p reports
@@ -108,6 +136,10 @@ _run_test: report_dir
         $(OPTION_OS_PASSWORD) \
         $(OPTION_VIM_CONFIG) \
         $(OPTION_OS_PROJECT_NAME) \
+        $(OPTION_VCD_AUTH_URL) \
+        $(OPTION_VCD_USERNAME) \
+        $(OPTION_VCD_PASSWORD) \
+        $(OPTION_VCD_TENANT_NAME) \
         $(OPTION_TEST_VNFD_DESCRIPTORS) \
         $(OPTION_TEST_NSD_DESCRIPTORS) \
         $(OPTION_DESCRIPTOR_BUILD_DIR) \
@@ -135,17 +167,23 @@ images/cache/Fedora-x86_64-20-20131211.1-sda-pong.qcow2:
 
 # images are prefixed with 'osm/' to separate osm uploaded images from VIM installed images
 OSM_IMAGE_PREFIX ?=
+sys_path ?= $(TOPDIR)systest
 
 ifdef OS_AUTH_URL
 images/%.qcow2 images/%.img:
 	$(Q)openstack image show $(OSM_IMAGE_PREFIX)$(shell basename $@) || \
           sh -c "make images/cache/$(shell basename $@) && openstack image create --file images/cache/$(shell basename $@) $(OSM_IMAGE_PREFIX)$(shell basename $@)"
+endif
+ifdef VCD_AUTH_URL
+images/%.img:
+	ovf_converter images/cache/$(OSM_IMAGE_PREFIX)$(shell basename $@) -n cirros
+	python $(TOPDIR)tools/vmware_ovf_upload.py $(VCD_AUTH_URL) $(VCD_USERNAME) $(VCD_PASSWORD) $(VCD_ORGANIZATION) $(sys_path)/images/cache/cirros.ovf
 else
 images/%.img:
 	echo "No method selected to upload image to VIM"
 endif
 
-cirros: check_OSM_HOSTNAME check_openstack_env \
+cirros: check_OSM_HOSTNAME check_openstack_env check_vcd_env \
         $(DESCRIPTOR_DIR)/vnfd/cirros_vnf/build/cirros_vnf.tar.gz \
         $(DESCRIPTOR_DIR)/nsd/cirros_ns/build/cirros_ns.tar.gz \
         images/cirros-0.3.5-x86_64-disk.img
@@ -155,7 +193,7 @@ cirros: check_OSM_HOSTNAME check_openstack_env \
         JUNITXML=pytest-$@.xml \
         PYTEST_OPTIONS="$(PYTEST_OPTIONS) -m vnf" _run_test
 
-ns_scale: check_OSM_HOSTNAME check_openstack_env \
+ns_scale: check_OSM_HOSTNAME check_openstack_env check_vcd_env \
         $(DESCRIPTOR_DIR)/vnfd/cirros_vnf/build/cirros_vnf.tar.gz \
         $(DESCRIPTOR_DIR)/nsd/cirros_ns/build/cirros_ns.tar.gz \
         images/cirros-0.3.5-x86_64-disk.img

systest/lib/vim/fixtures.py

@@ -22,6 +22,6 @@ def vim_add_options(parser):
     pass
 
 @pytest.fixture
-def vim(request,osm,openstack):
+def vim(request,osm,openstack,vmware):
     from lib.vim import vim 
-    return vim.Vim(osm,openstack)
+    return vim.Vim(osm,openstack,vmware)

systest/lib/vim/vim.py

@@ -18,7 +18,7 @@ from osmclient.common.exceptions import ClientException
 
 
 class Vim():
-    def __init__(self,osm,openstack):
+    def __init__(self,osm,openstack,vmware):
         self.vim_name='pytest'
         try:
             osm.get_api().vim.get(self.vim_name)

systest/lib/vmware/fixtures.py

@@ -30,7 +30,7 @@ def vmware_add_options(parser):
     parser.addoption("--vcd-username", default="", help="VMware vCloud username")
     parser.addoption("--vcd-password", default="", help="VMware vCloud password")
     parser.addoption("--vcd-tenant-name", default="", help="VMware vCloud tenant name")
-    parser.addoption("--config", default="", help="VMware vCloud config paramters")
+    parser.addoption("--vcd-org", default="", help="VMware vCloud Organization name")
 
 @pytest.fixture
 def vmware(request):
@@ -40,7 +40,8 @@ def vmware(request):
     access['vim-username'] = request.config.getoption("--vcd-username")
     access['vim-password'] = request.config.getoption("--vcd-password")
     access['vim-tenant-name'] = request.config.getoption("--vcd-tenant-name")
-    access['config'] = request.config.getoption("--config")
+    access['vcd-org'] = request.config.getoption("--vcd-org")
+    access['config'] = request.config.getoption("--vim-config")
     access['vim-type'] = 'vmware'
     access['description'] = 'pytest system test'
 

systest/testcases/vnfs/test_vnfs.py

@@ -78,7 +78,7 @@ class TestClass(object):
         # another way to check if the nsd is really ready via API?
         time.sleep(5)
 
-    def vnf_test(self,osm, openstack, vim, vnfd_file_list, nsd_file_list, ns_scale=False):
+    def vnf_test(self,osm, openstack, vim, vmware, vnfd_file_list, nsd_file_list, ns_scale=False):
         for file in nsd_file_list:
             nsd_desc = osm.get_api().package.get_key_val_from_pkg(file)
 
@@ -89,7 +89,7 @@ class TestClass(object):
             assert utils.wait_for_value(lambda: osm.get_api().ns.get_field(ns_name,'operational-status'),result='vnf-init-phase')
 
             # make sure ns is running
-            assert utils.wait_for_value(lambda: osm.get_api().ns.get_field(ns_name,'operational-status'),result='running',wait_time=120)
+            assert utils.wait_for_value(lambda: osm.get_api().ns.get_field(ns_name,'operational-status'),result='running',wait_time=300)
 
             if ns_scale:
                 # for each descriptor, scale it
@@ -101,7 +101,7 @@ class TestClass(object):
                     assert utils.wait_for_value(lambda: osm.get_api().ns.get_field(ns_name,'operational-status'),result='scaling-out',wait_time=120)
 
                     # wait for ns to be in running-state
-                    assert utils.wait_for_value(lambda: osm.get_api().ns.get_field(ns_name,'operational-status'),result='running',wait_time=120)
+                    assert utils.wait_for_value(lambda: osm.get_api().ns.get_field(ns_name,'operational-status'),result='running',wait_time=300)
 
             time.sleep(10)
 
@@ -115,18 +115,20 @@ class TestClass(object):
 
     @pytest.mark.openstack
     @pytest.mark.vnf
-    def test_vnf(self,osm, vim, openstack, cleanup_test_vnf):
+    @pytest.mark.vmware
+    def test_vnf(self,osm, vim, openstack, vmware, cleanup_test_vnf):
         vnfd_file_list = osm.vnfd_descriptors_list
         nsd_file_list = osm.nsd_descriptors_list
 
         self.vnf_upload_packages(osm, vnfd_file_list, nsd_file_list )
-        self.vnf_test(osm,openstack, vim, vnfd_file_list, nsd_file_list)
+        self.vnf_test(osm,openstack, vim, vmware, vnfd_file_list, nsd_file_list)
 
     @pytest.mark.openstack
     @pytest.mark.ns_scale
-    def test_scale_vnf(self,osm, vim, openstack, cleanup_test_vnf):
+    @pytest.mark.vmware
+    def test_scale_vnf(self,osm, vim, openstack, vmware, cleanup_test_vnf):
         vnfd_file_list = osm.vnfd_descriptors_list
         nsd_file_list = osm.nsd_descriptors_list
 
         self.vnf_upload_packages(osm, vnfd_file_list, nsd_file_list )
-        self.vnf_test(osm,openstack, vim, vnfd_file_list, nsd_file_list, ns_scale=True)
+        self.vnf_test(osm,openstack, vim, vmware, vnfd_file_list, nsd_file_list, ns_scale=True)

tools/vmware_ovf_upload.py

+# -*- coding: utf-8 -*-
+
+##
+# Copyright 2016-2017 VMware Inc.
+# This file is part of ETSI OSM
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact:  osslegalrouting@vmware.com
+##
+
+
+from pyvcloud.vcloudair import VCA
+from pyvcloud import Http
+from xml.etree import ElementTree as XmlElementTree
+from pyvcloud.schema.vcd.v1_5.schemas.vcloud import mediaType
+import sys,os
+import logging
+import requests
+import time
+import re
+
+
+STANDALONE = 'standalone'
+VCAVERSION = '5.9'
+
+class vCloudconfig(object):
+    def __init__(self, host=None, user=None, password=None,orgname=None, logger=None):
+        self.url = host
+        self.user = user
+        self.password = password
+        self.org = orgname
+        self.logger = logger
+
+    def connect(self):
+        """ Method connect as normal user to vCloud director.
+
+            Returns:
+                The return vca object that letter can be used to connect to vCloud director as admin for VDC
+        """
+
+        try:
+            self.logger.debug("Logging in to a vca {} as {} to datacenter {}.".format(self.org,
+                                                                                      self.user,
+                                                                                      self.org))
+            vca = VCA(host=self.url,
+                      username=self.user,
+                      service_type=STANDALONE,
+                      version=VCAVERSION,
+                      verify=False,
+                      log=False)
+
+            result = vca.login(password=self.password, org=self.org)
+            if not result:
+                raise vimconn.vimconnConnectionException("Can't connect to a vCloud director as: {}".format(self.user))
+            result = vca.login(token=vca.token, org=self.org, org_url=vca.vcloud_session.org_url)
+            if result is True:
+                self.logger.info(
+                    "Successfully logged to a vcloud direct org: {} as user: {}".format(self.org, self.user))
+
+        except:
+            raise vimconn.vimconnConnectionException("Can't connect to a vCloud director org: "
+                                                     "{} as user: {}".format(self.org, self.user))
+
+        return vca
+
+    def upload_ovf(self, catalog_name=None, image_name=None, media_file_name=None,
+                   description='', progress=False, chunk_bytes=128 * 1024):
+        """
+        Uploads a OVF file to a vCloud catalog
+
+        catalog_name: (str): The name of the catalog to upload the media.
+        media_file_name: (str): The name of the local media file to upload.
+        return: (bool) True if the media file was successfully uploaded, false otherwise.
+        """
+        vca = self.connect()   
+
+        # Creating new catalog in vCD
+        task = vca.create_catalog(catalog_name, catalog_name)
+        result = vca.block_until_completed(task)
+        if not result:
+            return False
+
+        os.path.isfile(media_file_name)
+        statinfo = os.stat(media_file_name)
+
+
+        #  find a catalog entry where we upload OVF.
+        #  create vApp Template and check the status if vCD able to read OVF it will respond with appropirate
+        #  status change.
+        #  if VCD can parse OVF we upload VMDK file
+        try:
+            for catalog in vca.get_catalogs():
+                if catalog_name != catalog.name:
+                    continue
+                link = filter(lambda link: link.get_type() == "application/vnd.vmware.vcloud.media+xml" and
+                                           link.get_rel() == 'add', catalog.get_Link())
+                assert len(link) == 1
+                data = """
+                <UploadVAppTemplateParams name="{}" xmlns="http://www.vmware.com/vcloud/v1.5" xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1"><Description>{} vApp Template</Description></UploadVAppTemplateParams>
+                """.format(catalog_name, catalog_name)
+                headers = vca.vcloud_session.get_vcloud_headers()
+                headers['Content-Type'] = 'application/vnd.vmware.vcloud.uploadVAppTemplateParams+xml'
+                response = Http.post(link[0].get_href(), headers=headers, data=data, verify=vca.verify, logger=self.logger)
+                if response.status_code == requests.codes.created:
+                    catalogItem = XmlElementTree.fromstring(response.content)
+                    entity = [child for child in catalogItem if
+                              child.get("type") == "application/vnd.vmware.vcloud.vAppTemplate+xml"][0]
+                    href = entity.get('href')
+                    template = href
+                    response = Http.get(href, headers=vca.vcloud_session.get_vcloud_headers(),
+                                        verify=vca.verify, logger=self.logger)
+
+                    if response.status_code == requests.codes.ok:
+                        media = mediaType.parseString(response.content, True)
+                        link = filter(lambda link: link.get_rel() == 'upload:default',
+                                      media.get_Files().get_File()[0].get_Link())[0]
+                        headers = vca.vcloud_session.get_vcloud_headers()
+                        headers['Content-Type'] = 'Content-Type text/xml'
+                        response = Http.put(link.get_href(),
+                                            data=open(media_file_name, 'rb'),
+                                            headers=headers,
+                                            verify=vca.verify, logger=self.logger)
+                        if response.status_code != requests.codes.ok:
+                            self.logger.debug(
+                                "Failed create vApp template for catalog name {} and image {}".format(catalog_name,
+                                                                                                      media_file_name))
+                            return False
+
+                    # TODO fix this with aync block
+                    time.sleep(5)
+
+                    self.logger.debug("vApp template for catalog name {} and image {}".format(catalog_name, media_file_name))
+
+                    # uploading VMDK file
+                    # check status of OVF upload and upload remaining files.
+                    response = Http.get(template,
+                                        headers=vca.vcloud_session.get_vcloud_headers(),
+                                        verify=vca.verify,
+                                        logger=self.logger)
+
+                    if response.status_code == requests.codes.ok:
+                        media = mediaType.parseString(response.content, True)
+                        number_of_files = len(media.get_Files().get_File())
+                        for index in xrange(0, number_of_files):
+                            links_list = filter(lambda link: link.get_rel() == 'upload:default',
+                                                media.get_Files().get_File()[index].get_Link())
+                            for link in links_list:
+                                # we skip ovf since it already uploaded.
+                                if 'ovf' in link.get_href():
+                                    continue
+                                # The OVF file and VMDK must be in a same directory
+                                head, tail = os.path.split(media_file_name)
+                                file_vmdk = head + '/' + link.get_href().split("/")[-1]
+                                if not os.path.isfile(file_vmdk):
+                                    return False
+                                statinfo = os.stat(file_vmdk)
+                                if statinfo.st_size == 0:
+                                    return False
+                                hrefvmdk = link.get_href()
+
+                                if progress:
+                                    print("Uploading file: {}".format(file_vmdk))
+                                if progress:
+                                    widgets = ['Uploading file: ', Percentage(), ' ', Bar(), ' ', ETA(), ' ',
+                                               FileTransferSpeed()]
+                                    progress_bar = ProgressBar(widgets=widgets, maxval=statinfo.st_size).start()
+
+                                bytes_transferred = 0
+                                f = open(file_vmdk, 'rb')
+                                while bytes_transferred < statinfo.st_size:
+                                    my_bytes = f.read(chunk_bytes)
+                                    if len(my_bytes) <= chunk_bytes:
+                                        headers = vca.vcloud_session.get_vcloud_headers()
+                                        headers['Content-Range'] = 'bytes %s-%s/%s' % (
+                                            bytes_transferred, len(my_bytes) - 1, statinfo.st_size)
+                                        headers['Content-Length'] = str(len(my_bytes))
+                                        response = Http.put(hrefvmdk,
+                                                            headers=headers,
+                                                            data=my_bytes,
+                                                            verify=vca.verify,
+                                                            logger=None)
+
+                                        if response.status_code == requests.codes.ok:
+                                            bytes_transferred += len(my_bytes)
+                                            if progress:
+                                                progress_bar.update(bytes_transferred)
+                                        else:
+                                            self.logger.debug(
+                                                'file upload failed with error: [%s] %s' % (response.status_code,
+                                                                                            response.content))
+
+                                            f.close()
+                                            return False
+                                f.close()
+                                if progress:
+                                    progress_bar.finish()
+                                time.sleep(15)
+                        self.logger.debug("OVF image sucessfully uploaded to the VMware vCloud Director")
+                        return True
+                    else:
+                        self.logger.debug("Failed retrieve vApp template for catalog name {} for OVF {}".
+                                          format(catalog_name, media_file_name))
+                        return False
+        except Exception as exp:
+            self.logger.debug("Failed while uploading OVF to catalog {} for OVF file {} with Exception {}"
+                .format(catalog_name,media_file_name, exp))
+            raise Exception("Failed while uploading OVF to catalog {} for OVF file {} with Exception {}" \
+                .format(catalog_name,media_file_name, exp))
+
+
+if __name__ == "__main__":
+
+    # vmware vcloud director credentials
+    vcd_hostname = sys.argv[1]
+    vcd_username = sys.argv[2]
+    vcd_password = sys.argv[3]
+    orgname = sys.argv[4]
+    # OVF image path to be upload to vCD
+    ovf_file_path = sys.argv[5]
+
+    # changing virtual system type in ovf file
+    fh = open(ovf_file_path,'r')
+    content = fh.read()
+    content = content.replace('<vssd:VirtualSystemType>vmx-7','<vssd:VirtualSystemType>vmx-07')
+    fh.close() 
+    fh1 = open(ovf_file_path,'w')
+    fh1.write(content)
+    fh1.close() 
+     
+
+    logging.basicConfig(filename='ovf_upload.log',level=logging.DEBUG)
+    logger = logging.getLogger(__name__)
+
+    obj = vCloudconfig(vcd_hostname, vcd_username, vcd_password, orgname, logger)
+
+    dirpath, filename = os.path.split(ovf_file_path)
+    filename_name, file_extension = os.path.splitext(filename)
+
+    # Get image name from cirros vnfd
+    cirros_yaml = '../descriptor-packages/vnfd/cirros_vnf/src/cirros_vnfd.yaml'
+    rh = open(cirros_yaml,'r')
+    match = re.search("image:\s'(.*?)'\n",rh.read())
+    if match: catalog = match.group(1)
+
+
+    if file_extension == '.ovf':
+        result = obj.upload_ovf(catalog_name=catalog, image_name='linux',
+                                                media_file_name=ovf_file_path,
+                                               description='', progress=False,
+                                                       chunk_bytes=128 * 1024)
+