From: Ananda Baitharu Date: Tue, 5 Mar 2019 17:34:31 +0000 (+0000) Subject: VCD VIMconnector: add config drive support; Adding genisoimage package installation... X-Git-Tag: v6.0.0~24 X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2FRO.git;a=commitdiff_plain;h=319c26fb8f4e6516556778df2db57fc572cee0a2 VCD VIMconnector: add config drive support; Adding genisoimage package installation to scripts/install-openmano.sh Change-Id: I2fdf100a8ecc04207d0e977e81eabc46862a8b6d Signed-off-by: Ananda Baitharu --- diff --git a/docker/Dockerfile-local b/docker/Dockerfile-local index 86bc9949..9897c4dc 100644 --- a/docker/Dockerfile-local +++ b/docker/Dockerfile-local @@ -14,6 +14,7 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get -y install python-cffi libmysqlclient-dev libssl-dev libffi-dev python-mysqldb && \ DEBIAN_FRONTEND=noninteractive apt-get -y install python-openstacksdk python-openstackclient && \ DEBIAN_FRONTEND=noninteractive apt-get -y install python-networkx && \ + DEBIAN_FRONTEND=noninteractive apt-get -y install genisoimage && \ DEBIAN_FRONTEND=noninteractive pip2 install untangle && \ DEBIAN_FRONTEND=noninteractive pip2 install -e git+https://github.com/python-oca/python-oca#egg=oca && \ DEBIAN_FRONTEND=noninteractive apt-get -y install mysql-client diff --git a/osm_ro/vimconn_vmware.py b/osm_ro/vimconn_vmware.py index d4b83e54..94bae393 100644 --- a/osm_ro/vimconn_vmware.py +++ b/osm_ro/vimconn_vmware.py @@ -29,6 +29,9 @@ from progressbar import Percentage, Bar, ETA, FileTransferSpeed, ProgressBar import vimconn import os +import shutil +import subprocess +import tempfile import traceback import itertools import requests @@ -1034,8 +1037,7 @@ class vimconnector(vimconn.vimconnector): """ for catalog in catalogs: if catalog['name'] == catalog_name: - return True - return False + return catalog['id'] def create_vimcatalog(self, vca=None, catalog_name=None): """ Create new catalog entry in vCloud director. @@ -1045,16 +1047,19 @@ class vimconnector(vimconn.vimconnector): catalog_name catalog that client wish to create. Note no validation done for a name. Client must make sure that provide valid string representation. - Return (bool) True if catalog created. + Returns catalog id if catalog created else None. """ try: - result = vca.create_catalog(catalog_name, catalog_name) - if result is not None: - return True + lxml_catalog_element = vca.create_catalog(catalog_name, catalog_name) + if lxml_catalog_element: + id_attr_value = lxml_catalog_element.get('id') # 'urn:vcloud:catalog:7490d561-d384-4dac-8229-3575fd1fc7b4' + return id_attr_value.split(':')[-1] catalogs = vca.list_catalogs() - except: - return False + except Exception as ex: + self.logger.error( + 'create_vimcatalog(): Creation of catalog "{}" failed with error: {}'.format(catalog_name, ex)) + raise return self.catalog_exists(catalog_name, catalogs) # noinspection PyIncorrectDocstring @@ -1324,8 +1329,7 @@ class vimconnector(vimconn.vimconnector): if len(catalogs) == 0: self.logger.info("Creating a new catalog entry {} in vcloud director".format(catalog_name)) - result = self.create_vimcatalog(org, catalog_md5_name) - if not result: + if self.create_vimcatalog(org, catalog_md5_name) is None: raise vimconn.vimconnException("Failed create new catalog {} ".format(catalog_md5_name)) result = self.upload_vimimage(vca=org, catalog_name=catalog_md5_name, @@ -1345,8 +1349,7 @@ class vimconnector(vimconn.vimconnector): # if we didn't find existing catalog we create a new one and upload image. self.logger.debug("Creating new catalog entry {} - {}".format(catalog_name, catalog_md5_name)) - result = self.create_vimcatalog(org, catalog_md5_name) - if not result: + if self.create_vimcatalog(org, catalog_md5_name) is None: raise vimconn.vimconnException("Failed create new catalog {} ".format(catalog_md5_name)) result = self.upload_vimimage(vca=org, catalog_name=catalog_md5_name, @@ -1967,7 +1970,32 @@ class vimconnector(vimconn.vimconnector): # cloud-init for ssh-key injection if cloud_config: - self.cloud_init(vapp,cloud_config) + # Create a catalog which will be carrying the config drive ISO + # This catalog is deleted during vApp deletion. The catalog name carries + # vApp UUID and thats how it gets identified during its deletion. + config_drive_catalog_name = 'cfg_drv-' + vapp_uuid + self.logger.info('new_vminstance(): Creating catalog "{}" to carry config drive ISO'.format( + config_drive_catalog_name)) + config_drive_catalog_id = self.create_vimcatalog(org, config_drive_catalog_name) + if config_drive_catalog_id is None: + error_msg = "new_vminstance(): Failed to create new catalog '{}' to carry the config drive " \ + "ISO".format(config_drive_catalog_name) + raise Exception(error_msg) + + # Create config-drive ISO + _, userdata = self._create_user_data(cloud_config) + # self.logger.debug('new_vminstance(): The userdata for cloud-init: {}'.format(userdata)) + iso_path = self.create_config_drive_iso(userdata) + self.logger.debug('new_vminstance(): The ISO is successfully created. Path: {}'.format(iso_path)) + + self.logger.info('new_vminstance(): uploading iso to catalog {}'.format(config_drive_catalog_name)) + self.upload_iso_to_catalog(config_drive_catalog_id, iso_path) + # Attach the config-drive ISO to the VM + self.logger.info('new_vminstance(): Attaching the config-drive ISO to the VM') + # The ISO remains in INVALID_STATE right after the PUT request (its a blocking call though) + time.sleep(5) + self.insert_media_to_vm(vapp, config_drive_catalog_id) + shutil.rmtree(os.path.dirname(iso_path), ignore_errors=True) # If VM has PCI devices or SRIOV reserve memory for VM if reserve_memory: @@ -1984,7 +2012,11 @@ class vimconnector(vimconn.vimconnector): self.logger.error("new_vminstance(): failed to power on vApp "\ "{}".format(vmname_andid)) - except Exception as exp : + except Exception as exp: + try: + self.delete_vminstance(vapp_uuid) + except Exception as exp2: + self.logger.error("new_vminstance rollback fail {}".format(exp2)) # it might be a case if specific mandatory entry in dict is empty or some other pyVcloud exception self.logger.error("new_vminstance(): Failed create new vm instance {} with exception {}" .format(name, exp)) @@ -2104,6 +2136,80 @@ class vimconnector(vimconn.vimconnector): else: raise vimconn.vimconnUnexpectedResponse("new_vminstance(): Failed create new vm instance {}".format(name)) + def create_config_drive_iso(self, user_data): + tmpdir = tempfile.mkdtemp() + iso_path = os.path.join(tmpdir, 'ConfigDrive.iso') + latest_dir = os.path.join(tmpdir, 'openstack', 'latest') + os.makedirs(latest_dir) + with open(os.path.join(latest_dir, 'meta_data.json'), 'w') as meta_file_obj, \ + open(os.path.join(latest_dir, 'user_data'), 'w') as userdata_file_obj: + userdata_file_obj.write(user_data) + meta_file_obj.write(json.dumps({"availability_zone": "nova", + "launch_index": 0, + "name": "ConfigDrive", + "uuid": str(uuid.uuid4())} + ) + ) + genisoimage_cmd = 'genisoimage -J -r -V config-2 -o {iso_path} {source_dir_path}'.format( + iso_path=iso_path, source_dir_path=tmpdir) + self.logger.info('create_config_drive_iso(): Creating ISO by running command "{}"'.format(genisoimage_cmd)) + try: + FNULL = open(os.devnull, 'w') + subprocess.check_call(genisoimage_cmd, shell=True, stdout=FNULL) + except subprocess.CalledProcessError as e: + shutil.rmtree(tmpdir, ignore_errors=True) + error_msg = 'create_config_drive_iso(): Exception while running genisoimage command: {}'.format(e) + self.logger.error(error_msg) + raise Exception(error_msg) + return iso_path + + def upload_iso_to_catalog(self, catalog_id, iso_file_path): + if not os.path.isfile(iso_file_path): + error_msg = "upload_iso_to_catalog(): Given iso file is not present. Given path: {}".format(iso_file_path) + self.logger.error(error_msg) + raise Exception(error_msg) + iso_file_stat = os.stat(iso_file_path) + xml_media_elem = ''' + + ISO image for config-drive + '''.format(iso_name=os.path.basename(iso_file_path), iso_size=iso_file_stat.st_size) + headers = {'Accept':'application/*+xml;version=' + API_VERSION, + 'x-vcloud-authorization': self.client._session.headers['x-vcloud-authorization']} + headers['Content-Type'] = 'application/vnd.vmware.vcloud.media+xml' + catalog_href = self.url + '/api/catalog/' + catalog_id + '/action/upload' + response = self.perform_request(req_type='POST', url=catalog_href, headers=headers, data=xml_media_elem) + + if response.status_code != 201: + error_msg = "upload_iso_to_catalog(): Failed to POST an action/upload request to {}".format(catalog_href) + self.logger.error(error_msg) + raise Exception(error_msg) + + catalogItem = XmlElementTree.fromstring(response.content) + entity = [child for child in catalogItem if child.get("type") == "application/vnd.vmware.vcloud.media+xml"][0] + entity_href = entity.get('href') + + response = self.perform_request(req_type='GET', url=entity_href, headers=headers) + if response.status_code != 200: + raise Exception("upload_iso_to_catalog(): Failed to GET entity href {}".format(entity_href)) + + match = re.search(r'\s+?\s+?\s+?', response.text, re.DOTALL) + if match: + media_upload_href = match.group(1) + else: + raise Exception('Could not parse the upload URL for the media file from the last response') + + headers['Content-Type'] = 'application/octet-stream' + response = self.perform_request(req_type='PUT', + url=media_upload_href, + headers=headers, + data=open(iso_file_path, 'rb')) + + if response.status_code != 200: + raise Exception('PUT request to "{}" failed'.format(media_upload_href)) def get_vcd_availibility_zones(self,respool_href, headers): """ Method to find presence of av zone is VIM resource pool @@ -2713,6 +2819,17 @@ class vimconnector(vimconn.vimconnector): self.logger.debug("delete_vminstance(): Failed delete uuid {} ".format(vm__vim_uuid)) else: self.logger.info("Deleted vm instance {} sccessfully".format(vm__vim_uuid)) + config_drive_catalog_name, config_drive_catalog_id = 'cfg_drv-' + vm__vim_uuid, None + catalog_list = self.get_image_list() + try: + config_drive_catalog_id = [catalog_['id'] for catalog_ in catalog_list + if catalog_['name'] == config_drive_catalog_name][0] + except IndexError: + pass + if config_drive_catalog_id: + self.logger.debug('delete_vminstance(): Found a config drive catalog {} matching ' + 'vapp_name"{}". Deleting it.'.format(config_drive_catalog_id, vapp_name)) + self.delete_image(config_drive_catalog_id) return vm__vim_uuid except: self.logger.debug(traceback.format_exc()) @@ -6212,10 +6329,17 @@ class vimconnector(vimconn.vimconnector): if iso_name and media_id: data =""" + xmlns="http://www.vmware.com/vcloud/versions" xmlns:ns2="http://schemas.dmtf.org/ovf/envelope/1" + xmlns:ns3="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" + xmlns:ns4="http://schemas.dmtf.org/wbem/wscim/1/common" + xmlns:ns5="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData" + xmlns:ns6="http://www.vmware.com/vcloud/v1.5" + xmlns:ns7="http://www.vmware.com/schema/ovf" + xmlns:ns8="http://schemas.dmtf.org/ovf/environment/1" + xmlns:ns9="http://www.vmware.com/vcloud/extension/v1.5"> """.format(iso_name, media_id, @@ -6233,9 +6357,10 @@ class vimconnector(vimconn.vimconnector): headers=headers) if response.status_code != 202: - self.logger.error("Failed to insert CD-ROM to vm") - raise vimconn.vimconnException("insert_media_to_vm() : Failed to insert"\ - "ISO image to vm") + error_msg = "insert_media_to_vm() : Failed to insert CD-ROM to vm. Reason {}. " \ + "Status code {}".format(response.text, response.status_code) + self.logger.error(error_msg) + raise vimconn.vimconnException(error_msg) else: task = self.get_task_from_response(response.content) result = self.client.get_task_monitor().wait_for_success(task=task) diff --git a/requirements.txt b/requirements.txt index 05d96bb2..79cb11f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,5 +18,6 @@ pyvmomi progressbar prettytable boto +genisoimage untangle oca diff --git a/scripts/install-openmano.sh b/scripts/install-openmano.sh index e4554ae7..bbf33e49 100755 --- a/scripts/install-openmano.sh +++ b/scripts/install-openmano.sh @@ -259,6 +259,8 @@ then pip2 install progressbar || exit 1 pip2 install prettytable || exit 1 pip2 install pyvmomi || exit 1 + [ "$_DISTRO" == "Ubuntu" ] && install_packages "genisoimage" + [ "$_DISTRO" == "CentOS" -o "$_DISTRO" == "Red" ] && install_packages "genisoimage" # required for OpenNebula connector pip2 install untangle || exit 1 diff --git a/stdeb.cfg b/stdeb.cfg index f0424fb3..3751334c 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -2,5 +2,5 @@ Suite: xenial XS-Python-Version: >= 2.7 Maintainer: Gerardo Garcia -Depends: python-pip, libmysqlclient-dev, libssl-dev, libffi-dev, python-argcomplete, python-boto, python-bottle, python-jsonschema, python-logutils, python-cinderclient, python-glanceclient, python-keystoneclient, python-neutronclient, python-networking-l2gw, python-novaclient, python-openstackclient, python-mysqldb, python-lib-osm-openvim, python-osm-im, python-networkx +Depends: python-pip, libmysqlclient-dev, libssl-dev, libffi-dev, python-argcomplete, python-boto, python-bottle, python-jsonschema, python-logutils, python-cinderclient, python-glanceclient, python-keystoneclient, python-neutronclient, python-networking-l2gw, python-novaclient, python-openstackclient, python-mysqldb, python-lib-osm-openvim, python-osm-im, python-networkx, genisoimage