From: kate Date: Wed, 6 Sep 2017 06:26:28 +0000 (-0700) Subject: Merge branch 'vio' into v2.0 X-Git-Tag: v3.0.0~17^2~3 X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2FRO.git;a=commitdiff_plain;h=5461675ac6705ee92916ed741da1914bd2162482;hp=721d79b1f7efe56ee3ff72f9884ae7f3db671c89 Merge branch 'vio' into v2.0 Change-Id: I380d554f9e9f03b0c4425fd998123fda300b81ff Signed-off-by: kate --- diff --git a/Dockerfile b/Dockerfile index 82c3ca98..254027cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:16.04 RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get -y install git build-essential && \ + DEBIAN_FRONTEND=noninteractive apt-get -y install git build-essential apt-utils && \ DEBIAN_FRONTEND=noninteractive apt-get -y install python python-dev python-all python-stdeb fakeroot pypi2deb && \ DEBIAN_FRONTEND=noninteractive apt-get -y install python-pip libmysqlclient-dev libssl-dev libffi-dev && \ DEBIAN_FRONTEND=noninteractive pip install --upgrade pip && \ diff --git a/Jenkinsfile b/Jenkinsfile index 13a85d03..bc6c2d0f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,36 +1,30 @@ -pipeline { - agent any - stages { - stage("Build") { - agent { - dockerfile true - } - steps { - sh 'make package' - stash name: "deb-files", includes: ".build/*.deb" - } - } - stage("Unittest") { - agent { - dockerfile true - } - steps { - sh 'echo "UNITTEST"' - } - } - stage("Repo Component") { - agent any - steps { - unstash "deb-files" - sh ''' - mkdir -p pool/RO - mv .build/*.deb pool/RO/ - mkdir -p dists/ReleaseOne/unstable/RO/binary-amd64/ - apt-ftparchive packages pool/RO > dists/ReleaseOne/unstable/RO/binary-amd64/Packages - gzip -9fk dists/ReleaseOne/unstable/RO/binary-amd64/Packages - ''' - archiveArtifacts artifacts: "dists/**,pool/RO/*.deb" - } - } - } +properties([ + parameters([ + string(defaultValue: env.BRANCH_NAME, description: '', name: 'GERRIT_BRANCH'), + string(defaultValue: 'osm/RO', description: '', name: 'GERRIT_PROJECT'), + string(defaultValue: env.GERRIT_REFSPEC, description: '', name: 'GERRIT_REFSPEC'), + string(defaultValue: env.GERRIT_PATCHSET_REVISION, description: '', name: 'GERRIT_PATCHSET_REVISION'), + string(defaultValue: 'https://osm.etsi.org/gerrit', description: '', name: 'PROJECT_URL_PREFIX'), + booleanParam(defaultValue: false, description: '', name: 'TEST_INSTALL'), + ]) +]) + +def devops_checkout() { + dir('devops') { + git url: "${PROJECT_URL_PREFIX}/osm/devops", branch: params.GERRIT_BRANCH + } +} + +node { + checkout scm + devops_checkout() + + ci_helper = load "devops/jenkins/ci-pipelines/ci_stage_2.groovy" + ci_helper.ci_pipeline( 'RO', + params.PROJECT_URL_PREFIX, + params.GERRIT_PROJECT, + params.GERRIT_BRANCH, + params.GERRIT_REFSPEC, + params.GERRIT_PATCHSET_REVISION, + params.TEST_INSTALL) } diff --git a/database_utils/migrate_mano_db.sh b/database_utils/migrate_mano_db.sh index d57194e0..d99203e7 100755 --- a/database_utils/migrate_mano_db.sh +++ b/database_utils/migrate_mano_db.sh @@ -27,13 +27,13 @@ DBUSER="mano" DBPASS="" -DEFAULT_DBPASS="maopw" +DEFAULT_DBPASS="manopw" DBHOST="" DBPORT="3306" DBNAME="mano_db" QUIET_MODE="" #TODO update it with the last database version -LAST_DB_VERSION=21 +LAST_DB_VERSION=24 # Detect paths MYSQL=$(which mysql) @@ -189,6 +189,9 @@ fi #[ $OPENMANO_VER_NUM -ge 5005 ] && DB_VERSION=19 #0.5.5 => 19 #[ $OPENMANO_VER_NUM -ge 5009 ] && DB_VERSION=20 #0.5.9 => 20 #[ $OPENMANO_VER_NUM -ge 5015 ] && DB_VERSION=21 #0.5.15 => 21 +#[ $OPENMANO_VER_NUM -ge 5016 ] && DB_VERSION=22 #0.5.16 => 22 +#[ $OPENMANO_VER_NUM -ge 5020 ] && DB_VERSION=23 #0.5.20 => 23 +#[ $OPENMANO_VER_NUM -ge 5021 ] && DB_VERSION=24 #0.5.21 => 24 #TODO ... put next versions here function upgrade_to_1(){ @@ -772,6 +775,48 @@ function downgrade_from_21(){ echo "DELETE FROM schema_version WHERE version_int='21';" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 } +function upgrade_to_22(){ + # echo " upgrade database from version 0.21 to version 0.22" + echo " Changed type of ram in 'flavors' from SMALLINT to MEDIUMINT" + echo "ALTER TABLE flavors CHANGE COLUMN ram ram MEDIUMINT(7) UNSIGNED NULL DEFAULT NULL AFTER disk;" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 + echo "INSERT INTO schema_version (version_int, version, openmano_ver, comments, date) VALUES (22, '0.22', '0.5.16', 'Changed type of ram in flavors from SMALLINT to MEDIUMINT', '2017-06-02');" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 +} +function downgrade_from_22(){ + # echo " downgrade database from version 0.22 to version 0.21" + echo " Changed type of ram in 'flavors' from MEDIUMINT to SMALLINT" + echo "ALTER TABLE flavors CHANGE COLUMN ram ram SMALLINT(5) UNSIGNED NULL DEFAULT NULL AFTER disk;" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 + echo "DELETE FROM schema_version WHERE version_int='22';" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 +} + +function upgrade_to_23(){ + # echo " upgrade database from version 0.22 to version 0.23" + echo " add column 'availability_zone' at table 'vms'" + echo "ALTER TABLE mano_db.vms ADD COLUMN availability_zone VARCHAR(255) NULL AFTER modified_at;" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 + echo "INSERT INTO schema_version (version_int, version, openmano_ver, comments, date) VALUES (23, '0.23', '0.5.20', 'Changed type of ram in flavors from SMALLINT to MEDIUMINT', '2017-08-29');" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 +} +function downgrade_from_23(){ + # echo " downgrade database from version 0.23 to version 0.22" + echo " remove column 'availability_zone' from table 'vms'" + echo "ALTER TABLE mano_db.vms DROP COLUMN availability_zone;" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 + echo "DELETE FROM schema_version WHERE version_int='23';" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 +} + +function upgrade_to_24(){ + # echo " upgrade database from version 0.23 to version 0.24" + echo " Add 'count' to table 'vms'" + echo "ALTER TABLE vms ADD COLUMN count SMALLINT NOT NULL DEFAULT '1' AFTER vnf_id;" | $DBCMD || + ! echo "ERROR. Aborted!" || exit -1 + echo "INSERT INTO schema_version (version_int, version, openmano_ver, comments, date) "\ + "VALUES (24, '0.24', '0.5.21', 'Added vnfd fields', '2017-08-29');" | $DBCMD || + ! echo "ERROR. Aborted!" || exit -1 +} +function downgrade_from_24(){ + # echo " downgrade database from version 0.24 to version 0.23" + echo " Remove 'count' from table 'vms'" + echo "ALTER TABLE vms DROP COLUMN count;" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 + echo "DELETE FROM schema_version WHERE version_int='24';" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 +} + function upgrade_to_X(){ echo " change 'datacenter_nets'" echo "ALTER TABLE datacenter_nets ADD COLUMN vim_tenant_id VARCHAR(36) NOT NULL AFTER datacenter_id, DROP INDEX name_datacenter_id, ADD UNIQUE INDEX name_datacenter_id (name, datacenter_id, vim_tenant_id);" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 diff --git a/devops-stages/stage-archive.sh b/devops-stages/stage-archive.sh new file mode 100755 index 00000000..cc1cfc05 --- /dev/null +++ b/devops-stages/stage-archive.sh @@ -0,0 +1,8 @@ +#!/bin/sh +rm -rf pool +rm -rf dists +mkdir -p pool/RO +mv .build/*.deb pool/RO/ +mkdir -p dists/unstable/RO/binary-amd64/ +apt-ftparchive packages pool/RO > dists/unstable/RO/binary-amd64/Packages +gzip -9fk dists/unstable/RO/binary-amd64/Packages diff --git a/devops-stages/stage-build.sh b/devops-stages/stage-build.sh new file mode 100755 index 00000000..85054995 --- /dev/null +++ b/devops-stages/stage-build.sh @@ -0,0 +1,2 @@ +#!/bin/sh +make package diff --git a/devops-stages/stage-test.sh b/devops-stages/stage-test.sh new file mode 100755 index 00000000..49296c71 --- /dev/null +++ b/devops-stages/stage-test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo "UNITTEST" diff --git a/openmano b/openmano index 6c0d15b2..ec588a98 100755 --- a/openmano +++ b/openmano @@ -23,13 +23,13 @@ # contact with: nfvlabs@tid.es ## -''' +""" openmano client used to interact with openmano-server (openmanod) -''' +""" __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes" __date__ = "$09-oct-2014 09:09:48$" -__version__ = "0.4.14-r521" -version_date = "May 2017" +__version__ = "0.4.15-r525" +version_date = "Jul 2017" from argcomplete.completers import FilesCompleter import os @@ -1033,6 +1033,7 @@ def datacenter_delete(args): print content['error']['description'] return result + def datacenter_list(args): #print "datacenter-list",args tenant='any' if args.all else _get_tenant() @@ -1050,6 +1051,7 @@ def datacenter_list(args): args.verbose += 1 return _print_verbose(mano_response, args.verbose) + def datacenter_sdn_port_mapping_set(args): tenant = _get_tenant() datacenter = _get_datacenter(args.name, tenant) @@ -1058,30 +1060,37 @@ def datacenter_sdn_port_mapping_set(args): if not args.file: raise OpenmanoCLIError( "No yaml/json has been provided specifying the SDN port mapping") + sdn_port_mapping = _load_file_or_yaml(args.file) + payload_req = json.dumps({"sdn_port_mapping": sdn_port_mapping}) - port_mapping = yaml.load(datacenter_sdn_port_mapping_list(args)) - if 'error' in port_mapping: + # read + URLrequest = "http://%s:%s/openmano/%s/datacenters/%s/sdn_mapping" % (mano_host, mano_port, tenant, datacenter) + mano_response = requests.get(URLrequest) + logger.debug("openmano response: %s", mano_response.text) + port_mapping = mano_response.json() + if mano_response.status_code != 200: + str(mano_response.json()) raise OpenmanoCLIError("openmano client error: {}".format(port_mapping['error']['description'])) if len(port_mapping["sdn_port_mapping"]["ports_mapping"]) > 0: if not args.force: r = raw_input("Datacenter %s already contains a port mapping. Overwrite? (y/N)? " % (datacenter)) if not (len(r) > 0 and r[0].lower() == "y"): return 0 - args.force = True - print datacenter_sdn_port_mapping_clear(args) - sdn_port_mapping = _load_file_or_yaml(args.file) - payload_req = json.dumps({"sdn_port_mapping": sdn_port_mapping}) + # clear + URLrequest = "http://%s:%s/openmano/%s/datacenters/%s/sdn_mapping" % (mano_host, mano_port, tenant, datacenter) + mano_response = requests.delete(URLrequest) + logger.debug("openmano response: %s", mano_response.text) + if mano_response.status_code != 200: + return _print_verbose(mano_response, args.verbose) + # set URLrequest = "http://%s:%s/openmano/%s/datacenters/%s/sdn_mapping" % (mano_host, mano_port, tenant, datacenter) logger.debug("openmano request: %s", payload_req) mano_response = requests.post(URLrequest, headers=headers_req, data=payload_req) logger.debug("openmano response: %s", mano_response.text) + return _print_verbose(mano_response, args.verbose) - if mano_response.status_code == 200: - return yaml.safe_dump(mano_response.json()) - else: - return mano_response.content def datacenter_sdn_port_mapping_list(args): tenant = _get_tenant() @@ -1091,10 +1100,8 @@ def datacenter_sdn_port_mapping_list(args): mano_response = requests.get(URLrequest) logger.debug("openmano response: %s", mano_response.text) - if mano_response.status_code != 200: - return mano_response.content + return _print_verbose(mano_response, 4) - return yaml.safe_dump(mano_response.json()) def datacenter_sdn_port_mapping_clear(args): tenant = _get_tenant() @@ -1102,26 +1109,27 @@ def datacenter_sdn_port_mapping_clear(args): if not args.force: r = raw_input("Clean SDN port mapping for datacenter %s (y/N)? " %(datacenter)) - if not (len(r)>0 and r[0].lower()=="y"): + if not (len(r) > 0 and r[0].lower() == "y"): return 0 URLrequest = "http://%s:%s/openmano/%s/datacenters/%s/sdn_mapping" % (mano_host, mano_port, tenant, datacenter) mano_response = requests.delete(URLrequest) logger.debug("openmano response: %s", mano_response.text) - if mano_response.status_code != 200: - if "No port mapping for datacenter" in mano_response.content: - return "No port mapping for datacenter " + datacenter + " has been found" - return mano_response.content + return _print_verbose(mano_response, args.verbose) - return yaml.safe_dump(mano_response.json()) def sdn_controller_create(args): tenant = _get_tenant() headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} - if not (args.ip and args.port and args.dpid and args.type): - raise OpenmanoCLIError("The following arguments are required: ip, port, dpid, type") + error_msg=[] + if not args.ip: error_msg.append("'ip'") + if not args.port: error_msg.append("'port'") + if not args.dpid: error_msg.append("'dpid'") + if not args.type: error_msg.append("'type'") + if error_msg: + raise OpenmanoCLIError("The following arguments are required: " + ",".join(error_msg)) controller_dict = {} controller_dict['name'] = args.name @@ -1145,42 +1153,41 @@ def sdn_controller_create(args): mano_response = requests.post(URLrequest, headers=headers_req, data=payload_req) logger.debug("openmano response: %s", mano_response.text) result = _print_verbose(mano_response, args.verbose) - return result + def sdn_controller_edit(args): tenant = _get_tenant() controller_uuid = _get_item_uuid("sdn_controllers", args.name, tenant) headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} - if not (args.new_name or args.ip or args.port or args.dpid or args.type): - raise OpenmanoCLIError("At least one parameter must be editd") - - if not args.force: - r = raw_input("Update SDN controller %s (y/N)? " %(args.name)) - if not (len(r)>0 and r[0].lower()=="y"): - return 0 - controller_dict = {} - if args.new_name != None: + if args.new_name: controller_dict['name'] = args.new_name - if args.ip != None: + if args.ip: controller_dict['ip'] = args.ip - if args.port != None: + if args.port: controller_dict['port'] = int(args.port) - if args.dpid != None: + if args.dpid: controller_dict['dpid'] = args.dpid - if args.type != None: + if args.type: controller_dict['type'] = args.type - if args.description != None: + if args.description: controller_dict['description'] = args.description - if args.user != None: + if args.user: controller_dict['user'] = args.user - if args.password != None: + if args.password: controller_dict['password'] = args.password - payload_req = json.dumps({"sdn_controller": controller_dict}) + if not controller_dict: + raise OpenmanoCLIError("At least one parameter must be edited") + if not args.force: + r = raw_input("Update SDN controller {} (y/N)? ".format(args.name)) + if not (len(r) > 0 and r[0].lower() == "y"): + return 0 + + payload_req = json.dumps({"sdn_controller": controller_dict}) # print payload_req URLrequest = "http://%s:%s/openmano/%s/sdn_controllers/%s" % (mano_host, mano_port, tenant, controller_uuid) @@ -1188,9 +1195,9 @@ def sdn_controller_edit(args): mano_response = requests.put(URLrequest, headers=headers_req, data=payload_req) logger.debug("openmano response: %s", mano_response.text) result = _print_verbose(mano_response, args.verbose) - return result + def sdn_controller_list(args): tenant = _get_tenant() headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} @@ -1208,8 +1215,9 @@ def sdn_controller_list(args): if args.name!=None: args.verbose += 1 - result = json.dumps(mano_response.json(), indent=4) - return result + # json.dumps(mano_response.json(), indent=4) + return _print_verbose(mano_response, args.verbose) + def sdn_controller_delete(args): tenant = _get_tenant() @@ -1223,9 +1231,7 @@ def sdn_controller_delete(args): URLrequest = "http://%s:%s/openmano/%s/sdn_controllers/%s" % (mano_host, mano_port, tenant, controller_uuid) mano_response = requests.delete(URLrequest) logger.debug("openmano response: %s", mano_response.text) - result = _print_verbose(mano_response, args.verbose) - - return result + return _print_verbose(mano_response, args.verbose) def vim_action(args): #print "datacenter-net-action",args @@ -1284,6 +1290,7 @@ def vim_action(args): args.verbose=0 return _print_verbose(mano_response, args.verbose) + def _get_items(item, item_name_id=None, datacenter=None, tenant=None): URLrequest = "http://%s:%s/openmano" %(mano_host, mano_port) if tenant: @@ -1299,6 +1306,7 @@ def _get_items(item, item_name_id=None, datacenter=None, tenant=None): return mano_response + def vim_net_sdn_attach(args): #Verify the network exists in the vim tenant = _get_tenant() @@ -1324,8 +1332,7 @@ def vim_net_sdn_attach(args): mano_response = requests.post(URLrequest, headers=headers_req, data=json.dumps(payload_req)) logger.debug("openmano response: %s", mano_response.text) result = _print_verbose(mano_response, args.verbose) - - return + return result def vim_net_sdn_detach(args): @@ -1358,8 +1365,8 @@ def vim_net_sdn_detach(args): mano_response = requests.delete(URLrequest) logger.debug("openmano response: %s", mano_response.text) result = _print_verbose(mano_response, args.verbose) + return result - return def datacenter_net_action(args): if args.action == "net-update": @@ -1471,6 +1478,7 @@ def datacenter_netmap_action(args): logger.debug("openmano response: %s", mano_response.text ) return _print_verbose(mano_response, args.verbose) + def element_edit(args): element = _get_item_uuid(args.element, args.name) headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} @@ -1544,6 +1552,7 @@ def datacenter_edit(args): args.verbose += 1 return _print_verbose(mano_response, args.verbose) + def version(args): headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} URLrequest = "http://%s:%s/openmano/version" % (mano_host, mano_port) diff --git a/openmanod b/openmanod index 8f8c3f1b..94924bd7 100755 --- a/openmanod +++ b/openmanod @@ -48,13 +48,14 @@ import osm_ro __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes" __date__ = "$26-aug-2014 11:09:29$" -__version__ = "0.5.15-r524" -version_date = "Jun 2017" -database_version = 21 #expected database schema version +__version__ = "0.5.21-r531" +version_date = "Aug 2017" +database_version = 24 # expected database schema version global global_config global logger + class LoadConfigurationException(Exception): pass diff --git a/osm_ro/db_base.py b/osm_ro/db_base.py index 4a877213..26e4c002 100644 --- a/osm_ro/db_base.py +++ b/osm_ro/db_base.py @@ -336,7 +336,31 @@ class db_base(): self.logger.debug(cmd) self.cur.execute(cmd) return self.cur.rowcount - + + def _new_uuid(self, root_uuid=None, used_table=None, created_time=0): + """ + Generate a new uuid. It DOES NOT begin or end the transaction, so self.con.cursor must be created + :param root_uuid: master uuid of the transaction + :param used_table: the table this uuid is intended for + :param created_time: time of creation + :return: the created uuid + """ + + uuid = str(myUuid.uuid1()) + # defining root_uuid if not provided + if root_uuid is None: + root_uuid = uuid + if created_time: + created_at = created_time + else: + created_at = time.time() + # inserting new uuid + cmd = "INSERT INTO uuids (uuid, root_uuid, used_at, created_at) VALUES ('{:s}','{:s}','{:s}', {:f})".format( + uuid, root_uuid, used_table, created_at) + self.logger.debug(cmd) + self.cur.execute(cmd) + return uuid + def _new_row_internal(self, table, INSERT, add_uuid=False, root_uuid=None, created_time=0): ''' Add one row into a table. It DOES NOT begin or end the transaction, so self.con.cursor must be created Attribute diff --git a/osm_ro/httpserver.py b/osm_ro/httpserver.py index 94544f6b..f9cbb9b4 100644 --- a/osm_ro/httpserver.py +++ b/osm_ro/httpserver.py @@ -1376,10 +1376,18 @@ def http_get_instance_id(tenant_id, instance_id): nfvo.refresh_instance(mydb, tenant_id, instance_dict) except (nfvo.NfvoException, db_base_Exception) as e: logger.warn("nfvo.refresh_instance couldn't refresh the status of the instance: %s" % str(e)) - #obtain data with results upated + # obtain data with results upated instance = mydb.get_instance_scenario(instance_id, tenant_id) + # Workaround to SO, convert vnfs:vms:interfaces:ip_address from ";" separated list to report the first value + for vnf in instance.get("vnfs", ()): + for vm in vnf.get("vms", ()): + for iface in vm.get("interfaces", ()): + if iface.get("ip_address"): + index = iface["ip_address"].find(";") + if index >= 0: + iface["ip_address"] = iface["ip_address"][:index] convert_datetime2str(instance) - #print json.dumps(instance, indent=4) + # print json.dumps(instance, indent=4) return format_out(instance) except (nfvo.NfvoException, db_base_Exception) as e: logger.error("http_get_instance_id error {}: {}".format(e.http_code, str(e))) diff --git a/osm_ro/nfvo.py b/osm_ro/nfvo.py index d45a3c79..fa87205f 100644 --- a/osm_ro/nfvo.py +++ b/osm_ro/nfvo.py @@ -38,6 +38,7 @@ import console_proxy_thread as cli import vimconn import logging import collections +from uuid import uuid4 from db_base import db_base_Exception import nfvo_db @@ -334,7 +335,9 @@ def rollback(mydb, vims, rollback_list): if item["where"]=="vim": if item["vim_id"] not in vims: continue - vim=vims[ item["vim_id"] ] + if is_task_id(item["uuid"]): + continue + vim = vims[item["vim_id"]] try: if item["what"]=="image": vim.delete_image(item["uuid"]) @@ -392,12 +395,12 @@ def check_vnf_descriptor(vnf_descriptor, vnf_descriptor_version=1): name_dict[ interface["name"] ] = "overlay" vnfc_interfaces[ vnfc["name"] ] = name_dict # check bood-data info - if "boot-data" in vnfc: - # check that user-data is incompatible with users and config-files - if (vnfc["boot-data"].get("users") or vnfc["boot-data"].get("config-files")) and vnfc["boot-data"].get("user-data"): - raise NfvoException( - "Error at vnf:VNFC:boot-data, fields 'users' and 'config-files' are not compatible with 'user-data'", - HTTP_Bad_Request) + # if "boot-data" in vnfc: + # # check that user-data is incompatible with users and config-files + # if (vnfc["boot-data"].get("users") or vnfc["boot-data"].get("config-files")) and vnfc["boot-data"].get("user-data"): + # raise NfvoException( + # "Error at vnf:VNFC:boot-data, fields 'users' and 'config-files' are not compatible with 'user-data'", + # HTTP_Bad_Request) #check if the info in external_connections matches with the one in the vnfcs name_list=[] @@ -481,7 +484,7 @@ def check_vnf_descriptor(vnf_descriptor, vnf_descriptor_version=1): HTTP_Bad_Request) -def create_or_use_image(mydb, vims, image_dict, rollback_list, only_create_at_vim=False, return_on_error = None): +def create_or_use_image(mydb, vims, image_dict, rollback_list, only_create_at_vim=False, return_on_error=None): #look if image exist if only_create_at_vim: image_mano_id = image_dict['uuid'] @@ -757,6 +760,7 @@ def new_vnf(mydb, tenant_id, vnf_descriptor): for vnfc in vnf_descriptor['vnf']['VNFC']: VNFCitem={} VNFCitem["name"] = vnfc['name'] + VNFCitem["availability_zone"] = vnfc.get('availability_zone') VNFCitem["description"] = vnfc.get("description", 'VM %s of the VNF %s' %(vnfc['name'],vnf_name)) #print "Flavor name: %s. Description: %s" % (VNFCitem["name"]+"-flv", VNFCitem["description"]) @@ -827,6 +831,7 @@ def new_vnf(mydb, tenant_id, vnf_descriptor): #print "Image id for VNFC %s: %s" % (vnfc['name'],image_id) VNFCDict[vnfc['name']]["image_id"] = image_id VNFCDict[vnfc['name']]["image_path"] = vnfc.get('VNFC image') + VNFCDict[vnfc['name']]["count"] = vnfc.get('count', 1) if vnfc.get("boot-data"): VNFCDict[vnfc['name']]["boot_data"] = yaml.safe_dump(vnfc["boot-data"], default_flow_style=True, width=256) @@ -962,6 +967,7 @@ def new_vnf_v02(mydb, tenant_id, vnf_descriptor): #print "Image id for VNFC %s: %s" % (vnfc['name'],image_id) VNFCDict[vnfc['name']]["image_id"] = image_id VNFCDict[vnfc['name']]["image_path"] = vnfc.get('VNFC image') + VNFCDict[vnfc['name']]["count"] = vnfc.get('count', 1) if vnfc.get("boot-data"): VNFCDict[vnfc['name']]["boot_data"] = yaml.safe_dump(vnfc["boot-data"], default_flow_style=True, width=256) @@ -1665,6 +1671,7 @@ def start_scenario(mydb, tenant_id, scenario_id, instance_scenario_name, instanc logger.debug("start_scenario 2. Creating new nets (vnf internal nets) in the VIM") #For each vnf net, we create it and we add it to instanceNetlist. + for sce_vnf in scenarioDict['vnfs']: for net in sce_vnf['nets']: #print "Net name: %s. Description: %s" % (net["name"], net["description"]) @@ -1696,6 +1703,17 @@ def start_scenario(mydb, tenant_id, scenario_id, instance_scenario_name, instanc #myvim.new_vminstance(self,vimURI,tenant_id,name,description,image_id,flavor_id,net_dict) i = 0 for sce_vnf in scenarioDict['vnfs']: + vnf_availability_zones = [] + for vm in sce_vnf['vms']: + vm_av = vm.get('availability_zone') + if vm_av and vm_av not in vnf_availability_zones: + vnf_availability_zones.append(vm_av) + + # check if there is enough availability zones available at vim level. + if myvims[datacenter_id].availability_zone and vnf_availability_zones: + if len(vnf_availability_zones) > len(myvims[datacenter_id].availability_zone): + raise NfvoException('No enough availability zones at VIM for this deployment', HTTP_Bad_Request) + for vm in sce_vnf['vms']: i += 1 myVMDict = {} @@ -1782,8 +1800,16 @@ def start_scenario(mydb, tenant_id, scenario_id, instance_scenario_name, instanc #print "networks", yaml.safe_dump(myVMDict['networks'], indent=4, default_flow_style=False) #print "interfaces", yaml.safe_dump(vm['interfaces'], indent=4, default_flow_style=False) #print ">>>>>>>>>>>>>>>>>>>>>>>>>>>" - vm_id = myvim.new_vminstance(myVMDict['name'],myVMDict['description'],myVMDict.get('start', None), - myVMDict['imageRef'],myVMDict['flavorRef'],myVMDict['networks']) + + if 'availability_zone' in myVMDict: + av_index = vnf_availability_zones.index(myVMDict['availability_zone']) + else: + av_index = None + + vm_id = myvim.new_vminstance(myVMDict['name'], myVMDict['description'], myVMDict.get('start', None), + myVMDict['imageRef'], myVMDict['flavorRef'], myVMDict['networks'], + availability_zone_index=av_index, + availability_zone_list=vnf_availability_zones) #print "VIM vm instance id (server id) for scenario %s: %s" % (scenarioDict['name'],vm_id) vm['vim_id'] = vm_id rollbackList.append({'what':'vm','where':'vim','vim_id':datacenter_id,'uuid':vm_id}) @@ -1813,10 +1839,10 @@ def start_scenario(mydb, tenant_id, scenario_id, instance_scenario_name, instanc def unify_cloud_config(cloud_config_preserve, cloud_config): - ''' join the cloud config information into cloud_config_preserve. + """ join the cloud config information into cloud_config_preserve. In case of conflict cloud_config_preserve preserves - None is admited - ''' + None is allowed + """ if not cloud_config_preserve and not cloud_config: return None @@ -1866,10 +1892,19 @@ def unify_cloud_config(cloud_config_preserve, cloud_config): new_cloud_config["boot-data-drive"] = cloud_config_preserve["boot-data-drive"] # user-data - if cloud_config and cloud_config.get("user-data") != None: - new_cloud_config["user-data"] = cloud_config["user-data"] - if cloud_config_preserve and cloud_config_preserve.get("user-data") != None: - new_cloud_config["user-data"] = cloud_config_preserve["user-data"] + new_cloud_config["user-data"] = [] + if cloud_config and cloud_config.get("user-data"): + if isinstance(cloud_config["user-data"], list): + new_cloud_config["user-data"] += cloud_config["user-data"] + else: + new_cloud_config["user-data"].append(cloud_config["user-data"]) + if cloud_config_preserve and cloud_config_preserve.get("user-data"): + if isinstance(cloud_config_preserve["user-data"], list): + new_cloud_config["user-data"] += cloud_config_preserve["user-data"] + else: + new_cloud_config["user-data"].append(cloud_config_preserve["user-data"]) + if not new_cloud_config["user-data"]: + del new_cloud_config["user-data"] # config files new_cloud_config["config-files"] = [] @@ -1923,6 +1958,7 @@ def get_vim_thread(mydb, tenant_id, datacenter_id_name=None, datacenter_tenant_i except db_base_Exception as e: raise NfvoException("{} {}".format(type(e).__name__ , str(e)), e.http_code) + def get_datacenter_by_name_uuid(mydb, tenant_id, datacenter_id_name=None, **extra_filter): datacenter_id = None datacenter_name = None @@ -1978,14 +2014,30 @@ def create_instance(mydb, tenant_id, instance_dict): #logger.debug(">>>>>>> InstanceDict:\n{}".format(yaml.safe_dump(instance_dict,default_flow_style=False, width=256))) #logger.debug(">>>>>>> ScenarioDict:\n{}".format(yaml.safe_dump(scenarioDict,default_flow_style=False, width=256))) - scenarioDict['datacenter_id'] = default_datacenter_id + uuid_list = [] + instance_name = instance_dict["name"] + instance_uuid = str(uuid4()) + uuid_list.append(instance_uuid) + db_instance_scenario = { + "uuid": instance_uuid, + "name": instance_name, + "tenant_id": tenant_id, + "scenario_id": scenarioDict['uuid'], + "datacenter_id": default_datacenter_id, + # filled bellow 'datacenter_tenant_id' + "description": instance_dict.get("description"), + } + db_ip_profiles=[] + if scenarioDict.get("cloud-config"): + db_instance_scenario["cloud_config"] = yaml.safe_dump(scenarioDict["cloud-config"], + default_flow_style=True, width=256) + vnf_net2instance = {} #Auxiliar dictionary. First key:'scenario' or sce_vnf uuid. Second Key: uuid of the net/sce_net. Value: vim_net_id + sce_net2instance = {} auxNetDict = {} #Auxiliar dictionary. First key:'scenario' or sce_vnf uuid. Second Key: uuid of the net/sce_net. Value: vim_net_id auxNetDict['scenario'] = {} logger.debug("Creating instance from scenario-dict:\n%s", yaml.safe_dump(scenarioDict, indent=4, default_flow_style=False)) #TODO remove - instance_name = instance_dict["name"] - instance_description = instance_dict.get("description") try: # 0 check correct parameters for net_name, net_instance_desc in instance_dict.get("networks",{}).iteritems(): @@ -2069,12 +2121,12 @@ def create_instance(mydb, tenant_id, instance_dict): #logger.debug(">>>>>>>> Merged dictionary") logger.debug("Creating instance scenario-dict MERGED:\n%s", yaml.safe_dump(scenarioDict, indent=4, default_flow_style=False)) - # 1. Creating new nets (sce_nets) in the VIM" + db_instance_nets = [] for sce_net in scenarioDict['nets']: - sce_net["vim_id_sites"]={} - descriptor_net = instance_dict.get("networks",{}).get(sce_net["name"],{}) + descriptor_net = instance_dict.get("networks",{}).get(sce_net["name"],{}) net_name = descriptor_net.get("vim-network-name") + sce_net2instance[sce_net['uuid']] = {} auxNetDict['scenario'][sce_net['uuid']] = {} sites = descriptor_net.get("sites", [ {} ]) @@ -2139,7 +2191,7 @@ def create_instance(mydb, tenant_id, instance_dict): if not create_network: raise NfvoException("No candidate VIM network found for " + filter_text, HTTP_Bad_Request ) else: - sce_net["vim_id_sites"][datacenter_id] = vim_nets[0]['id'] + vim_id = vim_nets[0]['id'] auxNetDict['scenario'][sce_net['uuid']][datacenter_id] = vim_nets[0]['id'] create_network = False if create_network: @@ -2149,13 +2201,41 @@ def create_instance(mydb, tenant_id, instance_dict): instance_tasks[task_id] = task tasks_to_launch[myvim_thread_id].append(task) #network_id = vim.new_network(net_vim_name, net_type, sce_net.get('ip_profile',None)) - sce_net["vim_id_sites"][datacenter_id] = task_id + vim_id = task_id auxNetDict['scenario'][sce_net['uuid']][datacenter_id] = task_id rollbackList.append({'what':'network', 'where':'vim', 'vim_id':datacenter_id, 'uuid':task_id}) sce_net["created"] = True + # fill database content + net_uuid = str(uuid4()) + uuid_list.append(net_uuid) + sce_net2instance[sce_net['uuid']][datacenter_id] = net_uuid + db_net = { + "uuid": net_uuid, + 'vim_net_id': vim_id, + "instance_scenario_id": instance_uuid, + "sce_net_id": sce_net["uuid"], + "created": create_network, + 'datacenter_id': datacenter_id, + 'datacenter_tenant_id': myvim_thread_id, + 'status': 'BUILD' if create_network else "ACTIVE" + } + db_instance_nets.append(db_net) + if 'ip_profile' in sce_net: + db_ip_profile={ + 'instance_net_id': net_uuid, + 'ip_version': sce_net['ip_profile']['ip_version'], + 'subnet_address': sce_net['ip_profile']['subnet_address'], + 'gateway_address': sce_net['ip_profile']['gateway_address'], + 'dns_address': sce_net['ip_profile']['dns_address'], + 'dhcp_enabled': sce_net['ip_profile']['dhcp_enabled'], + 'dhcp_start_address': sce_net['ip_profile']['dhcp_start_address'], + 'dhcp_count': sce_net['ip_profile']['dhcp_count'], + } + db_ip_profiles.append(db_ip_profile) + # 2. Creating new nets (vnf internal nets) in the VIM" - #For each vnf net, we create it and we add it to instanceNetlist. + # For each vnf net, we create it and we add it to instanceNetlist. for sce_vnf in scenarioDict['vnfs']: for net in sce_vnf['nets']: if sce_vnf.get("datacenter"): @@ -2177,20 +2257,65 @@ def create_instance(mydb, tenant_id, instance_dict): instance_tasks[task_id] = task tasks_to_launch[myvim_thread_id].append(task) # network_id = vim.new_network(net_name, net_type, net.get('ip_profile',None)) - net['vim_id'] = task_id + vim_id = task_id + if sce_vnf['uuid'] not in vnf_net2instance: + vnf_net2instance[sce_vnf['uuid']] = {} + vnf_net2instance[sce_vnf['uuid']][net['uuid']] = task_id if sce_vnf['uuid'] not in auxNetDict: auxNetDict[sce_vnf['uuid']] = {} auxNetDict[sce_vnf['uuid']][net['uuid']] = task_id rollbackList.append({'what':'network','where':'vim','vim_id':datacenter_id,'uuid':task_id}) net["created"] = True - - #print "auxNetDict:" - #print yaml.safe_dump(auxNetDict, indent=4, default_flow_style=False) + # fill database content + net_uuid = str(uuid4()) + uuid_list.append(net_uuid) + vnf_net2instance[sce_vnf['uuid']][net['uuid']] = net_uuid + db_net = { + "uuid": net_uuid, + 'vim_net_id': vim_id, + "instance_scenario_id": instance_uuid, + "net_id": net["uuid"], + "created": True, + 'datacenter_id': datacenter_id, + 'datacenter_tenant_id': myvim_thread_id, + } + db_instance_nets.append(db_net) + if 'ip_profile' in net: + db_ip_profile = { + 'instance_net_id': net_uuid, + 'ip_version': net['ip_profile']['ip_version'], + 'subnet_address': net['ip_profile']['subnet_address'], + 'gateway_address': net['ip_profile']['gateway_address'], + 'dns_address': net['ip_profile']['dns_address'], + 'dhcp_enabled': net['ip_profile']['dhcp_enabled'], + 'dhcp_start_address': net['ip_profile']['dhcp_start_address'], + 'dhcp_count': net['ip_profile']['dhcp_count'], + } + db_ip_profiles.append(db_ip_profile) + + #print "vnf_net2instance:" + #print yaml.safe_dump(vnf_net2instance, indent=4, default_flow_style=False) # 3. Creating new vm instances in the VIM + db_instance_vnfs = [] + db_instance_vms = [] + db_instance_interfaces = [] #myvim.new_vminstance(self,vimURI,tenant_id,name,description,image_id,flavor_id,net_dict) - for sce_vnf in scenarioDict['vnfs']: + sce_vnf_list = sorted(scenarioDict['vnfs'], key=lambda k: k['name']) + #for sce_vnf in scenarioDict['vnfs']: + for sce_vnf in sce_vnf_list: + vnf_availability_zones = [] + for vm in sce_vnf['vms']: + vm_av = vm.get('availability_zone') + if vm_av and vm_av not in vnf_availability_zones: + vnf_availability_zones.append(vm_av) + + # check if there is enough availability zones available at vim level. + if myvims[datacenter_id].availability_zone and vnf_availability_zones: + if len(vnf_availability_zones) > len(myvims[datacenter_id].availability_zone): + raise NfvoException('No enough availability zones at VIM for this deployment', HTTP_Bad_Request) + if sce_vnf.get("datacenter"): vim = myvims[ sce_vnf["datacenter"] ] myvim_thread_id = myvim_threads_id[ sce_vnf["datacenter"] ] @@ -2199,12 +2324,24 @@ def create_instance(mydb, tenant_id, instance_dict): vim = myvims[ default_datacenter_id ] myvim_thread_id = myvim_threads_id[ default_datacenter_id ] datacenter_id = default_datacenter_id - sce_vnf["datacenter_id"] = datacenter_id + sce_vnf["datacenter_id"] = datacenter_id i = 0 + + vnf_uuid = str(uuid4()) + uuid_list.append(vnf_uuid) + db_instance_vnf = { + 'uuid': vnf_uuid, + 'instance_scenario_id': instance_uuid, + 'vnf_id': sce_vnf['vnf_id'], + 'sce_vnf_id': sce_vnf['uuid'], + 'datacenter_id': datacenter_id, + 'datacenter_tenant_id': myvim_thread_id, + } + db_instance_vnfs.append(db_instance_vnf) + for vm in sce_vnf['vms']: - i += 1 myVMDict = {} - myVMDict['name'] = "{}.{}.{}".format(instance_name,sce_vnf['name'],chr(96+i)) + myVMDict['name'] = "{}.{}.{}".format(instance_name[:64], sce_vnf['name'][:64], vm["name"][:64]) myVMDict['description'] = myVMDict['name'][0:99] # if not startvms: # myVMDict['start'] = "no" @@ -2237,9 +2374,11 @@ def create_instance(mydb, tenant_id, instance_dict): vm['vim_flavor_id'] = flavor_id myVMDict['imageRef'] = vm['vim_image_id'] myVMDict['flavorRef'] = vm['vim_flavor_id'] + myVMDict['availability_zone'] = vm.get('availability_zone') myVMDict['networks'] = [] task_depends = {} #TODO ALF. connect_mgmt_interfaces. Connect management interfaces if this is true + db_vm_ifaces = [] for iface in vm['interfaces']: netDict = {} if iface['type']=="data": @@ -2286,48 +2425,114 @@ def create_instance(mydb, tenant_id, instance_dict): #print vnf_iface if vnf_iface['interface_id']==iface['uuid']: netDict['net_id'] = auxNetDict['scenario'][ vnf_iface['sce_net_id'] ][datacenter_id] + instance_net_id = sce_net2instance[ vnf_iface['sce_net_id'] ][datacenter_id] break else: netDict['net_id'] = auxNetDict[ sce_vnf['uuid'] ][ iface['net_id'] ] + instance_net_id = vnf_net2instance[ sce_vnf['uuid'] ][ iface['net_id'] ] if netDict.get('net_id') and is_task_id(netDict['net_id']): task_depends[netDict['net_id']] = instance_tasks[netDict['net_id']] #skip bridge ifaces not connected to any net #if 'net_id' not in netDict or netDict['net_id']==None: # continue myVMDict['networks'].append(netDict) - #print ">>>>>>>>>>>>>>>>>>>>>>>>>>>" - #print myVMDict['name'] - #print "networks", yaml.safe_dump(myVMDict['networks'], indent=4, default_flow_style=False) - #print "interfaces", yaml.safe_dump(vm['interfaces'], indent=4, default_flow_style=False) - #print ">>>>>>>>>>>>>>>>>>>>>>>>>>>" + db_vm_iface={ + # "uuid" + # 'instance_vm_id': instance_vm_uuid, + "instance_net_id": instance_net_id, + 'interface_id': iface['uuid'], + # 'vim_interface_id': , + 'type': 'external' if iface['external_name'] is not None else 'internal', + 'ip_address': iface.get('ip_address'), + 'floating_ip': int(iface.get('floating-ip', False)), + 'port_security': int(iface.get('port-security', True)) + } + db_vm_ifaces.append(db_vm_iface) + # print ">>>>>>>>>>>>>>>>>>>>>>>>>>>" + # print myVMDict['name'] + # print "networks", yaml.safe_dump(myVMDict['networks'], indent=4, default_flow_style=False) + # print "interfaces", yaml.safe_dump(vm['interfaces'], indent=4, default_flow_style=False) + # print ">>>>>>>>>>>>>>>>>>>>>>>>>>>" if vm.get("boot_data"): cloud_config_vm = unify_cloud_config(vm["boot_data"], cloud_config) else: cloud_config_vm = cloud_config - task = new_task("new-vm", (myVMDict['name'], myVMDict['description'], myVMDict.get('start', None), - myVMDict['imageRef'], myVMDict['flavorRef'], myVMDict['networks'], - cloud_config_vm, myVMDict['disks']), depends=task_depends) - instance_tasks[task["id"]] = task - tasks_to_launch[myvim_thread_id].append(task) - vm_id = task["id"] - vm['vim_id'] = vm_id - rollbackList.append({'what':'vm','where':'vim','vim_id':datacenter_id,'uuid':vm_id}) - #put interface uuid back to scenario[vnfs][vms[[interfaces] - for net in myVMDict['networks']: - if "vim_id" in net: - for iface in vm['interfaces']: - if net["name"]==iface["internal_name"]: - iface["vim_id"]=net["vim_id"] - break + if myVMDict.get('availability_zone'): + av_index = vnf_availability_zones.index(myVMDict['availability_zone']) + else: + av_index = None + for vm_index in range(0, vm.get('count', 1)): + vm_index_name = "" + if vm.get('count', 1) > 1: + vm_index_name += "." + chr(97 + vm_index) + task = new_task("new-vm", (myVMDict['name']+vm_index_name, myVMDict['description'], + myVMDict.get('start', None), myVMDict['imageRef'], + myVMDict['flavorRef'], myVMDict['networks'], + cloud_config_vm, myVMDict['disks'], av_index, + vnf_availability_zones), depends=task_depends) + instance_tasks[task["id"]] = task + tasks_to_launch[myvim_thread_id].append(task) + vm_id = task["id"] + vm['vim_id'] = vm_id + rollbackList.append({'what':'vm','where':'vim','vim_id':datacenter_id,'uuid':vm_id}) + # put interface uuid back to scenario[vnfs][vms[[interfaces] + for net in myVMDict['networks']: + if "vim_id" in net: + for iface in vm['interfaces']: + if net["name"]==iface["internal_name"]: + iface["vim_id"]=net["vim_id"] + break + vm_uuid = str(uuid4()) + uuid_list.append(vm_uuid) + db_vm = { + "uuid": vm_uuid, + 'instance_vnf_id': vnf_uuid, + "vim_vm_id": vm_id, + "vm_id": vm["uuid"], + # "status": + } + db_instance_vms.append(db_vm) + for db_vm_iface in db_vm_ifaces: + iface_uuid = str(uuid4()) + uuid_list.append(iface_uuid) + db_vm_iface_instance = { + "uuid": iface_uuid, + "instance_vm_id": vm_uuid + } + db_vm_iface_instance.update(db_vm_iface) + if db_vm_iface_instance.get("ip_address"): # increment ip_address + ip = db_vm_iface_instance.get("ip_address") + i = ip.rfind(".") + if i > 0: + try: + i += 1 + ip = ip[i:] + str(int(ip[:i]) +1) + db_vm_iface_instance["ip_address"] = ip + except: + db_vm_iface_instance["ip_address"] = None + db_instance_interfaces.append(db_vm_iface_instance) + scenarioDict["datacenter2tenant"] = myvim_threads_id + + db_instance_scenario['datacenter_tenant_id'] = myvim_threads_id[default_datacenter_id] + db_instance_scenario['datacenter_id'] = default_datacenter_id + db_tables=[ + {"instance_scenarios": db_instance_scenario}, + {"instance_vnfs": db_instance_vnfs}, + {"instance_nets": db_instance_nets}, + {"ip_profiles": db_ip_profiles}, + {"instance_vms": db_instance_vms}, + {"instance_interfaces": db_instance_interfaces}, + ] + logger.debug("create_instance Deployment done scenarioDict: %s", - yaml.safe_dump(scenarioDict, indent=4, default_flow_style=False) ) - instance_id = mydb.new_instance_scenario_as_a_whole(tenant_id,instance_name, instance_description, scenarioDict) + yaml.safe_dump(db_tables, indent=4, default_flow_style=False) ) + mydb.new_rows(db_tables, uuid_list) for myvim_thread_id,task_list in tasks_to_launch.items(): for task in task_list: vim_threads["running"][myvim_thread_id].insert_task(task) - global_instance_tasks[instance_id] = instance_tasks + global_instance_tasks[instance_uuid] = instance_tasks # Update database with those ended instance_tasks # for task in instance_tasks.values(): # if task["status"] == "ok": @@ -2337,7 +2542,7 @@ def create_instance(mydb, tenant_id, instance_dict): # elif task["name"] == "new-net": # mydb.update_rows("instance_nets", UPDATE={"vim_net_id": task["result"]}, # WHERE={"vim_net_id": task["id"]}) - return mydb.get_instance_scenario(instance_id) + return mydb.get_instance_scenario(instance_uuid) except (NfvoException, vimconn.vimconnException,db_base_Exception) as e: message = rollback(mydb, myvims, rollbackList) if isinstance(e, db_base_Exception): @@ -2789,20 +2994,24 @@ def new_datacenter(mydb, datacenter_descriptor): def edit_datacenter(mydb, datacenter_id_name, datacenter_descriptor): - #obtain data, check that only one exist + # obtain data, check that only one exist datacenter = mydb.get_table_by_uuid_name('datacenters', datacenter_id_name) - #edit data + + # edit data datacenter_id = datacenter['uuid'] where={'uuid': datacenter['uuid']} + remove_port_mapping = False if "config" in datacenter_descriptor: - if datacenter_descriptor['config']!=None: + if datacenter_descriptor['config'] != None: try: new_config_dict = datacenter_descriptor["config"] #delete null fields to_delete=[] for k in new_config_dict: - if new_config_dict[k]==None: + if new_config_dict[k] == None: to_delete.append(k) + if k == 'sdn-controller': + remove_port_mapping = True config_text = datacenter.get("config") if not config_text: @@ -2814,7 +3023,16 @@ def edit_datacenter(mydb, datacenter_id_name, datacenter_descriptor): del config_dict[k] except Exception as e: raise NfvoException("Bad format at datacenter:config " + str(e), HTTP_Bad_Request) - datacenter_descriptor["config"]= yaml.safe_dump(config_dict,default_flow_style=True,width=256) if len(config_dict)>0 else None + if config_dict: + datacenter_descriptor["config"] = yaml.safe_dump(config_dict, default_flow_style=True, width=256) + else: + datacenter_descriptor["config"] = None + if remove_port_mapping: + try: + datacenter_sdn_port_mapping_delete(mydb, None, datacenter_id) + except ovimException as e: + logger.error("Error deleting datacenter-port-mapping " + str(e)) + mydb.update_rows('datacenters', datacenter_descriptor, where) return datacenter_id @@ -2823,6 +3041,10 @@ def delete_datacenter(mydb, datacenter): #get nfvo_tenant info datacenter_dict = mydb.get_table_by_uuid_name('datacenters', datacenter, 'datacenter') mydb.delete_row_by_id("datacenters", datacenter_dict['uuid']) + try: + datacenter_sdn_port_mapping_delete(mydb, None, datacenter_dict['uuid']) + except ovimException as e: + logger.error("Error deleting datacenter-port-mapping " + str(e)) return datacenter_dict['uuid'] + " " + datacenter_dict['name'] diff --git a/osm_ro/nfvo_db.py b/osm_ro/nfvo_db.py index ac392c6d..071d03ab 100644 --- a/osm_ro/nfvo_db.py +++ b/osm_ro/nfvo_db.py @@ -38,6 +38,7 @@ tables_with_createdat_field=["datacenters","instance_nets","instance_scenarios", "interfaces","nets","nfvo_tenants","scenarios","sce_interfaces","sce_nets", "sce_vnfs","tenants_datacenters","datacenter_tenants","vms","vnfs", "datacenter_nets"] + class nfvo_db(db_base.db_base): def __init__(self, host=None, user=None, passwd=None, database=None, log_name='openmano.db', log_level=None): db_base.db_base.__init__(self, host, user, passwd, database, log_name, log_level) @@ -585,15 +586,19 @@ class nfvo_db(db_base.db_base): scenario_dict['vnfs'] = self.cur.fetchall() for vnf in scenario_dict['vnfs']: #sce_interfaces - cmd = "SELECT scei.uuid,scei.sce_net_id,scei.interface_id,i.external_name,scei.ip_address FROM sce_interfaces as scei join interfaces as i on scei.interface_id=i.uuid WHERE scei.sce_vnf_id='{}' ORDER BY scei.created_at".format(vnf['uuid']) + cmd = "SELECT scei.uuid,scei.sce_net_id,scei.interface_id,i.external_name,scei.ip_address"\ + " FROM sce_interfaces as scei join interfaces as i on scei.interface_id=i.uuid"\ + " WHERE scei.sce_vnf_id='{}' ORDER BY scei.created_at".format(vnf['uuid']) self.logger.debug(cmd) self.cur.execute(cmd) vnf['interfaces'] = self.cur.fetchall() #vms - cmd = "SELECT vms.uuid as uuid, flavor_id, image_id, vms.name as name, vms.description as description, vms.boot_data as boot_data " \ - " FROM vnfs join vms on vnfs.uuid=vms.vnf_id " \ - " WHERE vnfs.uuid='" + vnf['vnf_id'] +"'" \ - " ORDER BY vms.created_at" + cmd = "SELECT vms.uuid as uuid, flavor_id, image_id, vms.name as name," \ + " vms.description as description, vms.boot_data as boot_data, count," \ + " vms.availability_zone as availability_zone" \ + " FROM vnfs join vms on vnfs.uuid=vms.vnf_id" \ + " WHERE vnfs.uuid='" + vnf['vnf_id'] + "'" \ + " ORDER BY vms.created_at" self.logger.debug(cmd) self.cur.execute(cmd) vnf['vms'] = self.cur.fetchall() @@ -720,6 +725,43 @@ class nfvo_db(db_base.db_base): self._format_error(e, tries, "delete", "instances running") tries -= 1 + def new_rows(self, tables, uuid_list=None): + """ + Make a transactional insertion of rows at several tables + :param tables: list with dictionary where the keys are the table names and the values are a row or row list + with the values to be inserted at the table. Each row is a dictionary with the key values. E.g.: + tables = [ + {"table1": [ {"column1": value, "column2: value, ... }, {"column1": value, "column2: value, ... }, ...], + {"table2": [ {"column1": value, "column2: value, ... }, {"column1": value, "column2: value, ... }, ...], + {"table3": {"column1": value, "column2: value, ... } + } + :param uuid_list: list of created uuids, first one is the root (#TODO to store at uuid table) + :return: None if success, raise exception otherwise + """ + tries = 2 + while tries: + created_time = time.time() + try: + with self.con: + self.cur = self.con.cursor() + for table in tables: + for table_name, row_list in table.items(): + index = 0 + if isinstance(row_list, dict): + row_list = (row_list, ) #create a list with the single value + for row in row_list: + if table_name in self.tables_with_created_field: + created_time_param = created_time + index*0.00001 + else: + created_time_param=0 + self._new_row_internal(table_name, row, add_uuid=False, root_uuid=None, + created_time=created_time_param) + index += 1 + return + except (mdb.Error, AttributeError) as e: + self._format_error(e, tries) + tries -= 1 + def new_instance_scenario_as_a_whole(self,tenant_id,instance_scenario_name,instance_scenario_description,scenarioDict): tries = 2 while tries: diff --git a/osm_ro/openmano_schemas.py b/osm_ro/openmano_schemas.py index 7d218ec1..fb12d9f1 100644 --- a/osm_ro/openmano_schemas.py +++ b/osm_ro/openmano_schemas.py @@ -386,7 +386,7 @@ internal_connection_schema = { "name": name_schema, "description":description_schema, "type":{"type":"string", "enum":["bridge","data","ptp"]}, - "elements": {"type" : "array", "items": internal_connection_element_schema, "minItems":2} + "elements": {"type" : "array", "items": internal_connection_element_schema, "minItems":1} }, "required": ["name", "type", "elements"], "additionalProperties": False @@ -400,7 +400,7 @@ internal_connection_schema_v02 = { "type": {"type": "string", "enum":["e-line", "e-lan"]}, "implementation": {"type": "string", "enum":["overlay", "underlay"]}, "ip-profile": ip_profile_schema, - "elements": {"type" : "array", "items": internal_connection_element_schema_v02, "minItems":2} + "elements": {"type" : "array", "items": internal_connection_element_schema_v02, "minItems":1} }, "required": ["name", "type", "implementation", "elements"], "additionalProperties": False @@ -541,8 +541,10 @@ vnfc_schema = { "properties":{ "name": name_schema, "description": description_schema, - "VNFC image": {"oneOf": [path_schema, http_schema]}, + "count": integer1_schema, "image name": name_schema, + "availability_zone": name_schema, + "VNFC image": {"oneOf": [path_schema, http_schema]}, "image checksum": checksum_schema, "image metadata": metadata_schema, #"cloud-config": cloud_config_schema, #common for all vnfs in the scenario @@ -592,6 +594,7 @@ vnfd_schema_v01 = { "properties":{ "name": name_schema, "description": description_schema, + "class": nameshort_schema, "public": {"type" : "boolean"}, "physical": {"type" : "boolean"}, diff --git a/osm_ro/vim_thread.py b/osm_ro/vim_thread.py index 9f396a26..0074dfe9 100644 --- a/osm_ro/vim_thread.py +++ b/osm_ro/vim_thread.py @@ -144,8 +144,9 @@ class vim_thread(threading.Thread): #delete old port if task_interface.get("sdn_port_id"): try: - self.ovim.delete_port(task_interface["sdn_port_id"]) - task_interface["sdn_port_id"] = None + with self.db_lock: + self.ovim.delete_port(task_interface["sdn_port_id"]) + task_interface["sdn_port_id"] = None except ovimException as e: self.logger.error("ovimException deleting external_port={} ".format( task_interface["sdn_port_id"]) + str(e), exc_info=True) @@ -174,11 +175,15 @@ class vim_thread(threading.Thread): continue else: db_iface = db_ifaces[0] - #If there is no sdn_net_id, check if it is because an already created vim network is being used - #in that case, the sdn_net_id will be in that entry of the instance_nets table + # If there is no sdn_net_id, check if it is because an already created vim network is being used + # in that case, the sdn_net_id will be in that entry of the instance_nets table if not db_iface.get("sdn_net_id"): - result = self.db.get_rows(SELECT=('sdn_net_id',), FROM='instance_nets', - WHERE={'vim_net_id': db_iface.get("vim_net_id"), 'instance_scenario_id': None, "datacenter_tenant_id": self.datacenter_tenant_id}) + with self.db_lock: + result = self.db.get_rows( + SELECT=('sdn_net_id',), FROM='instance_nets', + WHERE={'vim_net_id': db_iface.get("vim_net_id"), + 'instance_scenario_id': None, + 'datacenter_tenant_id': self.datacenter_tenant_id}) if len(result) == 1: db_iface["sdn_net_id"] = result[0]['sdn_net_id'] @@ -187,15 +192,16 @@ class vim_thread(threading.Thread): sdn_port_name = sdn_net_id + "." + db_iface["vm_id"] sdn_port_name = sdn_port_name[:63] try: - sdn_port_id = self.ovim.new_external_port( - {"compute_node": interface["compute_node"], - "pci": interface["pci"], - "vlan": interface.get("vlan"), - "net_id": sdn_net_id, - "region": self.vim["config"]["datacenter_id"], - "name": sdn_port_name, - "mac": interface.get("mac_address")}) - interface["sdn_port_id"] = sdn_port_id + with self.db_lock: + sdn_port_id = self.ovim.new_external_port( + {"compute_node": interface["compute_node"], + "pci": interface["pci"], + "vlan": interface.get("vlan"), + "net_id": sdn_net_id, + "region": self.vim["config"]["datacenter_id"], + "name": sdn_port_name, + "mac": interface.get("mac_address")}) + interface["sdn_port_id"] = sdn_port_id except (ovimException, Exception) as e: self.logger.error( "ovimException creating new_external_port compute_node={} " \ @@ -248,7 +254,8 @@ class vim_thread(threading.Thread): if db_net.get("sdn_net_id"): # get ovim status try: - sdn_net = self.ovim.show_network(db_net["sdn_net_id"]) + with self.db_lock: + sdn_net = self.ovim.show_network(db_net["sdn_net_id"]) if sdn_net["status"] == "ERROR": if not vim_info.get("error_msg"): vim_info["error_msg"] = sdn_net["error_msg"] @@ -421,7 +428,8 @@ class vim_thread(threading.Thread): net_name, net_type, vim_net['encapsulation'])) network["vlan"] = vim_net.get('segmentation_id') try: - sdn_net_id = self.ovim.new_network(network) + with self.db_lock: + sdn_net_id = self.ovim.new_network(network) except (ovimException, Exception) as e: self.logger.error("task=%s cannot create SDN network vim_net_id=%s input='%s' ovimException='%s'", str(task_id), net_id, str(network), str(e)) @@ -531,7 +539,8 @@ class vim_thread(threading.Thread): for iface in interfaces: if iface.get("sdn_port_id"): try: - self.ovim.delete_port(iface["sdn_port_id"]) + with self.db_lock: + self.ovim.delete_port(iface["sdn_port_id"]) except ovimException as e: self.logger.error("ovimException deleting external_port={} at VM vim_id={} deletion ".format( iface["sdn_port_id"], vm_id) + str(e), exc_info=True) @@ -559,17 +568,19 @@ class vim_thread(threading.Thread): self._remove_refresh("get-net", net_id) result = self.vim.delete_network(net_id) if sdn_net_id: - #Delete any attached port to this sdn network - #At this point, there will be ports associated to this network in case it was manually done using 'openmano vim-net-sdn-attach' + # Delete any attached port to this sdn network + # At this point, there will be ports associated to this network in case it was manually done using 'openmano vim-net-sdn-attach' try: - port_list = self.ovim.get_ports(columns={'uuid'}, filter={'name': 'external_port', 'net_id': sdn_net_id}) + with self.db_lock: + port_list = self.ovim.get_ports(columns={'uuid'}, filter={'name': 'external_port', 'net_id': sdn_net_id}) except ovimException as e: raise vimconn.vimconnException( "ovimException obtaining external ports for net {}. ".format(sdn_net_id) + str(e)) for port in port_list: try: - self.ovim.delete_port(port['uuid']) + with self.db_lock: + self.ovim.delete_port(port['uuid']) except ovimException as e: raise vimconn.vimconnException( "ovimException deleting port {} for net {}. ".format(port['uuid'], sdn_net_id) + str(e)) diff --git a/osm_ro/vimconn.py b/osm_ro/vimconn.py index 18f4334d..7adaa366 100644 --- a/osm_ro/vimconn.py +++ b/osm_ro/vimconn.py @@ -82,6 +82,7 @@ class vimconnNotImplemented(vimconnException): def __init__(self, message, http_code=HTTP_Not_Implemented): vimconnException.__init__(self, message, http_code) + class vimconnector(): """Abstract base class for all the VIM connector plugins These plugins must implement a vimconnector class derived from this @@ -115,6 +116,7 @@ class vimconnector(): self.user = user self.passwd = passwd self.config = config + self.availability_zone = None self.logger = logging.getLogger('openmano.vim') if log_level: self.logger.setLevel( getattr(logging, log_level) ) @@ -353,8 +355,8 @@ class vimconnector(): """ raise vimconnNotImplemented( "Should have implemented this" ) - def new_vminstance(self, name, description, start, image_id, flavor_id, net_list, cloud_config=None, - disk_list=None): + def new_vminstance(self, name, description, start, image_id, flavor_id, net_list, cloud_config=None, disk_list=None, + availability_zone_index=None, availability_zone_list=None): """Adds a VM instance to VIM Params: 'start': (boolean) indicates if VM must start or created in pause mode. @@ -385,7 +387,8 @@ class vimconnector(): 'users': (optional) list of users to be inserted, each item is a dict with: 'name': (mandatory) user name, 'key-pairs': (optional) list of strings with the public key to be inserted to the user - 'user-data': (optional) string is a text script to be passed directly to cloud-init + 'user-data': (optional) can be a string with the text script to be passed directly to cloud-init, + or a list of strings, each one contains a script to be passed, usually with a MIMEmultipart file 'config-files': (optional). List of files to be transferred. Each item is a dict with: 'dest': (mandatory) string with the destination absolute path 'encoding': (optional, by default text). Can be one of: @@ -397,6 +400,9 @@ class vimconnector(): 'disk_list': (optional) list with additional disks to the VM. Each item is a dict with: 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted 'size': (mandatory) string with the size of the disk in GB + availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required + availability_zone_list: list of availability zones given by user in the VNFD descriptor. Ignore if + availability_zone_index is None Returns the instance identifier or raises an exception on error """ raise vimconnNotImplemented( "Should have implemented this" ) @@ -458,7 +464,7 @@ class vimconnector(): suffix: extra text, e.g. the http path and query string """ raise vimconnNotImplemented( "Should have implemented this" ) - + #NOT USED METHODS in current version def host_vim2gui(self, host, server_dict): diff --git a/osm_ro/vimconn_aws.py b/osm_ro/vimconn_aws.py index f0eebb66..3cccbfcf 100644 --- a/osm_ro/vimconn_aws.py +++ b/osm_ro/vimconn_aws.py @@ -351,12 +351,12 @@ class vimconnector(vimconn.vimconnector): if filter_dict != {}: if 'tenant_id' in filter_dict: tfilters['vpcId'] = filter_dict['tenant_id'] - subnets = self.conn_vpc.get_all_subnets(subnet_ids=filter_dict.get('id', None), filters=tfilters) + subnets = self.conn_vpc.get_all_subnets(subnet_ids=filter_dict.get('name', None), filters=tfilters) net_list = [] for net in subnets: net_list.append( {'id': str(net.id), 'name': str(net.id), 'status': str(net.state), 'vpc_id': str(net.vpc_id), - 'cidr_block': str(net.cidr_block)}) + 'cidr_block': str(net.cidr_block), 'type': 'bridge'}) return net_list except Exception as e: self.format_vimconn_exception(e) @@ -590,7 +590,7 @@ class vimconnector(vimconn.vimconnector): self.format_vimconn_exception(e) def new_vminstance(self, name, description, start, image_id, flavor_id, net_list, cloud_config=None, - disk_list=None): + disk_list=None, availability_zone_index=None, availability_zone_list=None): """Create a new VM/instance in AWS Params: name decription @@ -796,7 +796,10 @@ class vimconnector(vimconn.vimconnector): interface_dict['vim_interface_id'] = interface.id interface_dict['vim_net_id'] = interface.subnet_id interface_dict['mac_address'] = interface.mac_address - interface_dict['ip_address'] = interface.private_ip_address + if hasattr(interface, 'publicIp') and interface.publicIp != None: + interface_dict['ip_address'] = interface.publicIp + ";" + interface.private_ip_address + else: + interface_dict['ip_address'] = interface.private_ip_address instance_dict['interfaces'].append(interface_dict) except Exception as e: self.logger.error("Exception getting vm status: %s", str(e), exc_info=True) diff --git a/osm_ro/vimconn_openstack.py b/osm_ro/vimconn_openstack.py index b280da89..319f8c1a 100644 --- a/osm_ro/vimconn_openstack.py +++ b/osm_ro/vimconn_openstack.py @@ -35,6 +35,7 @@ import netaddr import time import yaml import random +import sys import re from novaclient import client as nClient, exceptions as nvExceptions @@ -51,8 +52,11 @@ from httplib import HTTPException from neutronclient.neutron import client as neClient from neutronclient.common import exceptions as neExceptions from requests.exceptions import ConnectionError +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText -'''contain the openstack virtual machine status to openmano status''' + +"""contain the openstack virtual machine status to openmano status""" vmStatus2manoFormat={'ACTIVE':'ACTIVE', 'PAUSED':'PAUSED', 'SUSPENDED': 'SUSPENDED', @@ -65,7 +69,7 @@ netStatus2manoFormat={'ACTIVE':'ACTIVE','PAUSED':'PAUSED','INACTIVE':'INACTIVE', #global var to have a timeout creating and deleting volumes volume_timeout = 60 -server_timeout = 60 +server_timeout = 300 class vimconnector(vimconn.vimconnector): def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None, @@ -94,6 +98,7 @@ class vimconnector(vimconn.vimconnector): if not url: raise TypeError, 'url param can not be NoneType' self.persistent_info = persistent_info + self.availability_zone = persistent_info.get('availability_zone', None) self.session = persistent_info.get('session', {'reload_client': True}) self.nova = self.session.get('nova') self.neutron = self.session.get('neutron') @@ -117,7 +122,7 @@ class vimconnector(vimconn.vimconnector): self.logger = logging.getLogger('openmano.vim.vio') if log_level: - self.logger.setLevel(getattr(logging, log_level)) + self.logger.setLevel( getattr(logging, log_level)) def __getitem__(self, index): """Get individuals parameters. @@ -171,7 +176,16 @@ class vimconnector(vimconn.vimconnector): else: self.keystone = ksClient_v2.Client(session=sess, endpoint_type=self.endpoint_type) self.session['keystone'] = self.keystone - self.nova = self.session['nova'] = nClient.Client("2.1", session=sess, endpoint_type=self.endpoint_type) + # In order to enable microversion functionality an explicit microversion must be specified in 'config'. + # This implementation approach is due to the warning message in + # https://developer.openstack.org/api-guide/compute/microversions.html + # where it is stated that microversion backwards compatibility is not guaranteed and clients should + # always require an specific microversion. + # To be able to use 'device role tagging' functionality define 'microversion: 2.32' in datacenter config + version = self.config.get("microversion") + if not version: + version = "2.1" + self.nova = self.session['nova'] = nClient.Client(str(version), session=sess, endpoint_type=self.endpoint_type) self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess, endpoint_type=self.endpoint_type) self.cinder = self.session['cinder'] = cClient.Client(2, session=sess, endpoint_type=self.endpoint_type) if self.endpoint_type == "internalURL": @@ -185,6 +199,9 @@ class vimconnector(vimconn.vimconnector): endpoint=glance_endpoint) self.session['reload_client'] = False self.persistent_info['session'] = self.session + # add availablity zone info inside self.persistent_info + self._set_availablity_zones() + self.persistent_info['availability_zone'] = self.availability_zone def __net_os2mano(self, net_list_dict): '''Transform the net openstack format to mano format @@ -214,7 +231,10 @@ class vimconnector(vimconn.vimconnector): raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception)) elif isinstance(exception, nvExceptions.Conflict): raise vimconn.vimconnConflictException(type(exception).__name__ + ": " + str(exception)) + elif isinstance(exception, vimconn.vimconnException): + raise else: # () + self.logger.error("General Exception " + str(exception), exc_info=True) raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception)) def get_tenant_list(self, filter_dict={}): @@ -308,19 +328,19 @@ class vimconnector(vimconn.vimconnector): ip_profile['subnet_address'] = "192.168.{}.0/24".format(subnet_rand) if 'ip_version' not in ip_profile: ip_profile['ip_version'] = "IPv4" - subnet={"name":net_name+"-subnet", + subnet = {"name":net_name+"-subnet", "network_id": new_net["network"]["id"], "ip_version": 4 if ip_profile['ip_version']=="IPv4" else 6, "cidr": ip_profile['subnet_address'] } - if 'gateway_address' in ip_profile: - subnet['gateway_ip'] = ip_profile['gateway_address'] + # Gateway should be set to None if not needed. Otherwise openstack assigns one by default + subnet['gateway_ip'] = ip_profile.get('gateway_address') if ip_profile.get('dns_address'): subnet['dns_nameservers'] = ip_profile['dns_address'].split(";") if 'dhcp_enabled' in ip_profile: subnet['enable_dhcp'] = False if ip_profile['dhcp_enabled']=="false" else True if 'dhcp_start_address' in ip_profile: - subnet['allocation_pools']=[] + subnet['allocation_pools'] = [] subnet['allocation_pools'].append(dict()) subnet['allocation_pools'][0]['start'] = ip_profile['dhcp_start_address'] if 'dhcp_count' in ip_profile: @@ -567,11 +587,11 @@ class vimconnector(vimconn.vimconnector): # if interface["dedicated"]=="yes": # raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable) # #TODO, add the key 'pci_passthrough:alias"="