X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_ro%2Fnfvo.py;h=bd9d3689f173df9330822865e4fc88cc6e14bb31;hb=66eba6ece53cd85d0efbe8b4ff4f414c812b347b;hp=d45a3c79865436d3f1238aa520489f91ab4f70fb;hpb=3e8e11a11215e19e41be646d9d4dcbe7f67b1ebd;p=osm%2FRO.git diff --git a/osm_ro/nfvo.py b/osm_ro/nfvo.py index d45a3c79..bd9d3689 100644 --- a/osm_ro/nfvo.py +++ b/osm_ro/nfvo.py @@ -38,13 +38,21 @@ import console_proxy_thread as cli import vimconn import logging import collections +import math +from uuid import uuid4 from db_base import db_base_Exception import nfvo_db from threading import Lock -from time import time +import time as t from lib_osm_openvim import ovim as ovim_module from lib_osm_openvim.ovim import ovimException +from Crypto.PublicKey import RSA + +import osm_im.vnfd as vnfd_catalog +import osm_im.nsd as nsd_catalog +from pyangbind.lib.serialise import pybindJSONDecoder +from itertools import chain global global_config global vimconn_imported @@ -60,10 +68,9 @@ vim_threads = {"running":{}, "deleting": {}, "names": []} # threads running vim_persistent_info = {} logger = logging.getLogger('openmano.nfvo') task_lock = Lock() -global_instance_tasks = {} last_task_id = 0.0 -db=None -db_lock=Lock() +db = None +db_lock = Lock() class NfvoException(Exception): def __init__(self, message, http_code): @@ -73,14 +80,16 @@ class NfvoException(Exception): def get_task_id(): global last_task_id - task_id = time() + task_id = t.time() if task_id <= last_task_id: task_id = last_task_id + 0.000001 last_task_id = task_id - return "TASK.{:.6f}".format(task_id) + return "ACTION-{:.6f}".format(task_id) + # return (t.strftime("%Y%m%dT%H%M%S.{}%Z", t.localtime(task_id))).format(int((task_id % 1)*1e6)) def new_task(name, params, depends=None): + """Deprected!!!""" task_id = get_task_id() task = {"status": "enqueued", "id": task_id, "name": name, "params": params} if depends: @@ -89,7 +98,7 @@ def new_task(name, params, depends=None): def is_task_id(id): - return True if id[:5] == "TASK." else False + return True if id[:5] == "TASK-" else False def get_non_used_vim_name(datacenter_name, datacenter_id, tenant_name, tenant_id): @@ -130,9 +139,14 @@ def start_service(mydb): 'log_level_of': 'DEBUG' } try: + # starts ovim library ovim = ovim_module.ovim(ovim_configuration) ovim.start_service() + #delete old unneeded vim_actions + clean_db(mydb) + + # starts vim_threads from_= 'tenants_datacenters as td join datacenters as d on td.datacenter_id=d.uuid join '\ 'datacenter_tenants as dt on td.datacenter_tenant_id=dt.uuid' select_ = ('type', 'd.config as config', 'd.uuid as datacenter_id', 'vim_url', 'vim_url_admin', @@ -174,6 +188,10 @@ def start_service(mydb): user=vim['user'], passwd=vim['passwd'], config=extra, persistent_info=vim_persistent_info[thread_id] ) + except vimconn.vimconnException as e: + myvim = e + logger.error("Cannot launch thread for VIM {} '{}': {}".format(vim['datacenter_name'], + vim['datacenter_id'], e)) except Exception as e: raise NfvoException("Error at VIM {}; {}: {}".format(vim["type"], type(e).__name__, e), HTTP_Internal_Server_Error) @@ -201,7 +219,7 @@ def stop_service(): if ovim: ovim.stop_service() for thread_id,thread in vim_threads["running"].items(): - thread.insert_task(new_task("exit", None)) + thread.insert_task("exit") vim_threads["deleting"][thread_id] = thread vim_threads["running"] = {} if global_config and global_config.get("console_thread"): @@ -212,6 +230,37 @@ def get_version(): return ("openmanod version {} {}\n(c) Copyright Telefonica".format(global_config["version"], global_config["version_date"] )) +def clean_db(mydb): + """ + Clean unused or old entries at database to avoid unlimited growing + :param mydb: database connector + :return: None + """ + # get and delete unused vim_actions: all elements deleted, one week before, instance not present + now = t.time()-3600*24*7 + instance_action_id = None + nb_deleted = 0 + while True: + actions_to_delete = mydb.get_rows( + SELECT=("item", "item_id", "instance_action_id"), + FROM="vim_actions as va join instance_actions as ia on va.instance_action_id=ia.uuid " + "left join instance_scenarios as i on ia.instance_id=i.uuid", + WHERE={"va.action": "DELETE", "va.modified_at<": now, "i.uuid": None, + "va.status": ("DONE", "SUPERSEDED")}, + LIMIT=100 + ) + for to_delete in actions_to_delete: + mydb.delete_row(FROM="vim_actions", WHERE=to_delete) + if instance_action_id != to_delete["instance_action_id"]: + instance_action_id = to_delete["instance_action_id"] + mydb.delete_row(FROM="instance_actions", WHERE={"uuid": instance_action_id}) + nb_deleted += len(actions_to_delete) + if len(actions_to_delete) < 100: + break + if nb_deleted: + logger.debug("Removed {} unused vim_actions".format(nb_deleted)) + + def get_flavorlist(mydb, vnf_id, nfvo_tenant=None): '''Obtain flavorList @@ -334,11 +383,13 @@ def rollback(mydb, vims, rollback_list): if item["where"]=="vim": if item["vim_id"] not in vims: continue - vim=vims[ item["vim_id"] ] + if is_task_id(item["uuid"]): + continue + vim = vims[item["vim_id"]] try: if item["what"]=="image": vim.delete_image(item["uuid"]) - mydb.delete_row(FROM="datacenters_images", WHERE={"datacenter_id": vim["id"], "vim_id":item["uuid"]}) + mydb.delete_row(FROM="datacenters_images", WHERE={"datacenter_vim_id": vim["id"], "vim_id":item["uuid"]}) elif item["what"]=="flavor": vim.delete_flavor(item["uuid"]) mydb.delete_row(FROM="datacenters_flavors", WHERE={"datacenter_id": vim["id"], "vim_id":item["uuid"]}) @@ -392,12 +443,12 @@ def check_vnf_descriptor(vnf_descriptor, vnf_descriptor_version=1): name_dict[ interface["name"] ] = "overlay" vnfc_interfaces[ vnfc["name"] ] = name_dict # check bood-data info - if "boot-data" in vnfc: - # check that user-data is incompatible with users and config-files - if (vnfc["boot-data"].get("users") or vnfc["boot-data"].get("config-files")) and vnfc["boot-data"].get("user-data"): - raise NfvoException( - "Error at vnf:VNFC:boot-data, fields 'users' and 'config-files' are not compatible with 'user-data'", - HTTP_Bad_Request) + # if "boot-data" in vnfc: + # # check that user-data is incompatible with users and config-files + # if (vnfc["boot-data"].get("users") or vnfc["boot-data"].get("config-files")) and vnfc["boot-data"].get("user-data"): + # raise NfvoException( + # "Error at vnf:VNFC:boot-data, fields 'users' and 'config-files' are not compatible with 'user-data'", + # HTTP_Bad_Request) #check if the info in external_connections matches with the one in the vnfcs name_list=[] @@ -481,7 +532,7 @@ def check_vnf_descriptor(vnf_descriptor, vnf_descriptor_version=1): HTTP_Bad_Request) -def create_or_use_image(mydb, vims, image_dict, rollback_list, only_create_at_vim=False, return_on_error = None): +def create_or_use_image(mydb, vims, image_dict, rollback_list, only_create_at_vim=False, return_on_error=None): #look if image exist if only_create_at_vim: image_mano_id = image_dict['uuid'] @@ -505,9 +556,11 @@ def create_or_use_image(mydb, vims, image_dict, rollback_list, only_create_at_vi rollback_list.append({"where":"mano", "what":"image","uuid":image_mano_id}) #create image at every vim for vim_id,vim in vims.iteritems(): + datacenter_vim_id = vim["config"]["datacenter_tenant_id"] image_created="false" #look at database - image_db = mydb.get_rows(FROM="datacenters_images", WHERE={'datacenter_id':vim_id, 'image_id':image_mano_id}) + image_db = mydb.get_rows(FROM="datacenters_images", + WHERE={'datacenter_vim_id': datacenter_vim_id, 'image_id': image_mano_id}) #look at VIM if this image exist try: if image_dict['location'] is not None: @@ -556,10 +609,13 @@ def create_or_use_image(mydb, vims, image_dict, rollback_list, only_create_at_vi #if we reach here, the image has been created or existed if len(image_db)==0: #add new vim_id at datacenters_images - mydb.new_row('datacenters_images', {'datacenter_id':vim_id, 'image_id':image_mano_id, 'vim_id': image_vim_id, 'created':image_created}) + mydb.new_row('datacenters_images', {'datacenter_vim_id': datacenter_vim_id, + 'image_id':image_mano_id, + 'vim_id': image_vim_id, + 'created':image_created}) elif image_db[0]["vim_id"]!=image_vim_id: #modify existing vim_id at datacenters_images - mydb.update_rows('datacenters_images', UPDATE={'vim_id':image_vim_id}, WHERE={'datacenter_id':vim_id, 'image_id':image_mano_id}) + mydb.update_rows('datacenters_images', UPDATE={'vim_id':image_vim_id}, WHERE={'datacenter_vim_id':vim_id, 'image_id':image_mano_id}) return image_vim_id if only_create_at_vim else image_mano_id @@ -618,9 +674,11 @@ def create_or_use_flavor(mydb, vims, flavor_dict, rollback_list, only_create_at_ del flavor_dict['uuid'] flavor_vim_id=None for vim_id,vim in vims.items(): + datacenter_vim_id = vim["config"]["datacenter_tenant_id"] flavor_created="false" #look at database - flavor_db = mydb.get_rows(FROM="datacenters_flavors", WHERE={'datacenter_id':vim_id, 'flavor_id':flavor_mano_id}) + flavor_db = mydb.get_rows(FROM="datacenters_flavors", + WHERE={'datacenter_vim_id': datacenter_vim_id, 'flavor_id': flavor_mano_id}) #look at VIM if this flavor exist SKIPPED #res_vim, flavor_vim_id = vim.get_flavor_id_from_path(flavor_dict['location']) #if res_vim < 0: @@ -628,11 +686,11 @@ def create_or_use_flavor(mydb, vims, flavor_dict, rollback_list, only_create_at_ # continue #elif res_vim==0: - #Create the flavor in VIM - #Translate images at devices from MANO id to VIM id + # Create the flavor in VIM + # Translate images at devices from MANO id to VIM id disk_list = [] if 'extended' in flavor_dict and flavor_dict['extended']!=None and "devices" in flavor_dict['extended']: - #make a copy of original devices + # make a copy of original devices devices_original=[] for device in flavor_dict["extended"].get("devices",[]): @@ -643,7 +701,9 @@ def create_or_use_flavor(mydb, vims, flavor_dict, rollback_list, only_create_at_ del device['image'] if 'image metadata' in device: del device['image metadata'] - dev_nb=0 + if 'image checksum' in device: + del device['image checksum'] + dev_nb = 0 for index in range(0,len(devices_original)) : device=devices_original[index] if "image" not in device and "image name" not in device: @@ -655,7 +715,7 @@ def create_or_use_flavor(mydb, vims, flavor_dict, rollback_list, only_create_at_ image_dict['universal_name']=device.get('image name') image_dict['description']=flavor_dict['name']+str(dev_nb)+"-img" image_dict['location']=device.get('image') - #image_dict['new_location']=device.get('image location') + # image_dict['new_location']=device.get('image location') image_dict['checksum']=device.get('image checksum') image_metadata_dict = device.get('image metadata', None) image_metadata_str = None @@ -708,15 +768,448 @@ def create_or_use_flavor(mydb, vims, flavor_dict, rollback_list, only_create_at_ extended_devices['disks'] = disk_list extended_devices_yaml = yaml.safe_dump(extended_devices,default_flow_style=True,width=256) mydb.new_row('datacenters_flavors', - {'datacenter_id':vim_id, 'flavor_id':flavor_mano_id, 'vim_id': flavor_vim_id, - 'created':flavor_created,'extended': extended_devices_yaml}) + {'datacenter_vim_id': datacenter_vim_id, 'flavor_id': flavor_mano_id, 'vim_id': flavor_vim_id, + 'created': flavor_created, 'extended': extended_devices_yaml}) elif flavor_db[0]["vim_id"]!=flavor_vim_id: #modify existing vim_id at datacenters_flavors - mydb.update_rows('datacenters_flavors', UPDATE={'vim_id':flavor_vim_id}, WHERE={'datacenter_id':vim_id, 'flavor_id':flavor_mano_id}) + mydb.update_rows('datacenters_flavors', UPDATE={'vim_id':flavor_vim_id}, + WHERE={'datacenter_vim_id': datacenter_vim_id, 'flavor_id': flavor_mano_id}) return flavor_vim_id if only_create_at_vim else flavor_mano_id +def get_str(obj, field, length): + """ + Obtain the str value, + :param obj: + :param length: + :return: + """ + value = obj.get(field) + if value is not None: + value = str(value)[:length] + return value + +def _lookfor_or_create_image(db_image, mydb, descriptor): + """ + fill image content at db_image dictionary. Check if the image with this image and checksum exist + :param db_image: dictionary to insert data + :param mydb: database connector + :param descriptor: yang descriptor + :return: uuid if the image exist at DB, or None if a new image must be created with the data filled at db_image + """ + + db_image["name"] = get_str(descriptor, "image", 255) + db_image["checksum"] = get_str(descriptor, "image-checksum", 32) + if not db_image["checksum"]: # Ensure that if empty string, None is stored + db_image["checksum"] = None + if db_image["name"].startswith("/"): + db_image["location"] = db_image["name"] + existing_images = mydb.get_rows(FROM="images", WHERE={'location': db_image["location"]}) + else: + db_image["universal_name"] = db_image["name"] + existing_images = mydb.get_rows(FROM="images", WHERE={'universal_name': db_image['universal_name'], + 'checksum': db_image['checksum']}) + if existing_images: + return existing_images[0]["uuid"] + else: + image_uuid = str(uuid4()) + db_image["uuid"] = image_uuid + return None + +def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): + """ + Parses an OSM IM vnfd_catalog and insert at DB + :param mydb: + :param tenant_id: + :param vnf_descriptor: + :return: The list of cretated vnf ids + """ + try: + myvnfd = vnfd_catalog.vnfd() + try: + pybindJSONDecoder.load_ietf_json(vnf_descriptor, None, None, obj=myvnfd) + except Exception as e: + raise NfvoException("Error. Invalid VNF descriptor format " + str(e), HTTP_Bad_Request) + db_vnfs = [] + db_nets = [] + db_vms = [] + db_vms_index = 0 + db_interfaces = [] + db_images = [] + db_flavors = [] + uuid_list = [] + vnfd_uuid_list = [] + vnfd_catalog_descriptor = vnf_descriptor.get("vnfd:vnfd-catalog") + if not vnfd_catalog_descriptor: + vnfd_catalog_descriptor = vnf_descriptor.get("vnfd-catalog") + vnfd_descriptor_list = vnfd_catalog_descriptor.get("vnfd") + if not vnfd_descriptor_list: + vnfd_descriptor_list = vnfd_catalog_descriptor.get("vnfd:vnfd") + for vnfd_yang in myvnfd.vnfd_catalog.vnfd.itervalues(): + vnfd = vnfd_yang.get() + + # table vnf + vnf_uuid = str(uuid4()) + uuid_list.append(vnf_uuid) + vnfd_uuid_list.append(vnf_uuid) + vnfd_id = get_str(vnfd, "id", 255) + db_vnf = { + "uuid": vnf_uuid, + "osm_id": vnfd_id, + "name": get_str(vnfd, "name", 255), + "description": get_str(vnfd, "description", 255), + "tenant_id": tenant_id, + "vendor": get_str(vnfd, "vendor", 255), + "short_name": get_str(vnfd, "short-name", 255), + "descriptor": str(vnf_descriptor)[:60000] + } + + for vnfd_descriptor in vnfd_descriptor_list: + if vnfd_descriptor["id"] == str(vnfd["id"]): + break + + # table nets (internal-vld) + net_id2uuid = {} # for mapping interface with network + for vld in vnfd.get("internal-vld").itervalues(): + net_uuid = str(uuid4()) + uuid_list.append(net_uuid) + db_net = { + "name": get_str(vld, "name", 255), + "vnf_id": vnf_uuid, + "uuid": net_uuid, + "description": get_str(vld, "description", 255), + "type": "bridge", # TODO adjust depending on connection point type + } + net_id2uuid[vld.get("id")] = net_uuid + db_nets.append(db_net) + + # table vms (vdus) + vdu_id2uuid = {} + vdu_id2db_table_index = {} + for vdu in vnfd.get("vdu").itervalues(): + vm_uuid = str(uuid4()) + uuid_list.append(vm_uuid) + vdu_id = get_str(vdu, "id", 255) + db_vm = { + "uuid": vm_uuid, + "osm_id": vdu_id, + "name": get_str(vdu, "name", 255), + "description": get_str(vdu, "description", 255), + "vnf_id": vnf_uuid, + } + vdu_id2uuid[db_vm["osm_id"]] = vm_uuid + vdu_id2db_table_index[db_vm["osm_id"]] = db_vms_index + if vdu.get("count"): + db_vm["count"] = int(vdu["count"]) + + # table image + image_present = False + if vdu.get("image"): + image_present = True + db_image = {} + image_uuid = _lookfor_or_create_image(db_image, mydb, vdu) + if not image_uuid: + image_uuid = db_image["uuid"] + db_images.append(db_image) + db_vm["image_id"] = image_uuid + + # volumes + devices = [] + if vdu.get("volumes"): + for volume_key in sorted(vdu["volumes"]): + volume = vdu["volumes"][volume_key] + if not image_present: + # Convert the first volume to vnfc.image + image_present = True + db_image = {} + image_uuid = _lookfor_or_create_image(db_image, mydb, volume) + if not image_uuid: + image_uuid = db_image["uuid"] + db_images.append(db_image) + db_vm["image_id"] = image_uuid + else: + # Add Openmano devices + device = {} + device["type"] = str(volume.get("device-type")) + if volume.get("size"): + device["size"] = int(volume["size"]) + if volume.get("image"): + device["image name"] = str(volume["image"]) + if volume.get("image-checksum"): + device["image checksum"] = str(volume["image-checksum"]) + devices.append(device) + + # cloud-init + boot_data = {} + if vdu.get("cloud-init"): + boot_data["user-data"] = str(vdu["cloud-init"]) + elif vdu.get("cloud-init-file"): + # TODO Where this file content is present??? + # boot_data["user-data"] = vnfd_yang.files[vdu["cloud-init-file"]] + boot_data["user-data"] = str(vdu["cloud-init-file"]) + + if vdu.get("supplemental-boot-data"): + if vdu["supplemental-boot-data"].get('boot-data-drive'): + boot_data['boot-data-drive'] = True + if vdu["supplemental-boot-data"].get('config-file'): + om_cfgfile_list = list() + for custom_config_file in vdu["supplemental-boot-data"]['config-file'].itervalues(): + # TODO Where this file content is present??? + cfg_source = str(custom_config_file["source"]) + om_cfgfile_list.append({"dest": custom_config_file["dest"], + "content": cfg_source}) + boot_data['config-files'] = om_cfgfile_list + if boot_data: + db_vm["boot_data"] = yaml.safe_dump(boot_data, default_flow_style=True, width=256) + + db_vms.append(db_vm) + db_vms_index += 1 + + # table interfaces (internal/external interfaces) + flavor_epa_interfaces = [] + cp_name2iface_uuid = {} + cp_name2vm_uuid = {} + cp_name2db_interface = {} + vdu_id2cp_name = {} # stored only when one external connection point is presented at this VDU + # for iface in chain(vdu.get("internal-interface").itervalues(), vdu.get("external-interface").itervalues()): + for iface in vdu.get("interface").itervalues(): + flavor_epa_interface = {} + iface_uuid = str(uuid4()) + uuid_list.append(iface_uuid) + db_interface = { + "uuid": iface_uuid, + "internal_name": get_str(iface, "name", 255), + "vm_id": vm_uuid, + } + flavor_epa_interface["name"] = db_interface["internal_name"] + if iface.get("virtual-interface").get("vpci"): + db_interface["vpci"] = get_str(iface.get("virtual-interface"), "vpci", 12) + flavor_epa_interface["vpci"] = db_interface["vpci"] + + if iface.get("virtual-interface").get("bandwidth"): + bps = int(iface.get("virtual-interface").get("bandwidth")) + db_interface["bw"] = int(math.ceil(bps/1000000.0)) + flavor_epa_interface["bandwidth"] = "{} Mbps".format(db_interface["bw"]) + + if iface.get("virtual-interface").get("type") == "OM-MGMT": + db_interface["type"] = "mgmt" + elif iface.get("virtual-interface").get("type") in ("VIRTIO", "E1000"): + db_interface["type"] = "bridge" + db_interface["model"] = get_str(iface.get("virtual-interface"), "type", 12) + elif iface.get("virtual-interface").get("type") in ("SR-IOV", "PCI-PASSTHROUGH"): + db_interface["type"] = "data" + db_interface["model"] = get_str(iface.get("virtual-interface"), "type", 12) + flavor_epa_interface["dedicated"] = "no" if iface["virtual-interface"]["type"] == "SR-IOV" \ + else "yes" + flavor_epa_interfaces.append(flavor_epa_interface) + else: + raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{}]':'vdu[{}]':'interface':'virtual" + "-interface':'type':'{}'. Interface type is not supported".format( + vnfd_id, vdu_id, iface.get("virtual-interface").get("type")), + HTTP_Bad_Request) + + if iface.get("external-connection-point-ref"): + try: + cp = vnfd.get("connection-point")[iface.get("external-connection-point-ref")] + db_interface["external_name"] = get_str(cp, "name", 255) + cp_name2iface_uuid[db_interface["external_name"]] = iface_uuid + cp_name2vm_uuid[db_interface["external_name"]] = vm_uuid + cp_name2db_interface[db_interface["external_name"]] = db_interface + for cp_descriptor in vnfd_descriptor["connection-point"]: + if cp_descriptor["name"] == db_interface["external_name"]: + break + else: + raise KeyError() + + if vdu_id in vdu_id2cp_name: + vdu_id2cp_name[vdu_id] = None # more than two connecdtion point for this VDU + else: + vdu_id2cp_name[vdu_id] = db_interface["external_name"] + + # port security + if str(cp_descriptor.get("port-security-enabled")).lower() == "false": + db_interface["port_security"] = 0 + elif str(cp_descriptor.get("port-security-enabled")).lower() == "true": + db_interface["port_security"] = 1 + except KeyError: + raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'vdu[{vdu}]':" + "'interface[{iface}]':'vnfd-connection-point-ref':'{cp}' is not present" + " at connection-point".format( + vnf=vnfd_id, vdu=vdu_id, iface=iface["name"], + cp=iface.get("vnfd-connection-point-ref")), + HTTP_Bad_Request) + elif iface.get("internal-connection-point-ref"): + try: + for vld in vnfd.get("internal-vld").itervalues(): + for cp in vld.get("internal-connection-point").itervalues(): + if cp.get("id-ref") == iface.get("internal-connection-point-ref"): + db_interface["net_id"] = net_id2uuid[vld.get("id")] + for cp_descriptor in vnfd_descriptor["connection-point"]: + if cp_descriptor["name"] == db_interface["external_name"]: + break + if str(cp_descriptor.get("port-security-enabled")).lower() == "false": + db_interface["port_security"] = 0 + elif str(cp_descriptor.get("port-security-enabled")).lower() == "true": + db_interface["port_security"] = 1 + break + except KeyError: + raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'vdu[{vdu}]':" + "'interface[{iface}]':'vdu-internal-connection-point-ref':'{cp}' is not" + " referenced by any internal-vld".format( + vnf=vnfd_id, vdu=vdu_id, iface=iface["name"], + cp=iface.get("vdu-internal-connection-point-ref")), + HTTP_Bad_Request) + if iface.get("position") is not None: + db_interface["created_at"] = int(iface.get("position")) - 1000 + db_interfaces.append(db_interface) + + # table flavors + db_flavor = { + "name": get_str(vdu, "name", 250) + "-flv", + "vcpus": int(vdu["vm-flavor"].get("vcpu-count", 1)), + "ram": int(vdu["vm-flavor"].get("memory-mb", 1)), + "disk": int(vdu["vm-flavor"].get("storage-gb", 1)), + } + # EPA TODO revise + extended = {} + numa = {} + if devices: + extended["devices"] = devices + if flavor_epa_interfaces: + numa["interfaces"] = flavor_epa_interfaces + if vdu.get("guest-epa"): # TODO or dedicated_int: + epa_vcpu_set = False + if vdu["guest-epa"].get("numa-node-policy"): # TODO or dedicated_int: + numa_node_policy = vdu["guest-epa"].get("numa-node-policy") + if numa_node_policy.get("node"): + numa_node = numa_node_policy["node"]['0'] + if numa_node.get("num-cores"): + numa["cores"] = numa_node["num-cores"] + epa_vcpu_set = True + if numa_node.get("paired-threads"): + if numa_node["paired-threads"].get("num-paired-threads"): + numa["paired-threads"] = int(numa_node["paired-threads"]["num-paired-threads"]) + epa_vcpu_set = True + if len(numa_node["paired-threads"].get("paired-thread-ids")): + numa["paired-threads-id"] = [] + for pair in numa_node["paired-threads"]["paired-thread-ids"].itervalues(): + numa["paired-threads-id"].append( + (str(pair["thread-a"]), str(pair["thread-b"])) + ) + if numa_node.get("num-threads"): + numa["threads"] = int(numa_node["num-threads"]) + epa_vcpu_set = True + if numa_node.get("memory-mb"): + numa["memory"] = max(int(numa_node["memory-mb"] / 1024), 1) + if vdu["guest-epa"].get("mempage-size"): + if vdu["guest-epa"]["mempage-size"] != "SMALL": + numa["memory"] = max(int(db_flavor["ram"] / 1024), 1) + if vdu["guest-epa"].get("cpu-pinning-policy") and not epa_vcpu_set: + if vdu["guest-epa"]["cpu-pinning-policy"] == "DEDICATED": + if vdu["guest-epa"].get("cpu-thread-pinning-policy") and \ + vdu["guest-epa"]["cpu-thread-pinning-policy"] != "PREFER": + numa["cores"] = max(db_flavor["vcpus"], 1) + else: + numa["threads"] = max(db_flavor["vcpus"], 1) + if numa: + extended["numas"] = [numa] + if extended: + extended_text = yaml.safe_dump(extended, default_flow_style=True, width=256) + db_flavor["extended"] = extended_text + # look if flavor exist + temp_flavor_dict = {'disk': db_flavor.get('disk', 1), + 'ram': db_flavor.get('ram'), + 'vcpus': db_flavor.get('vcpus'), + 'extended': db_flavor.get('extended') + } + existing_flavors = mydb.get_rows(FROM="flavors", WHERE=temp_flavor_dict) + if existing_flavors: + flavor_uuid = existing_flavors[0]["uuid"] + else: + flavor_uuid = str(uuid4()) + uuid_list.append(flavor_uuid) + db_flavor["uuid"] = flavor_uuid + db_flavors.append(db_flavor) + db_vm["flavor_id"] = flavor_uuid + + # VNF affinity and antiaffinity + for pg in vnfd.get("placement-groups").itervalues(): + pg_name = get_str(pg, "name", 255) + for vdu in pg.get("member-vdus").itervalues(): + vdu_id = get_str(vdu, "member-vdu-ref", 255) + if vdu_id not in vdu_id2db_table_index: + raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'placement-groups[{pg}]':" + "'member-vdus':'{vdu}'. Reference to a non-existing vdu".format( + vnf=vnfd_id, pg=pg_name, vdu=vdu_id), + HTTP_Bad_Request) + db_vms[vdu_id2db_table_index[vdu_id]]["availability_zone"] = pg_name + # TODO consider the case of isolation and not colocation + # if pg.get("strategy") == "ISOLATION": + + # VNF mgmt configuration + mgmt_access = {} + if vnfd["mgmt-interface"].get("vdu-id"): + mgmt_vdu_id = get_str(vnfd["mgmt-interface"], "vdu-id", 255) + if mgmt_vdu_id not in vdu_id2uuid: + raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'mgmt-interface':'vdu-id':" + "'{vdu}'. Reference to a non-existing vdu".format( + vnf=vnfd_id, vdu=mgmt_vdu_id), + HTTP_Bad_Request) + mgmt_access["vm_id"] = vdu_id2uuid[vnfd["mgmt-interface"]["vdu-id"]] + # if only one cp is defined by this VDU, mark this interface as of type "mgmt" + if vdu_id2cp_name.get(mgmt_vdu_id): + cp_name2db_interface[vdu_id2cp_name[mgmt_vdu_id]]["type"] = "mgmt" + + if vnfd["mgmt-interface"].get("ip-address"): + mgmt_access["ip-address"] = str(vnfd["mgmt-interface"].get("ip-address")) + if vnfd["mgmt-interface"].get("cp"): + if vnfd["mgmt-interface"]["cp"] not in cp_name2iface_uuid: + raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'mgmt-interface':'cp':'{cp}'. " + "Reference to a non-existing connection-point".format( + vnf=vnfd_id, cp=vnfd["mgmt-interface"]["cp"]), + HTTP_Bad_Request) + mgmt_access["vm_id"] = cp_name2vm_uuid[vnfd["mgmt-interface"]["cp"]] + mgmt_access["interface_id"] = cp_name2iface_uuid[vnfd["mgmt-interface"]["cp"]] + # mark this interface as of type mgmt + cp_name2db_interface[vnfd["mgmt-interface"]["cp"]]["type"] = "mgmt" + + default_user = get_str(vnfd.get("vnf-configuration", {}).get("config-access", {}).get("ssh-access", {}), + "default-user", 64) + + if default_user: + mgmt_access["default_user"] = default_user + required = get_str(vnfd.get("vnf-configuration", {}).get("config-access", {}).get("ssh-access", {}), + "required", 6) + if required: + mgmt_access["required"] = required + + if mgmt_access: + db_vnf["mgmt_access"] = yaml.safe_dump(mgmt_access, default_flow_style=True, width=256) + + + + db_vnfs.append(db_vnf) + db_tables=[ + {"vnfs": db_vnfs}, + {"nets": db_nets}, + {"images": db_images}, + {"flavors": db_flavors}, + {"vms": db_vms}, + {"interfaces": db_interfaces}, + ] + + logger.debug("create_vnf Deployment done vnfDict: %s", + yaml.safe_dump(db_tables, indent=4, default_flow_style=False) ) + mydb.new_rows(db_tables, uuid_list) + return vnfd_uuid_list + except NfvoException: + raise + except Exception as e: + logger.error("Exception {}".format(e)) + raise # NfvoException("Exception {}".format(e), HTTP_Bad_Request) + + def new_vnf(mydb, tenant_id, vnf_descriptor): global global_config @@ -757,6 +1250,7 @@ def new_vnf(mydb, tenant_id, vnf_descriptor): for vnfc in vnf_descriptor['vnf']['VNFC']: VNFCitem={} VNFCitem["name"] = vnfc['name'] + VNFCitem["availability_zone"] = vnfc.get('availability_zone') VNFCitem["description"] = vnfc.get("description", 'VM %s of the VNF %s' %(vnfc['name'],vnf_name)) #print "Flavor name: %s. Description: %s" % (VNFCitem["name"]+"-flv", VNFCitem["description"]) @@ -827,6 +1321,7 @@ def new_vnf(mydb, tenant_id, vnf_descriptor): #print "Image id for VNFC %s: %s" % (vnfc['name'],image_id) VNFCDict[vnfc['name']]["image_id"] = image_id VNFCDict[vnfc['name']]["image_path"] = vnfc.get('VNFC image') + VNFCDict[vnfc['name']]["count"] = vnfc.get('count', 1) if vnfc.get("boot-data"): VNFCDict[vnfc['name']]["boot_data"] = yaml.safe_dump(vnfc["boot-data"], default_flow_style=True, width=256) @@ -962,6 +1457,7 @@ def new_vnf_v02(mydb, tenant_id, vnf_descriptor): #print "Image id for VNFC %s: %s" % (vnfc['name'],image_id) VNFCDict[vnfc['name']]["image_id"] = image_id VNFCDict[vnfc['name']]["image_path"] = vnfc.get('VNFC image') + VNFCDict[vnfc['name']]["count"] = vnfc.get('count', 1) if vnfc.get("boot-data"): VNFCDict[vnfc['name']]["boot_data"] = yaml.safe_dump(vnfc["boot-data"], default_flow_style=True, width=256) @@ -996,14 +1492,15 @@ def get_vnf_id(mydb, tenant_id, vnf_id): where_or["public"] = True vnf = mydb.get_table_by_uuid_name('vnfs', vnf_id, "VNF", WHERE_OR=where_or, WHERE_AND_OR="AND") - vnf_id=vnf["uuid"] - filter_keys = ('uuid','name','description','public', "tenant_id", "created_at") + vnf_id = vnf["uuid"] + filter_keys = ('uuid', 'name', 'description', 'public', "tenant_id", "osm_id", "created_at") filtered_content = dict( (k,v) for k,v in vnf.iteritems() if k in filter_keys ) #change_keys_http2db(filtered_content, http2db_vnf, reverse=True) data={'vnf' : filtered_content} #GET VM content = mydb.get_rows(FROM='vnfs join vms on vnfs.uuid=vms.vnf_id', - SELECT=('vms.uuid as uuid','vms.name as name', 'vms.description as description', 'boot_data'), + SELECT=('vms.uuid as uuid', 'vms.osm_id as osm_id', 'vms.name as name', 'vms.description as description', + 'boot_data'), WHERE={'vnfs.uuid': vnf_id} ) if len(content)==0: raise NfvoException("vnf '{}' not found".format(vnf_id), HTTP_Not_Found) @@ -1039,8 +1536,7 @@ def get_vnf_id(mydb, tenant_id, vnf_id): content = mydb.get_rows(FROM='vnfs join vms on vnfs.uuid=vms.vnf_id join interfaces on vms.uuid=interfaces.vm_id',\ SELECT=('interfaces.uuid as uuid','interfaces.external_name as external_name', 'vms.name as vm_name', 'interfaces.vm_id as vm_id', \ 'interfaces.internal_name as internal_name', 'interfaces.type as type', 'interfaces.vpci as vpci','interfaces.bw as bw'),\ - WHERE={'vnfs.uuid': vnf_id}, - WHERE_NOT={'interfaces.external_name': None} ) + WHERE={'vnfs.uuid': vnf_id, 'interfaces.external_name<>': None} ) #print content data['vnf']['external-connections'] = content @@ -1089,7 +1585,7 @@ def delete_vnf(mydb,tenant_id,vnf_id,datacenter=None,vim_tenant=None): #delelte at VIM c = mydb.get_rows(FROM='datacenters_flavors', WHERE={'flavor_id':flavor}) for flavor_vim in c: - if flavor_vim["datacenter_id"] not in vims: + if flavor_vim["datacenter_vim_id"] not in vims: # TODO change to datacenter_tenant_id continue if flavor_vim['created']=='false': #skip this flavor because not created by openmano continue @@ -1120,7 +1616,7 @@ def delete_vnf(mydb,tenant_id,vnf_id,datacenter=None,vim_tenant=None): #delelte at VIM c = mydb.get_rows(FROM='datacenters_images', WHERE={'image_id':image}) for image_vim in c: - if image_vim["datacenter_id"] not in vims: + if image_vim["datacenter_vim_id"] not in vims: # TODO change to datacenter_tenant_id continue if image_vim['created']=='false': #skip this image because not created by openmano continue @@ -1227,8 +1723,7 @@ def new_scenario(mydb, tenant_id, topo): #1.2: Check that VNF are present at database table vnfs. Insert uuid, description and external interfaces for name,vnf in vnfs.items(): - where={} - where_or={"tenant_id": tenant_id, 'public': "true"} + where = {"OR": {"tenant_id": tenant_id, 'public': "true"}} error_text = "" error_pos = "'topology':'nodes':'" + name + "'" if 'vnf_id' in vnf: @@ -1237,14 +1732,12 @@ def new_scenario(mydb, tenant_id, topo): if 'VNF model' in vnf: error_text += " 'VNF model' " + vnf['VNF model'] where['name'] = vnf['VNF model'] - if len(where) == 0: + if len(where) == 1: raise NfvoException("Descriptor need a 'vnf_id' or 'VNF model' field at " + error_pos, HTTP_Bad_Request) vnf_db = mydb.get_rows(SELECT=('uuid','name','description'), FROM='vnfs', - WHERE=where, - WHERE_OR=where_or, - WHERE_AND_OR="AND") + WHERE=where) if len(vnf_db)==0: raise NfvoException("unknown" + error_text + " at " + error_pos, HTTP_Not_Found) elif len(vnf_db)>1: @@ -1254,7 +1747,7 @@ def new_scenario(mydb, tenant_id, topo): #get external interfaces ext_ifaces = mydb.get_rows(SELECT=('external_name as name','i.uuid as iface_uuid', 'i.type as type'), FROM='vnfs join vms on vnfs.uuid=vms.vnf_id join interfaces as i on vms.uuid=i.vm_id', - WHERE={'vnfs.uuid':vnf['uuid']}, WHERE_NOT={'external_name':None} ) + WHERE={'vnfs.uuid':vnf['uuid'], 'external_name<>': None} ) for ext_iface in ext_ifaces: vnf['ifaces'][ ext_iface['name'] ] = {'uuid':ext_iface['iface_uuid'], 'type':ext_iface['type']} @@ -1493,8 +1986,7 @@ def new_scenario_v02(mydb, tenant_id, scenario_dict, version): # 1: Check that VNF are present at database table vnfs and update content into scenario dict for name,vnf in scenario["vnfs"].iteritems(): - where={} - where_or={"tenant_id": tenant_id, 'public': "true"} + where = {"OR": {"tenant_id": tenant_id, 'public': "true"}} error_text = "" error_pos = "'scenario':'vnfs':'" + name + "'" if 'vnf_id' in vnf: @@ -1503,13 +1995,11 @@ def new_scenario_v02(mydb, tenant_id, scenario_dict, version): if 'vnf_name' in vnf: error_text += " 'vnf_name' " + vnf['vnf_name'] where['name'] = vnf['vnf_name'] - if len(where) == 0: + if len(where) == 1: raise NfvoException("Needed a 'vnf_id' or 'vnf_name' at " + error_pos, HTTP_Bad_Request) vnf_db = mydb.get_rows(SELECT=('uuid', 'name', 'description'), FROM='vnfs', - WHERE=where, - WHERE_OR=where_or, - WHERE_AND_OR="AND") + WHERE=where) if len(vnf_db) == 0: raise NfvoException("Unknown" + error_text + " at " + error_pos, HTTP_Not_Found) elif len(vnf_db) > 1: @@ -1520,7 +2010,7 @@ def new_scenario_v02(mydb, tenant_id, scenario_dict, version): # get external interfaces ext_ifaces = mydb.get_rows(SELECT=('external_name as name', 'i.uuid as iface_uuid', 'i.type as type'), FROM='vnfs join vms on vnfs.uuid=vms.vnf_id join interfaces as i on vms.uuid=i.vm_id', - WHERE={'vnfs.uuid':vnf['uuid']}, WHERE_NOT={'external_name': None} ) + WHERE={'vnfs.uuid':vnf['uuid'], 'external_name<>': None} ) for ext_iface in ext_ifaces: vnf['ifaces'][ ext_iface['name'] ] = {'uuid':ext_iface['iface_uuid'], 'type': ext_iface['type']} # TODO? get internal-connections from db.nets and their profiles, and update scenario[vnfs][internal-connections] accordingly @@ -1602,6 +2092,185 @@ def new_scenario_v02(mydb, tenant_id, scenario_dict, version): return scenario_id +def new_nsd_v3(mydb, tenant_id, nsd_descriptor): + """ + Parses an OSM IM nsd_catalog and insert at DB + :param mydb: + :param tenant_id: + :param nsd_descriptor: + :return: The list of cretated NSD ids + """ + try: + mynsd = nsd_catalog.nsd() + try: + pybindJSONDecoder.load_ietf_json(nsd_descriptor, None, None, obj=mynsd) + except Exception as e: + raise NfvoException("Error. Invalid NS descriptor format: " + str(e), HTTP_Bad_Request) + db_scenarios = [] + db_sce_nets = [] + db_sce_vnfs = [] + db_sce_interfaces = [] + db_ip_profiles = [] + db_ip_profiles_index = 0 + uuid_list = [] + nsd_uuid_list = [] + for nsd_yang in mynsd.nsd_catalog.nsd.itervalues(): + nsd = nsd_yang.get() + + # table sceanrios + scenario_uuid = str(uuid4()) + uuid_list.append(scenario_uuid) + nsd_uuid_list.append(scenario_uuid) + db_scenario = { + "uuid": scenario_uuid, + "osm_id": get_str(nsd, "id", 255), + "name": get_str(nsd, "name", 255), + "description": get_str(nsd, "description", 255), + "tenant_id": tenant_id, + "vendor": get_str(nsd, "vendor", 255), + "short_name": get_str(nsd, "short-name", 255), + "descriptor": str(nsd_descriptor)[:60000], + } + db_scenarios.append(db_scenario) + + # table sce_vnfs (constituent-vnfd) + vnf_index2scevnf_uuid = {} + vnf_index2vnf_uuid = {} + for vnf in nsd.get("constituent-vnfd").itervalues(): + existing_vnf = mydb.get_rows(FROM="vnfs", WHERE={'osm_id': str(vnf["vnfd-id-ref"])[:255], + 'tenant_id': tenant_id}) + if not existing_vnf: + raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'constituent-vnfd':'vnfd-id-ref':" + "'{}'. Reference to a non-existing VNFD in the catalog".format( + str(nsd["id"]), str(vnf["vnfd-id-ref"])[:255]), + HTTP_Bad_Request) + sce_vnf_uuid = str(uuid4()) + uuid_list.append(sce_vnf_uuid) + db_sce_vnf = { + "uuid": sce_vnf_uuid, + "scenario_id": scenario_uuid, + "name": existing_vnf[0]["name"][:200] + "." + get_str(vnf, "member-vnf-index", 5), + "vnf_id": existing_vnf[0]["uuid"], + "member_vnf_index": int(vnf["member-vnf-index"]), + # TODO 'start-by-default': True + } + vnf_index2scevnf_uuid[int(vnf['member-vnf-index'])] = sce_vnf_uuid + vnf_index2vnf_uuid[int(vnf['member-vnf-index'])] = existing_vnf[0]["uuid"] + db_sce_vnfs.append(db_sce_vnf) + + # table ip_profiles (ip-profiles) + ip_profile_name2db_table_index = {} + for ip_profile in nsd.get("ip-profiles").itervalues(): + db_ip_profile = { + "ip_version": str(ip_profile["ip-profile-params"].get("ip-version", "ipv4")), + "subnet_address": str(ip_profile["ip-profile-params"].get("subnet-address")), + "gateway_address": str(ip_profile["ip-profile-params"].get("gateway-address")), + "dhcp_enabled": str(ip_profile["ip-profile-params"]["dhcp-params"].get("enabled", True)), + "dhcp_start_address": str(ip_profile["ip-profile-params"]["dhcp-params"].get("start-address")), + "dhcp_count": str(ip_profile["ip-profile-params"]["dhcp-params"].get("count")), + } + dns_list = [] + for dns in ip_profile["ip-profile-params"]["dns-server"].itervalues(): + dns_list.append(str(dns.get("address"))) + db_ip_profile["dns_address"] = ";".join(dns_list) + if ip_profile["ip-profile-params"].get('security-group'): + db_ip_profile["security_group"] = ip_profile["ip-profile-params"]['security-group'] + ip_profile_name2db_table_index[str(ip_profile["name"])] = db_ip_profiles_index + db_ip_profiles_index += 1 + db_ip_profiles.append(db_ip_profile) + + # table sce_nets (internal-vld) + for vld in nsd.get("vld").itervalues(): + sce_net_uuid = str(uuid4()) + uuid_list.append(sce_net_uuid) + db_sce_net = { + "uuid": sce_net_uuid, + "name": get_str(vld, "name", 255), + "scenario_id": scenario_uuid, + # "type": #TODO + "multipoint": not vld.get("type") == "ELINE", + # "external": #TODO + "description": get_str(vld, "description", 255), + } + # guess type of network + if vld.get("mgmt-network"): + db_sce_net["type"] = "bridge" + db_sce_net["external"] = True + elif vld.get("provider-network").get("overlay-type") == "VLAN": + db_sce_net["type"] = "data" + else: + # later on it will be fixed to bridge or data depending on the type of interfaces attached to it + db_sce_net["type"] = None + db_sce_nets.append(db_sce_net) + + # ip-profile, link db_ip_profile with db_sce_net + if vld.get("ip-profile-ref"): + ip_profile_name = vld.get("ip-profile-ref") + if ip_profile_name not in ip_profile_name2db_table_index: + raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'vld[{}]':'ip-profile-ref':'{}'." + " Reference to a non-existing 'ip_profiles'".format( + str(nsd["id"]), str(vld["id"]), str(vld["ip-profile-ref"])), + HTTP_Bad_Request) + db_ip_profiles[ip_profile_name2db_table_index[ip_profile_name]]["sce_net_id"] = sce_net_uuid + + # table sce_interfaces (vld:vnfd-connection-point-ref) + for iface in vld.get("vnfd-connection-point-ref").itervalues(): + vnf_index = int(iface['member-vnf-index-ref']) + # check correct parameters + if vnf_index not in vnf_index2vnf_uuid: + raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'vld[{}]':'vnfd-connection-point" + "-ref':'member-vnf-index-ref':'{}'. Reference to a non-existing index at " + "'nsd':'constituent-vnfd'".format( + str(nsd["id"]), str(vld["id"]), str(iface["member-vnf-index-ref"])), + HTTP_Bad_Request) + + existing_ifaces = mydb.get_rows(SELECT=('i.uuid as uuid', 'i.type as iface_type'), + FROM="interfaces as i join vms on i.vm_id=vms.uuid", + WHERE={'vnf_id': vnf_index2vnf_uuid[vnf_index], + 'external_name': get_str(iface, "vnfd-connection-point-ref", + 255)}) + if not existing_ifaces: + raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'vld[{}]':'vnfd-connection-point" + "-ref':'vnfd-connection-point-ref':'{}'. Reference to a non-existing " + "connection-point name at VNFD '{}'".format( + str(nsd["id"]), str(vld["id"]), str(iface["vnfd-connection-point-ref"]), + str(iface.get("vnfd-id-ref"))[:255]), + HTTP_Bad_Request) + interface_uuid = existing_ifaces[0]["uuid"] + if existing_ifaces[0]["iface_type"] == "data" and not db_sce_net["type"]: + db_sce_net["type"] = "data" + sce_interface_uuid = str(uuid4()) + uuid_list.append(sce_net_uuid) + db_sce_interface = { + "uuid": sce_interface_uuid, + "sce_vnf_id": vnf_index2scevnf_uuid[vnf_index], + "sce_net_id": sce_net_uuid, + "interface_id": interface_uuid, + # "ip_address": #TODO + } + db_sce_interfaces.append(db_sce_interface) + if not db_sce_net["type"]: + db_sce_net["type"] = "bridge" + + db_tables = [ + {"scenarios": db_scenarios}, + {"sce_nets": db_sce_nets}, + {"ip_profiles": db_ip_profiles}, + {"sce_vnfs": db_sce_vnfs}, + {"sce_interfaces": db_sce_interfaces}, + ] + + logger.debug("create_vnf Deployment done vnfDict: %s", + yaml.safe_dump(db_tables, indent=4, default_flow_style=False) ) + mydb.new_rows(db_tables, uuid_list) + return nsd_uuid_list + except NfvoException: + raise + except Exception as e: + logger.error("Exception {}".format(e)) + raise # NfvoException("Exception {}".format(e), HTTP_Bad_Request) + + def edit_scenario(mydb, tenant_id, scenario_id, data): data["uuid"] = scenario_id data["tenant_id"] = tenant_id @@ -1619,7 +2288,7 @@ def start_scenario(mydb, tenant_id, scenario_id, instance_scenario_name, instanc rollbackList=[] try: #print "Checking that the scenario_id exists and getting the scenario dictionary" - scenarioDict = mydb.get_scenario(scenario_id, tenant_id, datacenter_id) + scenarioDict = mydb.get_scenario(scenario_id, tenant_id, datacenter_id=datacenter_id) scenarioDict['datacenter2tenant'] = { datacenter_id: myvim['config']['datacenter_tenant_id'] } scenarioDict['datacenter_id'] = datacenter_id #print '================scenarioDict=======================' @@ -1665,6 +2334,7 @@ def start_scenario(mydb, tenant_id, scenario_id, instance_scenario_name, instanc logger.debug("start_scenario 2. Creating new nets (vnf internal nets) in the VIM") #For each vnf net, we create it and we add it to instanceNetlist. + for sce_vnf in scenarioDict['vnfs']: for net in sce_vnf['nets']: #print "Net name: %s. Description: %s" % (net["name"], net["description"]) @@ -1696,6 +2366,17 @@ def start_scenario(mydb, tenant_id, scenario_id, instance_scenario_name, instanc #myvim.new_vminstance(self,vimURI,tenant_id,name,description,image_id,flavor_id,net_dict) i = 0 for sce_vnf in scenarioDict['vnfs']: + vnf_availability_zones = [] + for vm in sce_vnf['vms']: + vm_av = vm.get('availability_zone') + if vm_av and vm_av not in vnf_availability_zones: + vnf_availability_zones.append(vm_av) + + # check if there is enough availability zones available at vim level. + if myvims[datacenter_id].availability_zone and vnf_availability_zones: + if len(vnf_availability_zones) > len(myvims[datacenter_id].availability_zone): + raise NfvoException('No enough availability zones at VIM for this deployment', HTTP_Bad_Request) + for vm in sce_vnf['vms']: i += 1 myVMDict = {} @@ -1782,8 +2463,16 @@ def start_scenario(mydb, tenant_id, scenario_id, instance_scenario_name, instanc #print "networks", yaml.safe_dump(myVMDict['networks'], indent=4, default_flow_style=False) #print "interfaces", yaml.safe_dump(vm['interfaces'], indent=4, default_flow_style=False) #print ">>>>>>>>>>>>>>>>>>>>>>>>>>>" - vm_id = myvim.new_vminstance(myVMDict['name'],myVMDict['description'],myVMDict.get('start', None), - myVMDict['imageRef'],myVMDict['flavorRef'],myVMDict['networks']) + + if 'availability_zone' in myVMDict: + av_index = vnf_availability_zones.index(myVMDict['availability_zone']) + else: + av_index = None + + vm_id, _ = myvim.new_vminstance(myVMDict['name'], myVMDict['description'], myVMDict.get('start', None), + myVMDict['imageRef'], myVMDict['flavorRef'], myVMDict['networks'], + availability_zone_index=av_index, + availability_zone_list=vnf_availability_zones) #print "VIM vm instance id (server id) for scenario %s: %s" % (scenarioDict['name'],vm_id) vm['vim_id'] = vm_id rollbackList.append({'what':'vm','where':'vim','vim_id':datacenter_id,'uuid':vm_id}) @@ -1811,12 +2500,11 @@ def start_scenario(mydb, tenant_id, scenario_id, instance_scenario_name, instanc #logger.error("start_scenario %s", error_text) raise NfvoException(error_text, e.http_code) - def unify_cloud_config(cloud_config_preserve, cloud_config): - ''' join the cloud config information into cloud_config_preserve. + """ join the cloud config information into cloud_config_preserve. In case of conflict cloud_config_preserve preserves - None is admited - ''' + None is allowed + """ if not cloud_config_preserve and not cloud_config: return None @@ -1866,10 +2554,19 @@ def unify_cloud_config(cloud_config_preserve, cloud_config): new_cloud_config["boot-data-drive"] = cloud_config_preserve["boot-data-drive"] # user-data - if cloud_config and cloud_config.get("user-data") != None: - new_cloud_config["user-data"] = cloud_config["user-data"] - if cloud_config_preserve and cloud_config_preserve.get("user-data") != None: - new_cloud_config["user-data"] = cloud_config_preserve["user-data"] + new_cloud_config["user-data"] = [] + if cloud_config and cloud_config.get("user-data"): + if isinstance(cloud_config["user-data"], list): + new_cloud_config["user-data"] += cloud_config["user-data"] + else: + new_cloud_config["user-data"].append(cloud_config["user-data"]) + if cloud_config_preserve and cloud_config_preserve.get("user-data"): + if isinstance(cloud_config_preserve["user-data"], list): + new_cloud_config["user-data"] += cloud_config_preserve["user-data"] + else: + new_cloud_config["user-data"].append(cloud_config_preserve["user-data"]) + if not new_cloud_config["user-data"]: + del new_cloud_config["user-data"] # config files new_cloud_config["config-files"] = [] @@ -1923,6 +2620,29 @@ def get_vim_thread(mydb, tenant_id, datacenter_id_name=None, datacenter_tenant_i except db_base_Exception as e: raise NfvoException("{} {}".format(type(e).__name__ , str(e)), e.http_code) + +def get_datacenter_uuid(mydb, tenant_id, datacenter_id_name): + WHERE_dict={} + if utils.check_valid_uuid(datacenter_id_name): + WHERE_dict['d.uuid'] = datacenter_id_name + else: + WHERE_dict['d.name'] = datacenter_id_name + + if tenant_id: + WHERE_dict['nfvo_tenant_id'] = tenant_id + from_= "tenants_datacenters as td join datacenters as d on td.datacenter_id=d.uuid join datacenter_tenants as" \ + " dt on td.datacenter_tenant_id=dt.uuid" + else: + from_ = 'datacenters as d' + vimaccounts = mydb.get_rows(FROM=from_, SELECT=("d.uuid as uuid",), WHERE=WHERE_dict ) + if len(vimaccounts) == 0: + raise NfvoException("datacenter '{}' not found".format(str(datacenter_id_name)), HTTP_Not_Found) + elif len(vimaccounts)>1: + #print "nfvo.datacenter_action() error. Several datacenters found" + raise NfvoException("More than one datacenters found, try to identify with uuid", HTTP_Conflict) + return vimaccounts[0]["uuid"] + + def get_datacenter_by_name_uuid(mydb, tenant_id, datacenter_id_name=None, **extra_filter): datacenter_id = None datacenter_name = None @@ -1951,71 +2671,100 @@ def update(d, u): d[k] = u[k] return d - def create_instance(mydb, tenant_id, instance_dict): # print "Checking that nfvo_tenant_id exists and getting the VIM URI and the VIM tenant_id" # logger.debug("Creating instance...") scenario = instance_dict["scenario"] - #find main datacenter + # find main datacenter myvims = {} myvim_threads_id = {} - instance_tasks={} - tasks_to_launch={} datacenter = instance_dict.get("datacenter") default_datacenter_id, vim = get_datacenter_by_name_uuid(mydb, tenant_id, datacenter) myvims[default_datacenter_id] = vim myvim_threads_id[default_datacenter_id], _ = get_vim_thread(mydb, tenant_id, default_datacenter_id) - tasks_to_launch[myvim_threads_id[default_datacenter_id]] = [] - #myvim_tenant = myvim['tenant_id'] -# default_datacenter_name = vim['name'] - rollbackList=[] - - #print "Checking that the scenario exists and getting the scenario dictionary" - scenarioDict = mydb.get_scenario(scenario, tenant_id, default_datacenter_id) + tenant = mydb.get_rows_by_id('nfvo_tenants', tenant_id) + # myvim_tenant = myvim['tenant_id'] - #logger.debug(">>>>>>> Dictionaries before merging") - #logger.debug(">>>>>>> InstanceDict:\n{}".format(yaml.safe_dump(instance_dict,default_flow_style=False, width=256))) - #logger.debug(">>>>>>> ScenarioDict:\n{}".format(yaml.safe_dump(scenarioDict,default_flow_style=False, width=256))) + rollbackList=[] - scenarioDict['datacenter_id'] = default_datacenter_id + # print "Checking that the scenario exists and getting the scenario dictionary" + scenarioDict = mydb.get_scenario(scenario, tenant_id, datacenter_vim_id=myvim_threads_id[default_datacenter_id], + datacenter_id=default_datacenter_id) + + # logger.debug(">>>>>> Dictionaries before merging") + # logger.debug(">>>>>> InstanceDict:\n{}".format(yaml.safe_dump(instance_dict,default_flow_style=False, width=256))) + # logger.debug(">>>>>> ScenarioDict:\n{}".format(yaml.safe_dump(scenarioDict,default_flow_style=False, width=256))) + + db_instance_vnfs = [] + db_instance_vms = [] + db_instance_interfaces = [] + db_ip_profiles = [] + db_vim_actions = [] + uuid_list = [] + task_index = 0 + instance_name = instance_dict["name"] + instance_uuid = str(uuid4()) + uuid_list.append(instance_uuid) + db_instance_scenario = { + "uuid": instance_uuid, + "name": instance_name, + "tenant_id": tenant_id, + "scenario_id": scenarioDict['uuid'], + "datacenter_id": default_datacenter_id, + # filled bellow 'datacenter_tenant_id' + "description": instance_dict.get("description"), + } + if scenarioDict.get("cloud-config"): + db_instance_scenario["cloud_config"] = yaml.safe_dump(scenarioDict["cloud-config"], + default_flow_style=True, width=256) + instance_action_id = get_task_id() + db_instance_action = { + "uuid": instance_action_id, # same uuid for the instance and the action on create + "tenant_id": tenant_id, + "instance_id": instance_uuid, + "description": "CREATE", + } - auxNetDict = {} #Auxiliar dictionary. First key:'scenario' or sce_vnf uuid. Second Key: uuid of the net/sce_net. Value: vim_net_id - auxNetDict['scenario'] = {} + # Auxiliary dictionaries from x to y + vnf_net2instance = {} + sce_net2instance = {} + net2task_id = {'scenario': {}} - logger.debug("Creating instance from scenario-dict:\n%s", yaml.safe_dump(scenarioDict, indent=4, default_flow_style=False)) #TODO remove - instance_name = instance_dict["name"] - instance_description = instance_dict.get("description") + # logger.debug("Creating instance from scenario-dict:\n%s", + # yaml.safe_dump(scenarioDict, indent=4, default_flow_style=False)) try: # 0 check correct parameters - for net_name, net_instance_desc in instance_dict.get("networks",{}).iteritems(): + for net_name, net_instance_desc in instance_dict.get("networks", {}).iteritems(): found = False for scenario_net in scenarioDict['nets']: if net_name == scenario_net["name"]: found = True break if not found: - raise NfvoException("Invalid scenario network name '{}' at instance:networks".format(net_name), HTTP_Bad_Request) + raise NfvoException("Invalid scenario network name '{}' at instance:networks".format(net_name), + HTTP_Bad_Request) if "sites" not in net_instance_desc: net_instance_desc["sites"] = [ {} ] site_without_datacenter_field = False for site in net_instance_desc["sites"]: if site.get("datacenter"): + site["datacenter"] = get_datacenter_uuid(mydb, tenant_id, site["datacenter"]) if site["datacenter"] not in myvims: - #Add this datacenter to myvims + # Add this datacenter to myvims d, v = get_datacenter_by_name_uuid(mydb, tenant_id, site["datacenter"]) myvims[d] = v - myvim_threads_id[d],_ = get_vim_thread(mydb, tenant_id, site["datacenter"]) - tasks_to_launch[myvim_threads_id[d]] = [] - site["datacenter"] = d #change name to id + myvim_threads_id[d], _ = get_vim_thread(mydb, tenant_id, site["datacenter"]) + site["datacenter"] = d # change name to id else: if site_without_datacenter_field: - raise NfvoException("Found more than one entries without datacenter field at instance:networks:{}:sites".format(net_name), HTTP_Bad_Request) + raise NfvoException("Found more than one entries without datacenter field at " + "instance:networks:{}:sites".format(net_name), HTTP_Bad_Request) site_without_datacenter_field = True - site["datacenter"] = default_datacenter_id #change name to id + site["datacenter"] = default_datacenter_id # change name to id for vnf_name, vnf_instance_desc in instance_dict.get("vnfs",{}).iteritems(): - found=False + found = False for scenario_vnf in scenarioDict['vnfs']: if vnf_name == scenario_vnf['name']: found = True @@ -2023,20 +2772,24 @@ def create_instance(mydb, tenant_id, instance_dict): if not found: raise NfvoException("Invalid vnf name '{}' at instance:vnfs".format(vnf_instance_desc), HTTP_Bad_Request) if "datacenter" in vnf_instance_desc: - # Add this datacenter to myvims + # Add this datacenter to myvims + vnf_instance_desc["datacenter"] = get_datacenter_uuid(mydb, tenant_id, vnf_instance_desc["datacenter"]) if vnf_instance_desc["datacenter"] not in myvims: d, v = get_datacenter_by_name_uuid(mydb, tenant_id, vnf_instance_desc["datacenter"]) myvims[d] = v - myvim_threads_id[d],_ = get_vim_thread(mydb, tenant_id, vnf_instance_desc["datacenter"]) - tasks_to_launch[myvim_threads_id[d]] = [] + myvim_threads_id[d], _ = get_vim_thread(mydb, tenant_id, vnf_instance_desc["datacenter"]) scenario_vnf["datacenter"] = vnf_instance_desc["datacenter"] - #0.1 parse cloud-config parameters + # 0.1 parse cloud-config parameters cloud_config = unify_cloud_config(instance_dict.get("cloud-config"), scenarioDict.get("cloud-config")) - - #0.2 merge instance information into scenario - #Ideally, the operation should be as simple as: update(scenarioDict,instance_dict) - #However, this is not possible yet. + # We add the RO key to cloud_config + if tenant[0].get('RO_pub_key'): + RO_key = {"key-pairs": [tenant[0]['RO_pub_key']]} + cloud_config = unify_cloud_config(cloud_config, RO_key) + + # 0.2 merge instance information into scenario + # Ideally, the operation should be as simple as: update(scenarioDict,instance_dict) + # However, this is not possible yet. for net_name, net_instance_desc in instance_dict.get("networks",{}).iteritems(): for scenario_net in scenarioDict['nets']: if net_name == scenario_net["name"]: @@ -2066,16 +2819,17 @@ def create_instance(mydb, tenant_id, instance_dict): if interface['vnf_interface'] == vnf_interface['external_name']: vnf_interface['ip_address']=interface['ip_address'] - #logger.debug(">>>>>>>> Merged dictionary") - logger.debug("Creating instance scenario-dict MERGED:\n%s", yaml.safe_dump(scenarioDict, indent=4, default_flow_style=False)) - + # logger.debug(">>>>>>>> Merged dictionary") + # logger.debug("Creating instance scenario-dict MERGED:\n%s", + # yaml.safe_dump(scenarioDict, indent=4, default_flow_style=False)) # 1. Creating new nets (sce_nets) in the VIM" + db_instance_nets = [] for sce_net in scenarioDict['nets']: - sce_net["vim_id_sites"]={} - descriptor_net = instance_dict.get("networks",{}).get(sce_net["name"],{}) + descriptor_net = instance_dict.get("networks", {}).get(sce_net["name"], {}) net_name = descriptor_net.get("vim-network-name") - auxNetDict['scenario'][sce_net['uuid']] = {} + sce_net2instance[sce_net['uuid']] = {} + net2task_id['scenario'][sce_net['uuid']] = {} sites = descriptor_net.get("sites", [ {} ]) for site in sites: @@ -2088,109 +2842,187 @@ def create_instance(mydb, tenant_id, instance_dict): datacenter_id = default_datacenter_id myvim_thread_id = myvim_threads_id[default_datacenter_id] net_type = sce_net['type'] - lookfor_filter = {'admin_state_up': True, 'status': 'ACTIVE'} #'shared': True - if sce_net["external"]: - if not net_name: + lookfor_filter = {'admin_state_up': True, 'status': 'ACTIVE'} # 'shared': True + + if not net_name: + if sce_net["external"]: net_name = sce_net["name"] - if "netmap-use" in site or "netmap-create" in site: - create_network = False - lookfor_network = False - if "netmap-use" in site: - lookfor_network = True - if utils.check_valid_uuid(site["netmap-use"]): - filter_text = "scenario id '%s'" % site["netmap-use"] - lookfor_filter["id"] = site["netmap-use"] - else: - filter_text = "scenario name '%s'" % site["netmap-use"] - lookfor_filter["name"] = site["netmap-use"] - if "netmap-create" in site: - create_network = True - net_vim_name = net_name - if site["netmap-create"]: - net_vim_name = site["netmap-create"] - - elif sce_net['vim_id'] != None: - #there is a netmap at datacenter_nets database #TODO REVISE!!!! + else: + net_name = "{}.{}".format(instance_name, sce_net["name"]) + net_name = net_name[:255] # limit length + + if "netmap-use" in site or "netmap-create" in site: + create_network = False + lookfor_network = False + if "netmap-use" in site: + lookfor_network = True + if utils.check_valid_uuid(site["netmap-use"]): + filter_text = "scenario id '%s'" % site["netmap-use"] + lookfor_filter["id"] = site["netmap-use"] + else: + filter_text = "scenario name '%s'" % site["netmap-use"] + lookfor_filter["name"] = site["netmap-use"] + if "netmap-create" in site: + create_network = True + net_vim_name = net_name + if site["netmap-create"]: + net_vim_name = site["netmap-create"] + elif sce_net["external"]: + if sce_net['vim_id'] != None: + # there is a netmap at datacenter_nets database # TODO REVISE!!!! create_network = False lookfor_network = True lookfor_filter["id"] = sce_net['vim_id'] - filter_text = "vim_id '%s' datacenter_netmap name '%s'. Try to reload vims with datacenter-net-update" % (sce_net['vim_id'], sce_net["name"]) - #look for network at datacenter and return error + filter_text = "vim_id '{}' datacenter_netmap name '{}'. Try to reload vims with "\ + "datacenter-net-update".format(sce_net['vim_id'], sce_net["name"]) + # look for network at datacenter and return error else: - #There is not a netmap, look at datacenter for a net with this name and create if not found + # There is not a netmap, look at datacenter for a net with this name and create if not found create_network = True lookfor_network = True lookfor_filter["name"] = sce_net["name"] net_vim_name = sce_net["name"] filter_text = "scenario name '%s'" % sce_net["name"] else: - if not net_name: - net_name = "%s.%s" %(instance_name, sce_net["name"]) - net_name = net_name[:255] #limit length net_vim_name = net_name create_network = True lookfor_network = False - if lookfor_network: - vim_nets = vim.get_network_list(filter_dict=lookfor_filter) - if len(vim_nets) > 1: - raise NfvoException("More than one candidate VIM network found for " + filter_text, HTTP_Bad_Request ) - elif len(vim_nets) == 0: - if not create_network: - raise NfvoException("No candidate VIM network found for " + filter_text, HTTP_Bad_Request ) - else: - sce_net["vim_id_sites"][datacenter_id] = vim_nets[0]['id'] - auxNetDict['scenario'][sce_net['uuid']][datacenter_id] = vim_nets[0]['id'] - create_network = False + task_extra = {} if create_network: - #if network is not external - task = new_task("new-net", (net_vim_name, net_type, sce_net.get('ip_profile',None))) - task_id = task["id"] - instance_tasks[task_id] = task - tasks_to_launch[myvim_thread_id].append(task) - #network_id = vim.new_network(net_vim_name, net_type, sce_net.get('ip_profile',None)) - sce_net["vim_id_sites"][datacenter_id] = task_id - auxNetDict['scenario'][sce_net['uuid']][datacenter_id] = task_id - rollbackList.append({'what':'network', 'where':'vim', 'vim_id':datacenter_id, 'uuid':task_id}) - sce_net["created"] = True + task_action = "CREATE" + task_extra["params"] = (net_vim_name, net_type, sce_net.get('ip_profile', None)) + if lookfor_network: + task_extra["find"] = (lookfor_filter,) + elif lookfor_network: + task_action = "FIND" + task_extra["params"] = (lookfor_filter,) + + # fill database content + net_uuid = str(uuid4()) + uuid_list.append(net_uuid) + sce_net2instance[sce_net['uuid']][datacenter_id] = net_uuid + db_net = { + "uuid": net_uuid, + 'vim_net_id': None, + "instance_scenario_id": instance_uuid, + "sce_net_id": sce_net["uuid"], + "created": create_network, + 'datacenter_id': datacenter_id, + 'datacenter_tenant_id': myvim_thread_id, + 'status': 'BUILD' if create_network else "ACTIVE" + } + db_instance_nets.append(db_net) + db_vim_action = { + "instance_action_id": instance_action_id, + "status": "SCHEDULED", + "task_index": task_index, + "datacenter_vim_id": myvim_thread_id, + "action": task_action, + "item": "instance_nets", + "item_id": net_uuid, + "extra": yaml.safe_dump(task_extra, default_flow_style=True, width=256) + } + net2task_id['scenario'][sce_net['uuid']][datacenter_id] = task_index + task_index += 1 + db_vim_actions.append(db_vim_action) + + if 'ip_profile' in sce_net: + db_ip_profile={ + 'instance_net_id': net_uuid, + 'ip_version': sce_net['ip_profile']['ip_version'], + 'subnet_address': sce_net['ip_profile']['subnet_address'], + 'gateway_address': sce_net['ip_profile']['gateway_address'], + 'dns_address': sce_net['ip_profile']['dns_address'], + 'dhcp_enabled': sce_net['ip_profile']['dhcp_enabled'], + 'dhcp_start_address': sce_net['ip_profile']['dhcp_start_address'], + 'dhcp_count': sce_net['ip_profile']['dhcp_count'], + } + db_ip_profiles.append(db_ip_profile) # 2. Creating new nets (vnf internal nets) in the VIM" - #For each vnf net, we create it and we add it to instanceNetlist. + # For each vnf net, we create it and we add it to instanceNetlist. for sce_vnf in scenarioDict['vnfs']: for net in sce_vnf['nets']: if sce_vnf.get("datacenter"): - vim = myvims[ sce_vnf["datacenter"] ] datacenter_id = sce_vnf["datacenter"] - myvim_thread_id = myvim_threads_id[ sce_vnf["datacenter"]] + myvim_thread_id = myvim_threads_id[sce_vnf["datacenter"]] else: - vim = myvims[ default_datacenter_id ] datacenter_id = default_datacenter_id myvim_thread_id = myvim_threads_id[default_datacenter_id] - descriptor_net = instance_dict.get("vnfs",{}).get(sce_vnf["name"],{}) + descriptor_net = instance_dict.get("vnfs", {}).get(sce_vnf["name"], {}) net_name = descriptor_net.get("name") if not net_name: - net_name = "%s.%s" %(instance_name, net["name"]) - net_name = net_name[:255] #limit length + net_name = "{}.{}".format(instance_name, net["name"]) + net_name = net_name[:255] # limit length net_type = net['type'] - task = new_task("new-net", (net_name, net_type, net.get('ip_profile',None))) - task_id = task["id"] - instance_tasks[task_id] = task - tasks_to_launch[myvim_thread_id].append(task) - # network_id = vim.new_network(net_name, net_type, net.get('ip_profile',None)) - net['vim_id'] = task_id - if sce_vnf['uuid'] not in auxNetDict: - auxNetDict[sce_vnf['uuid']] = {} - auxNetDict[sce_vnf['uuid']][net['uuid']] = task_id - rollbackList.append({'what':'network','where':'vim','vim_id':datacenter_id,'uuid':task_id}) - net["created"] = True - - #print "auxNetDict:" - #print yaml.safe_dump(auxNetDict, indent=4, default_flow_style=False) + if sce_vnf['uuid'] not in vnf_net2instance: + vnf_net2instance[sce_vnf['uuid']] = {} + if sce_vnf['uuid'] not in net2task_id: + net2task_id[sce_vnf['uuid']] = {} + net2task_id[sce_vnf['uuid']][net['uuid']] = task_index + + # fill database content + net_uuid = str(uuid4()) + uuid_list.append(net_uuid) + vnf_net2instance[sce_vnf['uuid']][net['uuid']] = net_uuid + db_net = { + "uuid": net_uuid, + 'vim_net_id': None, + "instance_scenario_id": instance_uuid, + "net_id": net["uuid"], + "created": True, + 'datacenter_id': datacenter_id, + 'datacenter_tenant_id': myvim_thread_id, + } + db_instance_nets.append(db_net) + + db_vim_action = { + "instance_action_id": instance_action_id, + "task_index": task_index, + "datacenter_vim_id": myvim_thread_id, + "status": "SCHEDULED", + "action": "CREATE", + "item": "instance_nets", + "item_id": net_uuid, + "extra": yaml.safe_dump({"params": (net_name, net_type, net.get('ip_profile',None))}, + default_flow_style=True, width=256) + } + task_index += 1 + db_vim_actions.append(db_vim_action) + + if 'ip_profile' in net: + db_ip_profile = { + 'instance_net_id': net_uuid, + 'ip_version': net['ip_profile']['ip_version'], + 'subnet_address': net['ip_profile']['subnet_address'], + 'gateway_address': net['ip_profile']['gateway_address'], + 'dns_address': net['ip_profile']['dns_address'], + 'dhcp_enabled': net['ip_profile']['dhcp_enabled'], + 'dhcp_start_address': net['ip_profile']['dhcp_start_address'], + 'dhcp_count': net['ip_profile']['dhcp_count'], + } + db_ip_profiles.append(db_ip_profile) + + # print "vnf_net2instance:" + # print yaml.safe_dump(vnf_net2instance, indent=4, default_flow_style=False) # 3. Creating new vm instances in the VIM - #myvim.new_vminstance(self,vimURI,tenant_id,name,description,image_id,flavor_id,net_dict) - for sce_vnf in scenarioDict['vnfs']: + # myvim.new_vminstance(self,vimURI,tenant_id,name,description,image_id,flavor_id,net_dict) + sce_vnf_list = sorted(scenarioDict['vnfs'], key=lambda k: k['name']) + for sce_vnf in sce_vnf_list: + vnf_availability_zones = [] + for vm in sce_vnf['vms']: + vm_av = vm.get('availability_zone') + if vm_av and vm_av not in vnf_availability_zones: + vnf_availability_zones.append(vm_av) + + # check if there is enough availability zones available at vim level. + if myvims[datacenter_id].availability_zone and vnf_availability_zones: + if len(vnf_availability_zones) > len(myvims[datacenter_id].availability_zone): + raise NfvoException('No enough availability zones at VIM for this deployment', HTTP_Bad_Request) + if sce_vnf.get("datacenter"): vim = myvims[ sce_vnf["datacenter"] ] myvim_thread_id = myvim_threads_id[ sce_vnf["datacenter"] ] @@ -2199,34 +3031,46 @@ def create_instance(mydb, tenant_id, instance_dict): vim = myvims[ default_datacenter_id ] myvim_thread_id = myvim_threads_id[ default_datacenter_id ] datacenter_id = default_datacenter_id - sce_vnf["datacenter_id"] = datacenter_id + sce_vnf["datacenter_id"] = datacenter_id i = 0 + + vnf_uuid = str(uuid4()) + uuid_list.append(vnf_uuid) + db_instance_vnf = { + 'uuid': vnf_uuid, + 'instance_scenario_id': instance_uuid, + 'vnf_id': sce_vnf['vnf_id'], + 'sce_vnf_id': sce_vnf['uuid'], + 'datacenter_id': datacenter_id, + 'datacenter_tenant_id': myvim_thread_id, + } + db_instance_vnfs.append(db_instance_vnf) + for vm in sce_vnf['vms']: - i += 1 myVMDict = {} - myVMDict['name'] = "{}.{}.{}".format(instance_name,sce_vnf['name'],chr(96+i)) + myVMDict['name'] = "{}.{}.{}".format(instance_name[:64], sce_vnf['name'][:64], vm["name"][:64]) myVMDict['description'] = myVMDict['name'][0:99] # if not startvms: # myVMDict['start'] = "no" - myVMDict['name'] = myVMDict['name'][0:255] #limit name length + myVMDict['name'] = myVMDict['name'][0:255] # limit name length #create image at vim in case it not exist image_dict = mydb.get_table_by_uuid_name("images", vm['image_id']) image_id = create_or_use_image(mydb, {datacenter_id: vim}, image_dict, [], True) vm['vim_image_id'] = image_id - #create flavor at vim in case it not exist + # create flavor at vim in case it not exist flavor_dict = mydb.get_table_by_uuid_name("flavors", vm['flavor_id']) if flavor_dict['extended']!=None: - flavor_dict['extended']= yaml.load(flavor_dict['extended']) + flavor_dict['extended'] = yaml.load(flavor_dict['extended']) flavor_id = create_or_use_flavor(mydb, {datacenter_id: vim}, flavor_dict, rollbackList, True) - #Obtain information for additional disks + # Obtain information for additional disks extended_flavor_dict = mydb.get_rows(FROM='datacenters_flavors', SELECT=('extended',), WHERE={'vim_id': flavor_id}) if not extended_flavor_dict: raise NfvoException("flavor '{}' not found".format(flavor_id), HTTP_Not_Found) return - #extended_flavor_dict_yaml = yaml.load(extended_flavor_dict[0]) + # extended_flavor_dict_yaml = yaml.load(extended_flavor_dict[0]) myVMDict['disks'] = None extended_info = extended_flavor_dict[0]['extended'] if extended_info != None: @@ -2237,17 +3081,20 @@ def create_instance(mydb, tenant_id, instance_dict): vm['vim_flavor_id'] = flavor_id myVMDict['imageRef'] = vm['vim_image_id'] myVMDict['flavorRef'] = vm['vim_flavor_id'] + myVMDict['availability_zone'] = vm.get('availability_zone') myVMDict['networks'] = [] - task_depends = {} - #TODO ALF. connect_mgmt_interfaces. Connect management interfaces if this is true + task_depends_on = [] + # TODO ALF. connect_mgmt_interfaces. Connect management interfaces if this is true + db_vm_ifaces = [] for iface in vm['interfaces']: netDict = {} if iface['type']=="data": netDict['type'] = iface['model'] elif "model" in iface and iface["model"]!=None: netDict['model']=iface['model'] - #TODO in future, remove this because mac_address will not be set, and the type of PV,VF is obtained from iterface table model - #discover type of interface looking at flavor + # TODO in future, remove this because mac_address will not be set, and the type of PV,VF + # is obtained from iterface table model + # discover type of interface looking at flavor for numa in flavor_dict.get('extended',{}).get('numas',[]): for flavor_iface in numa.get('interfaces',[]): if flavor_iface.get('name') == iface['internal_name']: @@ -2282,63 +3129,134 @@ def create_instance(mydb, tenant_id, instance_dict): netDict['name'] = iface['internal_name'] if iface['net_id'] is None: for vnf_iface in sce_vnf["interfaces"]: - #print iface - #print vnf_iface + # print iface + # print vnf_iface if vnf_iface['interface_id']==iface['uuid']: - netDict['net_id'] = auxNetDict['scenario'][ vnf_iface['sce_net_id'] ][datacenter_id] + netDict['net_id'] = "TASK-{}".format(net2task_id['scenario'][ vnf_iface['sce_net_id'] ][datacenter_id]) + instance_net_id = sce_net2instance[ vnf_iface['sce_net_id'] ][datacenter_id] + task_depends_on.append(net2task_id['scenario'][ vnf_iface['sce_net_id'] ][datacenter_id]) break else: - netDict['net_id'] = auxNetDict[ sce_vnf['uuid'] ][ iface['net_id'] ] - if netDict.get('net_id') and is_task_id(netDict['net_id']): - task_depends[netDict['net_id']] = instance_tasks[netDict['net_id']] - #skip bridge ifaces not connected to any net - #if 'net_id' not in netDict or netDict['net_id']==None: - # continue + netDict['net_id'] = "TASK-{}".format(net2task_id[ sce_vnf['uuid'] ][ iface['net_id'] ]) + instance_net_id = vnf_net2instance[ sce_vnf['uuid'] ][ iface['net_id'] ] + task_depends_on.append(net2task_id[sce_vnf['uuid'] ][ iface['net_id']]) + # skip bridge ifaces not connected to any net + if 'net_id' not in netDict or netDict['net_id']==None: + continue myVMDict['networks'].append(netDict) - #print ">>>>>>>>>>>>>>>>>>>>>>>>>>>" - #print myVMDict['name'] - #print "networks", yaml.safe_dump(myVMDict['networks'], indent=4, default_flow_style=False) - #print "interfaces", yaml.safe_dump(vm['interfaces'], indent=4, default_flow_style=False) - #print ">>>>>>>>>>>>>>>>>>>>>>>>>>>" + db_vm_iface={ + # "uuid" + # 'instance_vm_id': instance_vm_uuid, + "instance_net_id": instance_net_id, + 'interface_id': iface['uuid'], + # 'vim_interface_id': , + 'type': 'external' if iface['external_name'] is not None else 'internal', + 'ip_address': iface.get('ip_address'), + 'floating_ip': int(iface.get('floating-ip', False)), + 'port_security': int(iface.get('port-security', True)) + } + db_vm_ifaces.append(db_vm_iface) + # print ">>>>>>>>>>>>>>>>>>>>>>>>>>>" + # print myVMDict['name'] + # print "networks", yaml.safe_dump(myVMDict['networks'], indent=4, default_flow_style=False) + # print "interfaces", yaml.safe_dump(vm['interfaces'], indent=4, default_flow_style=False) + # print ">>>>>>>>>>>>>>>>>>>>>>>>>>>" if vm.get("boot_data"): cloud_config_vm = unify_cloud_config(vm["boot_data"], cloud_config) else: cloud_config_vm = cloud_config - task = new_task("new-vm", (myVMDict['name'], myVMDict['description'], myVMDict.get('start', None), - myVMDict['imageRef'], myVMDict['flavorRef'], myVMDict['networks'], - cloud_config_vm, myVMDict['disks']), depends=task_depends) - instance_tasks[task["id"]] = task - tasks_to_launch[myvim_thread_id].append(task) - vm_id = task["id"] - vm['vim_id'] = vm_id - rollbackList.append({'what':'vm','where':'vim','vim_id':datacenter_id,'uuid':vm_id}) - #put interface uuid back to scenario[vnfs][vms[[interfaces] - for net in myVMDict['networks']: - if "vim_id" in net: - for iface in vm['interfaces']: - if net["name"]==iface["internal_name"]: - iface["vim_id"]=net["vim_id"] - break + if myVMDict.get('availability_zone'): + av_index = vnf_availability_zones.index(myVMDict['availability_zone']) + else: + av_index = None + for vm_index in range(0, vm.get('count', 1)): + vm_index_name = "" + if vm.get('count', 1) > 1: + vm_index_name += "." + chr(97 + vm_index) + task_params = (myVMDict['name']+vm_index_name, myVMDict['description'], myVMDict.get('start', None), + myVMDict['imageRef'], myVMDict['flavorRef'], myVMDict['networks'], cloud_config_vm, + myVMDict['disks'], av_index, vnf_availability_zones) + # put interface uuid back to scenario[vnfs][vms[[interfaces] + for net in myVMDict['networks']: + if "vim_id" in net: + for iface in vm['interfaces']: + if net["name"]==iface["internal_name"]: + iface["vim_id"]=net["vim_id"] + break + vm_uuid = str(uuid4()) + uuid_list.append(vm_uuid) + db_vm = { + "uuid": vm_uuid, + 'instance_vnf_id': vnf_uuid, + #TODO delete "vim_vm_id": vm_id, + "vm_id": vm["uuid"], + # "status": + } + db_instance_vms.append(db_vm) + + iface_index = 0 + for db_vm_iface in db_vm_ifaces: + iface_uuid = str(uuid4()) + uuid_list.append(iface_uuid) + db_vm_iface_instance = { + "uuid": iface_uuid, + "instance_vm_id": vm_uuid + } + db_vm_iface_instance.update(db_vm_iface) + if db_vm_iface_instance.get("ip_address"): # increment ip_address + ip = db_vm_iface_instance.get("ip_address") + i = ip.rfind(".") + if i > 0: + try: + i += 1 + ip = ip[i:] + str(int(ip[:i]) +1) + db_vm_iface_instance["ip_address"] = ip + except: + db_vm_iface_instance["ip_address"] = None + db_instance_interfaces.append(db_vm_iface_instance) + myVMDict['networks'][iface_index]["uuid"] = iface_uuid + iface_index += 1 + + db_vim_action = { + "instance_action_id": instance_action_id, + "task_index": task_index, + "datacenter_vim_id": myvim_thread_id, + "action": "CREATE", + "status": "SCHEDULED", + "item": "instance_vms", + "item_id": vm_uuid, + "extra": yaml.safe_dump({"params": task_params, "depends_on": task_depends_on}, + default_flow_style=True, width=256) + } + task_index += 1 + db_vim_actions.append(db_vim_action) + scenarioDict["datacenter2tenant"] = myvim_threads_id - logger.debug("create_instance Deployment done scenarioDict: %s", - yaml.safe_dump(scenarioDict, indent=4, default_flow_style=False) ) - instance_id = mydb.new_instance_scenario_as_a_whole(tenant_id,instance_name, instance_description, scenarioDict) - for myvim_thread_id,task_list in tasks_to_launch.items(): - for task in task_list: - vim_threads["running"][myvim_thread_id].insert_task(task) - - global_instance_tasks[instance_id] = instance_tasks - # Update database with those ended instance_tasks - # for task in instance_tasks.values(): - # if task["status"] == "ok": - # if task["name"] == "new-vm": - # mydb.update_rows("instance_vms", UPDATE={"vim_vm_id": task["result"]}, - # WHERE={"vim_vm_id": task["id"]}) - # elif task["name"] == "new-net": - # mydb.update_rows("instance_nets", UPDATE={"vim_net_id": task["result"]}, - # WHERE={"vim_net_id": task["id"]}) - return mydb.get_instance_scenario(instance_id) - except (NfvoException, vimconn.vimconnException,db_base_Exception) as e: + + db_instance_action["number_tasks"] = task_index + db_instance_scenario['datacenter_tenant_id'] = myvim_threads_id[default_datacenter_id] + db_instance_scenario['datacenter_id'] = default_datacenter_id + db_tables=[ + {"instance_scenarios": db_instance_scenario}, + {"instance_vnfs": db_instance_vnfs}, + {"instance_nets": db_instance_nets}, + {"ip_profiles": db_ip_profiles}, + {"instance_vms": db_instance_vms}, + {"instance_interfaces": db_instance_interfaces}, + {"instance_actions": db_instance_action}, + {"vim_actions": db_vim_actions} + ] + + logger.debug("create_instance done DB tables: %s", + yaml.safe_dump(db_tables, indent=4, default_flow_style=False) ) + mydb.new_rows(db_tables, uuid_list) + for myvim_thread_id in myvim_threads_id.values(): + vim_threads["running"][myvim_thread_id].insert_task(db_vim_actions) + + returned_instance = mydb.get_instance_scenario(instance_uuid) + returned_instance["action_id"] = instance_action_id + return returned_instance + except (NfvoException, vimconn.vimconnException, db_base_Exception) as e: message = rollback(mydb, myvims, rollbackList) if isinstance(e, db_base_Exception): error_text = "database Exception" @@ -2347,29 +3265,43 @@ def create_instance(mydb, tenant_id, instance_dict): else: error_text = "Exception" error_text += " {} {}. {}".format(type(e).__name__, str(e), message) - #logger.error("create_instance: %s", error_text) + # logger.error("create_instance: %s", error_text) raise NfvoException(error_text, e.http_code) def delete_instance(mydb, tenant_id, instance_id): - #print "Checking that the instance_id exists and getting the instance dictionary" + # print "Checking that the instance_id exists and getting the instance dictionary" instanceDict = mydb.get_instance_scenario(instance_id, tenant_id) - #print yaml.safe_dump(instanceDict, indent=4, default_flow_style=False) + # print yaml.safe_dump(instanceDict, indent=4, default_flow_style=False) tenant_id = instanceDict["tenant_id"] - #print "Checking that nfvo_tenant_id exists and getting the VIM URI and the VIM tenant_id" + # print "Checking that nfvo_tenant_id exists and getting the VIM URI and the VIM tenant_id" - #1. Delete from Database + # 1. Delete from Database message = mydb.delete_instance_scenario(instance_id, tenant_id) - #2. delete from VIM + # 2. delete from VIM error_msg = "" myvims = {} myvim_threads = {} + vimthread_affected = {} + net2vm_dependencies = {} + + task_index = 0 + instance_action_id = get_task_id() + db_vim_actions = [] + db_instance_action = { + "uuid": instance_action_id, # same uuid for the instance and the action on create + "tenant_id": tenant_id, + "instance_id": instance_id, + "description": "DELETE", + # "number_tasks": 0 # filled bellow + } - #2.1 deleting VMs - #vm_fail_list=[] + # 2.1 deleting VMs + # vm_fail_list=[] for sce_vnf in instanceDict['vnfs']: datacenter_key = (sce_vnf["datacenter_id"], sce_vnf["datacenter_tenant_id"]) + vimthread_affected[sce_vnf["datacenter_tenant_id"]] = None if datacenter_key not in myvims: try: _,myvim_thread = get_vim_thread(mydb, tenant_id, sce_vnf["datacenter_id"], sce_vnf["datacenter_tenant_id"]) @@ -2391,41 +3323,30 @@ def delete_instance(mydb, tenant_id, instance_id): if not myvim: error_msg += "\n VM id={} cannot be deleted because datacenter={} not found".format(vm['vim_vm_id'], sce_vnf["datacenter_id"]) continue - try: - task=None - if is_task_id(vm['vim_vm_id']): - task_id = vm['vim_vm_id'] - old_task = global_instance_tasks[instance_id].get(task_id) - if not old_task: - error_msg += "\n VM was scheduled for create, but task {} is not found".format(task_id) - continue - with task_lock: - if old_task["status"] == "enqueued": - old_task["status"] = "deleted" - elif old_task["status"] == "error": - continue - elif old_task["status"] == "processing": - task = new_task("del-vm", (task_id, vm["interfaces"]), depends={task_id: old_task}) - else: #ok - task = new_task("del-vm", (old_task["result"], vm["interfaces"])) - else: - task = new_task("del-vm", (vm['vim_vm_id'], vm["interfaces"]) ) - if task: - myvim_thread.insert_task(task) - except vimconn.vimconnNotFoundException as e: - error_msg+="\n VM VIM_id={} not found at datacenter={}".format(vm['vim_vm_id'], sce_vnf["datacenter_id"]) - logger.warn("VM instance '%s'uuid '%s', VIM id '%s', from VNF_id '%s' not found", - vm['name'], vm['uuid'], vm['vim_vm_id'], sce_vnf['vnf_id']) - except vimconn.vimconnException as e: - error_msg+="\n VM VIM_id={} at datacenter={} Error: {} {}".format(vm['vim_vm_id'], sce_vnf["datacenter_id"], e.http_code, str(e)) - logger.error("Error %d deleting VM instance '%s'uuid '%s', VIM_id '%s', from VNF_id '%s': %s", - e.http_code, vm['name'], vm['uuid'], vm['vim_vm_id'], sce_vnf['vnf_id'], str(e)) + db_vim_action = { + "instance_action_id": instance_action_id, + "task_index": task_index, + "datacenter_vim_id": sce_vnf["datacenter_tenant_id"], + "action": "DELETE", + "status": "SCHEDULED", + "item": "instance_vms", + "item_id": vm["uuid"], + "extra": yaml.safe_dump({"params": vm["interfaces"]}, + default_flow_style=True, width=256) + } + db_vim_actions.append(db_vim_action) + for interface in vm["interfaces"]: + if not interface.get("instance_net_id"): + continue + if interface["instance_net_id"] not in net2vm_dependencies: + net2vm_dependencies[interface["instance_net_id"]] = [] + net2vm_dependencies[interface["instance_net_id"]].append(task_index) + task_index += 1 - #2.2 deleting NETS - #net_fail_list=[] + # 2.2 deleting NETS + # net_fail_list=[] for net in instanceDict['nets']: - if not net['created']: - continue #skip not created nets + vimthread_affected[net["datacenter_tenant_id"]] = None datacenter_key = (net["datacenter_id"], net["datacenter_tenant_id"]) if datacenter_key not in myvims: try: @@ -2447,41 +3368,39 @@ def delete_instance(mydb, tenant_id, instance_id): if not myvim: error_msg += "\n Net VIM_id={} cannot be deleted because datacenter={} not found".format(net['vim_net_id'], net["datacenter_id"]) continue - try: - task = None - if is_task_id(net['vim_net_id']): - task_id = net['vim_net_id'] - old_task = global_instance_tasks[instance_id].get(task_id) - if not old_task: - error_msg += "\n NET was scheduled for create, but task {} is not found".format(task_id) - continue - with task_lock: - if old_task["status"] == "enqueued": - old_task["status"] = "deleted" - elif old_task["status"] == "error": - continue - elif old_task["status"] == "processing": - task = new_task("del-net", task_id, depends={task_id: old_task}) - else: # ok - task = new_task("del-net", old_task["result"]) - else: - task = new_task("del-net", (net['vim_net_id'], net['sdn_net_id'])) - if task: - myvim_thread.insert_task(task) - except vimconn.vimconnNotFoundException as e: - error_msg += "\n NET VIM_id={} not found at datacenter={}".format(net['vim_net_id'], net["datacenter_id"]) - logger.warn("NET '%s', VIM_id '%s', from VNF_net_id '%s' not found", - net['uuid'], net['vim_net_id'], str(net['vnf_net_id'])) - except vimconn.vimconnException as e: - error_msg += "\n NET VIM_id={} at datacenter={} Error: {} {}".format(net['vim_net_id'], - net["datacenter_id"], - e.http_code, str(e)) - logger.error("Error %d deleting NET '%s', VIM_id '%s', from VNF_net_id '%s': %s", - e.http_code, net['uuid'], net['vim_net_id'], str(net['vnf_net_id']), str(e)) + extra = {"params": (net['vim_net_id'], net['sdn_net_id'])} + if net2vm_dependencies.get(net["uuid"]): + extra["depends_on"] = net2vm_dependencies[net["uuid"]] + db_vim_action = { + "instance_action_id": instance_action_id, + "task_index": task_index, + "datacenter_vim_id": net["datacenter_tenant_id"], + "action": "DELETE", + "status": "SCHEDULED", + "item": "instance_nets", + "item_id": net["uuid"], + "extra": yaml.safe_dump(extra, default_flow_style=True, width=256) + } + task_index += 1 + db_vim_actions.append(db_vim_action) + + db_instance_action["number_tasks"] = task_index + db_tables = [ + {"instance_actions": db_instance_action}, + {"vim_actions": db_vim_actions} + ] + + logger.debug("delete_instance done DB tables: %s", + yaml.safe_dump(db_tables, indent=4, default_flow_style=False)) + mydb.new_rows(db_tables, ()) + for myvim_thread_id in vimthread_affected.keys(): + vim_threads["running"][myvim_thread_id].insert_task(db_vim_actions) + if len(error_msg) > 0: - return 'instance ' + message + ' deleted but some elements could not be deleted, or already deleted (error: 404) from VIM: ' + error_msg + return 'action_id={} instance {} deleted but some elements could not be deleted, or already deleted '\ + '(error: 404) from VIM: {}'.format(instance_action_id, message, error_msg) else: - return 'instance ' + message + ' deleted' + return "action_id={} instance {} deleted".format(instance_action_id, message) def refresh_instance(mydb, nfvo_tenant, instanceDict, datacenter=None, vim_tenant=None): @@ -2648,7 +3567,6 @@ def refresh_instance(mydb, nfvo_tenant, instanceDict, datacenter=None, vim_tenan return 0, 'Scenario instance ' + instance_id + ' refreshed.' - def instance_action(mydb,nfvo_tenant,instance_id, action_dict): #print "Checking that the instance_id exists and getting the instance dictionary" instanceDict = mydb.get_instance_scenario(instance_id, nfvo_tenant) @@ -2660,6 +3578,13 @@ def instance_action(mydb,nfvo_tenant,instance_id, action_dict): raise NfvoException("datacenter '{}' not found".format(str(instanceDict['datacenter_id'])), HTTP_Not_Found) myvim = vims.values()[0] + if action_dict.get("create-vdu"): + for vdu in action_dict["create-vdu"]: + vdu_id = vdu.get("vdu-id") + vdu_count = vdu.get("count", 1) + # get from database TODO + # insert tasks TODO + pass input_vnfs = action_dict.pop("vnfs", []) input_vms = action_dict.pop("vms", []) @@ -2671,47 +3596,72 @@ def instance_action(mydb,nfvo_tenant,instance_id, action_dict): for vm in sce_vnf['vms']: if not action_over_all: if sce_vnf['uuid'] not in input_vnfs and sce_vnf['vnf_name'] not in input_vnfs and \ - vm['uuid'] not in input_vms and vm['name'] not in input_vms: + vm['uuid'] not in input_vms and vm['name'] not in input_vms: continue try: - data = myvim.action_vminstance(vm['vim_vm_id'], action_dict) - if "console" in action_dict: - if not global_config["http_console_proxy"]: - vm_result[ vm['uuid'] ] = {"vim_result": 200, - "description": "{protocol}//{ip}:{port}/{suffix}".format( - protocol=data["protocol"], - ip = data["server"], - port = data["port"], - suffix = data["suffix"]), - "name":vm['name'] - } - vm_ok +=1 - elif data["server"]=="127.0.0.1" or data["server"]=="localhost": - vm_result[ vm['uuid'] ] = {"vim_result": -HTTP_Unauthorized, - "description": "this console is only reachable by local interface", - "name":vm['name'] - } - vm_error+=1 - else: - #print "console data", data + if "add_public_key" in action_dict: + mgmt_access = {} + if sce_vnf.get('mgmt_access'): + mgmt_access = yaml.load(sce_vnf['mgmt_access']) + ssh_access = mgmt_access['config-access']['ssh-access'] + tenant = mydb.get_rows_by_id('nfvo_tenants', nfvo_tenant) try: - console_thread = create_or_use_console_proxy_thread(data["server"], data["port"]) + if ssh_access['required'] and ssh_access['default-user']: + if 'ip_address' in vm: + mgmt_ip = vm['ip_address'].split(';') + password = mgmt_access['config-access'].get('password') + priv_RO_key = decrypt_key(tenant[0]['encrypted_RO_priv_key'], tenant[0]['uuid']) + myvim.inject_user_key(mgmt_ip[0], ssh_access['default-user'], + action_dict['add_public_key'], + password=password, ro_key=priv_RO_key) + else: + raise NfvoException("Unable to inject ssh key in vm: {} - Aborting".format(vm['uuid']), + HTTP_Internal_Server_Error) + except KeyError: + raise NfvoException("Unable to inject ssh key in vm: {} - Aborting".format(vm['uuid']), + HTTP_Internal_Server_Error) + else: + raise NfvoException("Unable to inject ssh key in vm: {} - Aborting".format(vm['uuid']), + HTTP_Internal_Server_Error) + else: + data = myvim.action_vminstance(vm['vim_vm_id'], action_dict) + if "console" in action_dict: + if not global_config["http_console_proxy"]: vm_result[ vm['uuid'] ] = {"vim_result": 200, "description": "{protocol}//{ip}:{port}/{suffix}".format( protocol=data["protocol"], - ip = global_config["http_console_host"], - port = console_thread.port, + ip = data["server"], + port = data["port"], suffix = data["suffix"]), "name":vm['name'] } vm_ok +=1 - except NfvoException as e: - vm_result[ vm['uuid'] ] = {"vim_result": e.http_code, "name":vm['name'], "description": str(e)} + elif data["server"]=="127.0.0.1" or data["server"]=="localhost": + vm_result[ vm['uuid'] ] = {"vim_result": -HTTP_Unauthorized, + "description": "this console is only reachable by local interface", + "name":vm['name'] + } vm_error+=1 + else: + #print "console data", data + try: + console_thread = create_or_use_console_proxy_thread(data["server"], data["port"]) + vm_result[ vm['uuid'] ] = {"vim_result": 200, + "description": "{protocol}//{ip}:{port}/{suffix}".format( + protocol=data["protocol"], + ip = global_config["http_console_host"], + port = console_thread.port, + suffix = data["suffix"]), + "name":vm['name'] + } + vm_ok +=1 + except NfvoException as e: + vm_result[ vm['uuid'] ] = {"vim_result": e.http_code, "name":vm['name'], "description": str(e)} + vm_error+=1 - else: - vm_result[ vm['uuid'] ] = {"vim_result": 200, "description": "ok", "name":vm['name']} - vm_ok +=1 + else: + vm_result[ vm['uuid'] ] = {"vim_result": 200, "description": "ok", "name":vm['name']} + vm_ok +=1 except vimconn.vimconnException as e: vm_result[ vm['uuid'] ] = {"vim_result": e.http_code, "name":vm['name'], "description": str(e)} vm_error+=1 @@ -2721,6 +3671,19 @@ def instance_action(mydb,nfvo_tenant,instance_id, action_dict): else: return vm_result +def instance_action_get(mydb, nfvo_tenant, instance_id, action_id): + filter={} + if nfvo_tenant and nfvo_tenant != "any": + filter["tenant_id"] = nfvo_tenant + if instance_id and instance_id != "any": + filter["instance_id"] = instance_id + if action_id: + filter["uuid"] = action_id + rows = mydb.get_rows(FROM="instance_actions", WHERE=filter) + if not rows and action_id: + raise NfvoException("Not found any action with this criteria", HTTP_Not_Found) + return {"ations": rows} + def create_or_use_console_proxy_thread(console_server, console_port): #look for a non-used port @@ -2754,11 +3717,18 @@ def check_tenant(mydb, tenant_id): raise NfvoException("tenant '{}' not found".format(tenant_id), HTTP_Not_Found) return - def new_tenant(mydb, tenant_dict): - tenant_id = mydb.new_row("nfvo_tenants", tenant_dict, add_uuid=True) - return tenant_id + tenant_uuid = str(uuid4()) + tenant_dict['uuid'] = tenant_uuid + try: + pub_key, priv_key = create_RO_keypair(tenant_uuid) + tenant_dict['RO_pub_key'] = pub_key + tenant_dict['encrypted_RO_priv_key'] = priv_key + mydb.new_row("nfvo_tenants", tenant_dict, confidential_data=True) + except db_base_Exception as e: + raise NfvoException("Error creating the new tenant: {} ".format(tenant_dict['name']) + str(e), HTTP_Internal_Server_Error) + return tenant_uuid def delete_tenant(mydb, tenant): #get nfvo_tenant info @@ -2784,25 +3754,29 @@ def new_datacenter(mydb, datacenter_descriptor): # file.close(module_info[0]) raise NfvoException("Incorrect datacenter type '{}'. Plugin '{}'.py not installed".format(datacenter_type, module), HTTP_Bad_Request) - datacenter_id = mydb.new_row("datacenters", datacenter_descriptor, add_uuid=True) + datacenter_id = mydb.new_row("datacenters", datacenter_descriptor, add_uuid=True, confidential_data=True) return datacenter_id def edit_datacenter(mydb, datacenter_id_name, datacenter_descriptor): - #obtain data, check that only one exist + # obtain data, check that only one exist datacenter = mydb.get_table_by_uuid_name('datacenters', datacenter_id_name) - #edit data + + # edit data datacenter_id = datacenter['uuid'] where={'uuid': datacenter['uuid']} + remove_port_mapping = False if "config" in datacenter_descriptor: - if datacenter_descriptor['config']!=None: + if datacenter_descriptor['config'] != None: try: new_config_dict = datacenter_descriptor["config"] #delete null fields to_delete=[] for k in new_config_dict: - if new_config_dict[k]==None: + if new_config_dict[k] == None: to_delete.append(k) + if k == 'sdn-controller': + remove_port_mapping = True config_text = datacenter.get("config") if not config_text: @@ -2814,7 +3788,16 @@ def edit_datacenter(mydb, datacenter_id_name, datacenter_descriptor): del config_dict[k] except Exception as e: raise NfvoException("Bad format at datacenter:config " + str(e), HTTP_Bad_Request) - datacenter_descriptor["config"]= yaml.safe_dump(config_dict,default_flow_style=True,width=256) if len(config_dict)>0 else None + if config_dict: + datacenter_descriptor["config"] = yaml.safe_dump(config_dict, default_flow_style=True, width=256) + else: + datacenter_descriptor["config"] = None + if remove_port_mapping: + try: + datacenter_sdn_port_mapping_delete(mydb, None, datacenter_id) + except ovimException as e: + logger.error("Error deleting datacenter-port-mapping " + str(e)) + mydb.update_rows('datacenters', datacenter_descriptor, where) return datacenter_id @@ -2823,77 +3806,87 @@ def delete_datacenter(mydb, datacenter): #get nfvo_tenant info datacenter_dict = mydb.get_table_by_uuid_name('datacenters', datacenter, 'datacenter') mydb.delete_row_by_id("datacenters", datacenter_dict['uuid']) + try: + datacenter_sdn_port_mapping_delete(mydb, None, datacenter_dict['uuid']) + except ovimException as e: + logger.error("Error deleting datacenter-port-mapping " + str(e)) return datacenter_dict['uuid'] + " " + datacenter_dict['name'] def associate_datacenter_to_tenant(mydb, nfvo_tenant, datacenter, vim_tenant_id=None, vim_tenant_name=None, vim_username=None, vim_password=None, config=None): - #get datacenter info - datacenter_id, myvim = get_datacenter_by_name_uuid(mydb, None, datacenter, vim_user=vim_username, vim_passwd=vim_password) - datacenter_name = myvim["name"] - - create_vim_tenant = True if not vim_tenant_id and not vim_tenant_name else False - - # get nfvo_tenant info - tenant_dict = mydb.get_table_by_uuid_name('nfvo_tenants', nfvo_tenant) - if vim_tenant_name==None: - vim_tenant_name=tenant_dict['name'] - - #check that this association does not exist before - tenants_datacenter_dict={"nfvo_tenant_id":tenant_dict['uuid'], "datacenter_id":datacenter_id } - tenants_datacenters = mydb.get_rows(FROM='tenants_datacenters', WHERE=tenants_datacenter_dict) - if len(tenants_datacenters)>0: - raise NfvoException("datacenter '{}' and tenant'{}' are already attached".format(datacenter_id, tenant_dict['uuid']), HTTP_Conflict) - - vim_tenant_id_exist_atdb=False - if not create_vim_tenant: - where_={"datacenter_id": datacenter_id} - if vim_tenant_id!=None: - where_["vim_tenant_id"] = vim_tenant_id - if vim_tenant_name!=None: - where_["vim_tenant_name"] = vim_tenant_name - #check if vim_tenant_id is already at database - datacenter_tenants_dict = mydb.get_rows(FROM='datacenter_tenants', WHERE=where_) - if len(datacenter_tenants_dict)>=1: - datacenter_tenants_dict = datacenter_tenants_dict[0] - vim_tenant_id_exist_atdb=True - #TODO check if a field has changed and edit entry at datacenter_tenants at DB - else: #result=0 + # get datacenter info + try: + datacenter_id = get_datacenter_uuid(mydb, None, datacenter) + + create_vim_tenant = True if not vim_tenant_id and not vim_tenant_name else False + + # get nfvo_tenant info + tenant_dict = mydb.get_table_by_uuid_name('nfvo_tenants', nfvo_tenant) + if vim_tenant_name==None: + vim_tenant_name=tenant_dict['name'] + + #check that this association does not exist before + tenants_datacenter_dict={"nfvo_tenant_id":tenant_dict['uuid'], "datacenter_id":datacenter_id } + tenants_datacenters = mydb.get_rows(FROM='tenants_datacenters', WHERE=tenants_datacenter_dict) + if len(tenants_datacenters)>0: + raise NfvoException("datacenter '{}' and tenant'{}' are already attached".format(datacenter_id, tenant_dict['uuid']), HTTP_Conflict) + + vim_tenant_id_exist_atdb=False + if not create_vim_tenant: + where_={"datacenter_id": datacenter_id} + if vim_tenant_id!=None: + where_["vim_tenant_id"] = vim_tenant_id + if vim_tenant_name!=None: + where_["vim_tenant_name"] = vim_tenant_name + #check if vim_tenant_id is already at database + datacenter_tenants_dict = mydb.get_rows(FROM='datacenter_tenants', WHERE=where_) + if len(datacenter_tenants_dict)>=1: + datacenter_tenants_dict = datacenter_tenants_dict[0] + vim_tenant_id_exist_atdb=True + #TODO check if a field has changed and edit entry at datacenter_tenants at DB + else: #result=0 + datacenter_tenants_dict = {} + #insert at table datacenter_tenants + else: #if vim_tenant_id==None: + #create tenant at VIM if not provided + try: + _, myvim = get_datacenter_by_name_uuid(mydb, None, datacenter, vim_user=vim_username, + vim_passwd=vim_password) + datacenter_name = myvim["name"] + vim_tenant_id = myvim.new_tenant(vim_tenant_name, "created by openmano for datacenter "+datacenter_name) + except vimconn.vimconnException as e: + raise NfvoException("Not possible to create vim_tenant {} at VIM: {}".format(vim_tenant_id, str(e)), HTTP_Internal_Server_Error) datacenter_tenants_dict = {} - #insert at table datacenter_tenants - else: #if vim_tenant_id==None: - #create tenant at VIM if not provided - try: - vim_tenant_id = myvim.new_tenant(vim_tenant_name, "created by openmano for datacenter "+datacenter_name) - except vimconn.vimconnException as e: - raise NfvoException("Not possible to create vim_tenant {} at VIM: {}".format(vim_tenant_id, str(e)), HTTP_Internal_Server_Error) - datacenter_tenants_dict = {} - datacenter_tenants_dict["created"]="true" - - #fill datacenter_tenants table - if not vim_tenant_id_exist_atdb: - datacenter_tenants_dict["vim_tenant_id"] = vim_tenant_id - datacenter_tenants_dict["vim_tenant_name"] = vim_tenant_name - datacenter_tenants_dict["user"] = vim_username - datacenter_tenants_dict["passwd"] = vim_password - datacenter_tenants_dict["datacenter_id"] = datacenter_id - if config: - datacenter_tenants_dict["config"] = yaml.safe_dump(config, default_flow_style=True, width=256) - id_ = mydb.new_row('datacenter_tenants', datacenter_tenants_dict, add_uuid=True) - datacenter_tenants_dict["uuid"] = id_ - - #fill tenants_datacenters table - datacenter_tenant_id = datacenter_tenants_dict["uuid"] - tenants_datacenter_dict["datacenter_tenant_id"] = datacenter_tenant_id - mydb.new_row('tenants_datacenters', tenants_datacenter_dict) - # create thread - datacenter_id, myvim = get_datacenter_by_name_uuid(mydb, tenant_dict['uuid'], datacenter_id) # reload data - thread_name = get_non_used_vim_name(datacenter_name, datacenter_id, tenant_dict['name'], tenant_dict['uuid']) - new_thread = vim_thread.vim_thread(myvim, task_lock, thread_name, datacenter_name, datacenter_tenant_id, - db=db, db_lock=db_lock, ovim=ovim) - new_thread.start() - thread_id = datacenter_tenants_dict["uuid"] - vim_threads["running"][thread_id] = new_thread - return datacenter_id + datacenter_tenants_dict["created"]="true" + + #fill datacenter_tenants table + if not vim_tenant_id_exist_atdb: + datacenter_tenants_dict["vim_tenant_id"] = vim_tenant_id + datacenter_tenants_dict["vim_tenant_name"] = vim_tenant_name + datacenter_tenants_dict["user"] = vim_username + datacenter_tenants_dict["passwd"] = vim_password + datacenter_tenants_dict["datacenter_id"] = datacenter_id + if config: + datacenter_tenants_dict["config"] = yaml.safe_dump(config, default_flow_style=True, width=256) + id_ = mydb.new_row('datacenter_tenants', datacenter_tenants_dict, add_uuid=True, confidential_data=True) + datacenter_tenants_dict["uuid"] = id_ + + #fill tenants_datacenters table + datacenter_tenant_id = datacenter_tenants_dict["uuid"] + tenants_datacenter_dict["datacenter_tenant_id"] = datacenter_tenant_id + mydb.new_row('tenants_datacenters', tenants_datacenter_dict) + # create thread + datacenter_id, myvim = get_datacenter_by_name_uuid(mydb, tenant_dict['uuid'], datacenter_id) # reload data + datacenter_name = myvim["name"] + thread_name = get_non_used_vim_name(datacenter_name, datacenter_id, tenant_dict['name'], tenant_dict['uuid']) + new_thread = vim_thread.vim_thread(myvim, task_lock, thread_name, datacenter_name, datacenter_tenant_id, + db=db, db_lock=db_lock, ovim=ovim) + new_thread.start() + thread_id = datacenter_tenants_dict["uuid"] + vim_threads["running"][thread_id] = new_thread + return datacenter_id + except vimconn.vimconnException as e: + raise NfvoException(str(e), HTTP_Bad_Request) def edit_datacenter_to_tenant(mydb, nfvo_tenant, datacenter_id, vim_tenant_id=None, vim_tenant_name=None, @@ -2935,9 +3928,6 @@ def edit_datacenter_to_tenant(mydb, nfvo_tenant, datacenter_id, vim_tenant_id=No return datacenter_id def deassociate_datacenter_to_tenant(mydb, tenant_id, datacenter, vim_tenant_id=None): - #get datacenter info - datacenter_id, myvim = get_datacenter_by_name_uuid(mydb, None, datacenter) - #get nfvo_tenant info if not tenant_id or tenant_id=="any": tenant_uuid = None @@ -2945,8 +3935,9 @@ def deassociate_datacenter_to_tenant(mydb, tenant_id, datacenter, vim_tenant_id= tenant_dict = mydb.get_table_by_uuid_name('nfvo_tenants', tenant_id) tenant_uuid = tenant_dict['uuid'] + datacenter_id = get_datacenter_uuid(mydb, tenant_uuid, datacenter) #check that this association exist before - tenants_datacenter_dict={"datacenter_id":datacenter_id } + tenants_datacenter_dict={"datacenter_id": datacenter_id } if tenant_uuid: tenants_datacenter_dict["nfvo_tenant_id"] = tenant_uuid tenant_datacenter_list = mydb.get_rows(FROM='tenants_datacenters', WHERE=tenants_datacenter_dict) @@ -2966,6 +3957,7 @@ def deassociate_datacenter_to_tenant(mydb, tenant_id, datacenter, vim_tenant_id= if vim_tenant_dict['created']=='true': #delete tenant at VIM if created by NFVO try: + datacenter_id, myvim = get_datacenter_by_name_uuid(mydb, tenant_id, datacenter) myvim.delete_tenant(vim_tenant_dict['vim_tenant_id']) except vimconn.vimconnException as e: warning = "Not possible to delete vim_tenant_id {} from VIM: {} ".format(vim_tenant_dict['vim_tenant_id'], str(e)) @@ -2975,7 +3967,7 @@ def deassociate_datacenter_to_tenant(mydb, tenant_id, datacenter, vim_tenant_id= pass # the error will be caused because dependencies, vim_tenant can not be deleted thread_id = tenant_datacenter_item["datacenter_tenant_id"] thread = vim_threads["running"][thread_id] - thread.insert_task(new_task("exit", None)) + thread.insert_task("exit") vim_threads["deleting"][thread_id] = thread return "datacenter {} detached. {}".format(datacenter_id, warning) @@ -3090,8 +4082,7 @@ def get_sdn_net_id(mydb, tenant_id, datacenter, network_id): datacenter_id, myvim = get_datacenter_by_name_uuid(mydb, tenant_id, datacenter) network = myvim.get_network_list(filter_dict=filter_dict) except vimconn.vimconnException as e: - print "vim_action Not possible to get_%s_list from VIM: %s " % (item, str(e)) - raise NfvoException("Not possible to get_{}_list from VIM: {}".format(item, str(e)), e.http_code) + raise NfvoException("Not possible to get_sdn_net_id from VIM: {}".format(str(e)), e.http_code) # ensure the network is defined if len(network) == 0: @@ -3501,3 +4492,42 @@ def datacenter_sdn_port_mapping_list(mydb, tenant_id, datacenter_id): def datacenter_sdn_port_mapping_delete(mydb, tenant_id, datacenter_id): return ovim.clear_of_port_mapping(db_filter={"region":datacenter_id}) + +def create_RO_keypair(tenant_id): + """ + Creates a public / private keys for a RO tenant and returns their values + Params: + tenant_id: ID of the tenant + Return: + public_key: Public key for the RO tenant + private_key: Encrypted private key for RO tenant + """ + + bits = 2048 + key = RSA.generate(bits) + try: + public_key = key.publickey().exportKey('OpenSSH') + if isinstance(public_key, ValueError): + raise NfvoException("Unable to create public key: {}".format(public_key), HTTP_Internal_Server_Error) + private_key = key.exportKey(passphrase=tenant_id, pkcs=8) + except (ValueError, NameError) as e: + raise NfvoException("Unable to create private key: {}".format(e), HTTP_Internal_Server_Error) + return public_key, private_key + +def decrypt_key (key, tenant_id): + """ + Decrypts an encrypted RSA key + Params: + key: Private key to be decrypted + tenant_id: ID of the tenant + Return: + unencrypted_key: Unencrypted private key for RO tenant + """ + try: + key = RSA.importKey(key,tenant_id) + unencrypted_key = key.exportKey('PEM') + if isinstance(unencrypted_key, ValueError): + raise NfvoException("Unable to decrypt the private key: {}".format(unencrypted_key), HTTP_Internal_Server_Error) + except ValueError as e: + raise NfvoException("Unable to decrypt the private key: {}".format(e), HTTP_Internal_Server_Error) + return unencrypted_key