X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_ro%2Fnfvo.py;h=8cde0d6f4a23cbf69d0baf78cadf8d43144c7575;hb=1df468d2c3f6c32f2bb703e4732180a8596c48ad;hp=f6ec56806166996a071ecfd64cf502d9d6231c24;hpb=e18ba43ea07577aeebb54d4c2b5966f856f3063d;p=osm%2FRO.git diff --git a/osm_ro/nfvo.py b/osm_ro/nfvo.py index f6ec5680..8cde0d6f 100644 --- a/osm_ro/nfvo.py +++ b/osm_ro/nfvo.py @@ -38,6 +38,7 @@ import console_proxy_thread as cli import vimconn import logging import collections +import math from uuid import uuid4 from db_base import db_base_Exception @@ -51,7 +52,8 @@ from Crypto.PublicKey import RSA import osm_im.vnfd as vnfd_catalog import osm_im.nsd as nsd_catalog from pyangbind.lib.serialise import pybindJSONDecoder -from itertools import chain +from copy import deepcopy + global global_config global vimconn_imported @@ -138,9 +140,14 @@ def start_service(mydb): 'log_level_of': 'DEBUG' } try: + # starts ovim library ovim = ovim_module.ovim(ovim_configuration) ovim.start_service() + #delete old unneeded vim_actions + clean_db(mydb) + + # starts vim_threads from_= 'tenants_datacenters as td join datacenters as d on td.datacenter_id=d.uuid join '\ 'datacenter_tenants as dt on td.datacenter_tenant_id=dt.uuid' select_ = ('type', 'd.config as config', 'd.uuid as datacenter_id', 'vim_url', 'vim_url_admin', @@ -224,6 +231,37 @@ def get_version(): return ("openmanod version {} {}\n(c) Copyright Telefonica".format(global_config["version"], global_config["version_date"] )) +def clean_db(mydb): + """ + Clean unused or old entries at database to avoid unlimited growing + :param mydb: database connector + :return: None + """ + # get and delete unused vim_actions: all elements deleted, one week before, instance not present + now = t.time()-3600*24*7 + instance_action_id = None + nb_deleted = 0 + while True: + actions_to_delete = mydb.get_rows( + SELECT=("item", "item_id", "instance_action_id"), + FROM="vim_actions as va join instance_actions as ia on va.instance_action_id=ia.uuid " + "left join instance_scenarios as i on ia.instance_id=i.uuid", + WHERE={"va.action": "DELETE", "va.modified_at<": now, "i.uuid": None, + "va.status": ("DONE", "SUPERSEDED")}, + LIMIT=100 + ) + for to_delete in actions_to_delete: + mydb.delete_row(FROM="vim_actions", WHERE=to_delete) + if instance_action_id != to_delete["instance_action_id"]: + instance_action_id = to_delete["instance_action_id"] + mydb.delete_row(FROM="instance_actions", WHERE={"uuid": instance_action_id}) + nb_deleted += len(actions_to_delete) + if len(actions_to_delete) < 100: + break + if nb_deleted: + logger.debug("Removed {} unused vim_actions".format(nb_deleted)) + + def get_flavorlist(mydb, vnf_id, nfvo_tenant=None): '''Obtain flavorList @@ -248,26 +286,28 @@ def get_flavorlist(mydb, vnf_id, nfvo_tenant=None): def get_imagelist(mydb, vnf_id, nfvo_tenant=None): - '''Obtain imageList - return result, content: - <0, error_text upon error - nb_records, flavor_list on success - ''' - WHERE_dict={} - WHERE_dict['vnf_id'] = vnf_id - if nfvo_tenant is not None: - WHERE_dict['nfvo_tenant_id'] = nfvo_tenant - - #result, content = mydb.get_table(FROM='vms join vnfs on vms-vnf_id = vnfs.uuid',SELECT=('uuid'),WHERE=WHERE_dict ) - images = mydb.get_rows(FROM='vms join images on vms.image_id=images.uuid',SELECT=('image_id',),WHERE=WHERE_dict ) - imageList=[] - for image in images: - imageList.append(image['image_id']) - return imageList + """ + Get used images of all vms belonging to this VNFD + :param mydb: database conector + :param vnf_id: vnfd uuid + :param nfvo_tenant: tenant, not used + :return: The list of image uuid used + """ + image_list = [] + vms = mydb.get_rows(SELECT=('image_id','image_list'), FROM='vms', WHERE={'vnf_id': vnf_id}) + for vm in vms: + if vm["image_id"] not in image_list: + image_list.append(vm["image_id"]) + if vm["image_list"]: + vm_image_list = yaml.load(vm["image_list"]) + for image_dict in vm_image_list: + if image_dict["image_id"] not in image_list: + image_list.append(image_dict["image_id"]) + return image_list def get_vim(mydb, nfvo_tenant=None, datacenter_id=None, datacenter_name=None, datacenter_tenant_id=None, - vim_tenant=None, vim_tenant_name=None, vim_user=None, vim_passwd=None): + vim_tenant=None, vim_tenant_name=None, vim_user=None, vim_passwd=None, ignore_errors=False): '''Obtain a dictionary of VIM (datacenter) classes with some of the input parameters return dictionary with {datacenter_id: vim_class, ... }. vim_class contain: 'nfvo_tenant_id','datacenter_id','vim_tenant_id','vim_url','vim_url_admin','datacenter_name','type','user','passwd' @@ -293,7 +333,8 @@ def get_vim(mydb, nfvo_tenant=None, datacenter_id=None, datacenter_name=None, da vim_dict={} for vim in vims: extra={'datacenter_tenant_id': vim.get('datacenter_tenant_id'), - 'datacenter_id': vim.get('datacenter_id')} + 'datacenter_id': vim.get('datacenter_id'), + '_vim_type_internal': vim.get('type')} if vim["config"]: extra.update(yaml.load(vim["config"])) if vim.get('dt_config'): @@ -310,6 +351,10 @@ def get_vim(mydb, nfvo_tenant=None, datacenter_id=None, datacenter_name=None, da except (IOError, ImportError) as e: # if module_info and module_info[0]: # file.close(module_info[0]) + if ignore_errors: + logger.error("Unknown vim type '{}'. Can not open file '{}.py'; {}: {}".format( + vim["type"], module, type(e).__name__, str(e))) + continue raise NfvoException("Unknown vim type '{}'. Can not open file '{}.py'; {}: {}".format( vim["type"], module, type(e).__name__, str(e)), HTTP_Bad_Request) @@ -332,7 +377,13 @@ def get_vim(mydb, nfvo_tenant=None, datacenter_id=None, datacenter_name=None, da config=extra, persistent_info=persistent_info ) except Exception as e: - raise NfvoException("Error at VIM {}; {}: {}".format(vim["type"], type(e).__name__, str(e)), HTTP_Internal_Server_Error) + if ignore_errors: + logger.error("Error at VIM {}; {}: {}".format(vim["type"], type(e).__name__, str(e))) + continue + http_code = HTTP_Internal_Server_Error + if isinstance(e, vimconn.vimconnException): + http_code = e.http_code + raise NfvoException("Error at VIM {}; {}: {}".format(vim["type"], type(e).__name__, str(e)), http_code) return vim_dict except db_base_Exception as e: raise NfvoException(str(e) + " at nfvo.get_vim", e.http_code) @@ -355,7 +406,7 @@ def rollback(mydb, vims, rollback_list): mydb.delete_row(FROM="datacenters_images", WHERE={"datacenter_vim_id": vim["id"], "vim_id":item["uuid"]}) elif item["what"]=="flavor": vim.delete_flavor(item["uuid"]) - mydb.delete_row(FROM="datacenters_flavors", WHERE={"datacenter_id": vim["id"], "vim_id":item["uuid"]}) + mydb.delete_row(FROM="datacenters_flavors", WHERE={"datacenter_vim_id": vim["id"], "vim_id":item["uuid"]}) elif item["what"]=="network": vim.delete_network(item["uuid"]) elif item["what"]=="vm": @@ -584,7 +635,7 @@ def create_or_use_image(mydb, vims, image_dict, rollback_list, only_create_at_vi def create_or_use_flavor(mydb, vims, flavor_dict, rollback_list, only_create_at_vim=False, return_on_error = None): - temp_flavor_dict= {'disk':flavor_dict.get('disk',1), + temp_flavor_dict= {'disk':flavor_dict.get('disk',0), 'ram':flavor_dict.get('ram'), 'vcpus':flavor_dict.get('vcpus'), } @@ -671,7 +722,7 @@ def create_or_use_flavor(mydb, vims, flavor_dict, rollback_list, only_create_at_ device=devices_original[index] if "image" not in device and "image name" not in device: if 'size' in device: - disk_list.append({'size': device.get('size', default_volume_size)}) + disk_list.append({'size': device.get('size', default_volume_size), 'name': device.get('name')}) continue image_dict={} image_dict['name']=device.get('image name',flavor_dict['name']+str(dev_nb)+"-img") @@ -791,7 +842,7 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): try: myvnfd = vnfd_catalog.vnfd() try: - pybindJSONDecoder.load_ietf_json(vnf_descriptor, None, None, obj=myvnfd) + pybindJSONDecoder.load_ietf_json(vnf_descriptor, None, None, obj=myvnfd, path_helper=True) except Exception as e: raise NfvoException("Error. Invalid VNF descriptor format " + str(e), HTTP_Bad_Request) db_vnfs = [] @@ -801,6 +852,8 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): db_interfaces = [] db_images = [] db_flavors = [] + db_ip_profiles_index = 0 + db_ip_profiles = [] uuid_list = [] vnfd_uuid_list = [] vnfd_catalog_descriptor = vnf_descriptor.get("vnfd:vnfd-catalog") @@ -816,9 +869,10 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): vnf_uuid = str(uuid4()) uuid_list.append(vnf_uuid) vnfd_uuid_list.append(vnf_uuid) + vnfd_id = get_str(vnfd, "id", 255) db_vnf = { "uuid": vnf_uuid, - "osm_id": get_str(vnfd, "id", 255), + "osm_id": vnfd_id, "name": get_str(vnfd, "name", 255), "description": get_str(vnfd, "description", 255), "tenant_id": tenant_id, @@ -831,6 +885,27 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): if vnfd_descriptor["id"] == str(vnfd["id"]): break + # table ip_profiles (ip-profiles) + ip_profile_name2db_table_index = {} + for ip_profile in vnfd.get("ip-profiles").itervalues(): + db_ip_profile = { + "ip_version": str(ip_profile["ip-profile-params"].get("ip-version", "ipv4")), + "subnet_address": str(ip_profile["ip-profile-params"].get("subnet-address")), + "gateway_address": str(ip_profile["ip-profile-params"].get("gateway-address")), + "dhcp_enabled": str(ip_profile["ip-profile-params"]["dhcp-params"].get("enabled", True)), + "dhcp_start_address": str(ip_profile["ip-profile-params"]["dhcp-params"].get("start-address")), + "dhcp_count": str(ip_profile["ip-profile-params"]["dhcp-params"].get("count")), + } + dns_list = [] + for dns in ip_profile["ip-profile-params"]["dns-server"].itervalues(): + dns_list.append(str(dns.get("address"))) + db_ip_profile["dns_address"] = ";".join(dns_list) + if ip_profile["ip-profile-params"].get('security-group'): + db_ip_profile["security_group"] = ip_profile["ip-profile-params"]['security-group'] + ip_profile_name2db_table_index[str(ip_profile["name"])] = db_ip_profiles_index + db_ip_profiles_index += 1 + db_ip_profiles.append(db_ip_profile) + # table nets (internal-vld) net_id2uuid = {} # for mapping interface with network for vld in vnfd.get("internal-vld").itervalues(): @@ -841,20 +916,47 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): "vnf_id": vnf_uuid, "uuid": net_uuid, "description": get_str(vld, "description", 255), + "osm_id": get_str(vld, "id", 255), "type": "bridge", # TODO adjust depending on connection point type } net_id2uuid[vld.get("id")] = net_uuid db_nets.append(db_net) + # ip-profile, link db_ip_profile with db_sce_net + if vld.get("ip-profile-ref"): + ip_profile_name = vld.get("ip-profile-ref") + if ip_profile_name not in ip_profile_name2db_table_index: + raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{}]':'vld[{}]':'ip-profile-ref':" + "'{}'. Reference to a non-existing 'ip_profiles'".format( + str(vnfd["id"]), str(vld["id"]), str(vld["ip-profile-ref"])), + HTTP_Bad_Request) + db_ip_profiles[ip_profile_name2db_table_index[ip_profile_name]]["net_id"] = net_uuid + else: #check no ip-address has been defined + for icp in vld.get("internal-connection-point").itervalues(): + if icp.get("ip-address"): + raise NfvoException("Error at 'vnfd[{}]':'vld[{}]':'internal-connection-point[{}]' " + "contains an ip-address but no ip-profile has been defined at VLD".format( + str(vnfd["id"]), str(vld["id"]), str(icp["id"])), + HTTP_Bad_Request) + + # connection points vaiable declaration + cp_name2iface_uuid = {} + cp_name2vm_uuid = {} + cp_name2db_interface = {} # table vms (vdus) vdu_id2uuid = {} vdu_id2db_table_index = {} for vdu in vnfd.get("vdu").itervalues(): + + for vdu_descriptor in vnfd_descriptor["vdu"]: + if vdu_descriptor["id"] == str(vdu["id"]): + break vm_uuid = str(uuid4()) uuid_list.append(vm_uuid) + vdu_id = get_str(vdu, "id", 255) db_vm = { "uuid": vm_uuid, - "osm_id": get_str(vdu, "id", 255), + "osm_id": vdu_id, "name": get_str(vdu, "name", 255), "description": get_str(vdu, "description", 255), "vnf_id": vnf_uuid, @@ -874,11 +976,27 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): image_uuid = db_image["uuid"] db_images.append(db_image) db_vm["image_id"] = image_uuid + if vdu.get("alternative-images"): + vm_alternative_images = [] + for alt_image in vdu.get("alternative-images").itervalues(): + db_image = {} + image_uuid = _lookfor_or_create_image(db_image, mydb, alt_image) + if not image_uuid: + image_uuid = db_image["uuid"] + db_images.append(db_image) + vm_alternative_images.append({ + "image_id": image_uuid, + "vim_type": str(alt_image["vim-type"]), + # "universal_name": str(alt_image["image"]), + # "checksum": str(alt_image["image-checksum"]) if alt_image.get("image-checksum") else None + }) + + db_vm["image_list"] = yaml.safe_dump(vm_alternative_images, default_flow_style=True, width=256) # volumes devices = [] if vdu.get("volumes"): - for volume_key in sorted(vdu["volumes"]): + for volume_key in vdu["volumes"]: volume = vdu["volumes"][volume_key] if not image_present: # Convert the first volume to vnfc.image @@ -891,7 +1009,7 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): db_vm["image_id"] = image_uuid else: # Add Openmano devices - device = {} + device = {"name": str(volume.get("name"))} device["type"] = str(volume.get("device-type")) if volume.get("size"): device["size"] = int(volume["size"]) @@ -899,75 +1017,8 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): device["image name"] = str(volume["image"]) if volume.get("image-checksum"): device["image checksum"] = str(volume["image-checksum"]) - devices.append(device) - - # table flavors - db_flavor = { - "name": get_str(vdu, "name", 250) + "-flv", - "vcpus": int(vdu["vm-flavor"].get("vcpu-count", 1)), - "ram": int(vdu["vm-flavor"].get("memory-mb", 1)), - "disk": int(vdu["vm-flavor"].get("storage-gb", 1)), - } - # EPA TODO revise - extended = {} - numa = {} - if devices: - extended["devices"] = devices - if vdu.get("guest-epa"): # TODO or dedicated_int: - epa_vcpu_set = False - if vdu["guest-epa"].get("numa-node-policy"): # TODO or dedicated_int: - numa_node_policy = vdu["guest-epa"].get("numa-node-policy") - if numa_node_policy.get("node"): - numa_node = numa_node_policy["node"]['0'] - if numa_node.get("num-cores"): - numa["cores"] = numa_node["num-cores"] - epa_vcpu_set = True - if numa_node.get("paired-threads"): - if numa_node["paired-threads"].get("num-paired-threads"): - numa["paired-threads"] = int(numa_node["paired-threads"]["num-paired-threads"]) - epa_vcpu_set = True - if len(numa_node["paired-threads"].get("paired-thread-ids")): - numa["paired-threads-id"] = [] - for pair in numa_node["paired-threads"]["paired-thread-ids"].itervalues(): - numa["paired-threads-id"].append( - (str(pair["thread-a"]), str(pair["thread-b"])) - ) - if numa_node.get("num-threads"): - numa["threads"] = int(numa_node["num-threads"]) - epa_vcpu_set = True - if numa_node.get("memory-mb"): - numa["memory"] = max(int(numa_node["memory-mb"] / 1024), 1) - if vdu["guest-epa"].get("mempage-size"): - if vdu["guest-epa"]["mempage-size"] != "SMALL": - numa["memory"] = max(int(db_flavor["ram"] / 1024), 1) - if vdu["guest-epa"].get("cpu-pinning-policy") and not epa_vcpu_set: - if vdu["guest-epa"]["cpu-pinning-policy"] == "DEDICATED": - if vdu["guest-epa"].get("cpu-thread-pinning-policy") and \ - vdu["guest-epa"]["cpu-thread-pinning-policy"] != "PREFER": - numa["cores"] = max(db_flavor["vcpus"], 1) - else: - numa["threads"] = max(db_flavor["vcpus"], 1) - if numa: - extended["numas"] = [numa] - if extended: - extended_text = yaml.safe_dump(extended, default_flow_style=True, width=256) - db_flavor["extended"] = extended_text - # look if flavor exist - temp_flavor_dict = {'disk': db_flavor.get('disk', 1), - 'ram': db_flavor.get('ram'), - 'vcpus': db_flavor.get('vcpus'), - 'extended': db_flavor.get('extended') - } - existing_flavors = mydb.get_rows(FROM="flavors", WHERE=temp_flavor_dict) - if existing_flavors: - flavor_uuid = existing_flavors[0]["uuid"] - else: - flavor_uuid = str(uuid4()) - uuid_list.append(flavor_uuid) - db_flavor["uuid"] = flavor_uuid - db_flavors.append(db_flavor) - db_vm["flavor_id"] = flavor_uuid + devices.append(device) # cloud-init boot_data = {} @@ -996,10 +1047,11 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): db_vms_index += 1 # table interfaces (internal/external interfaces) - cp_name2iface_uuid = {} - cp_name2vm_uuid = {} + flavor_epa_interfaces = [] + vdu_id2cp_name = {} # stored only when one external connection point is presented at this VDU # for iface in chain(vdu.get("internal-interface").itervalues(), vdu.get("external-interface").itervalues()): for iface in vdu.get("interface").itervalues(): + flavor_epa_interface = {} iface_uuid = str(uuid4()) uuid_list.append(iface_uuid) db_interface = { @@ -1007,12 +1059,15 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): "internal_name": get_str(iface, "name", 255), "vm_id": vm_uuid, } + flavor_epa_interface["name"] = db_interface["internal_name"] if iface.get("virtual-interface").get("vpci"): db_interface["vpci"] = get_str(iface.get("virtual-interface"), "vpci", 12) + flavor_epa_interface["vpci"] = db_interface["vpci"] if iface.get("virtual-interface").get("bandwidth"): bps = int(iface.get("virtual-interface").get("bandwidth")) - db_interface["bw"] = bps/1000 + db_interface["bw"] = int(math.ceil(bps/1000000.0)) + flavor_epa_interface["bandwidth"] = "{} Mbps".format(db_interface["bw"]) if iface.get("virtual-interface").get("type") == "OM-MGMT": db_interface["type"] = "mgmt" @@ -1022,11 +1077,13 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): elif iface.get("virtual-interface").get("type") in ("SR-IOV", "PCI-PASSTHROUGH"): db_interface["type"] = "data" db_interface["model"] = get_str(iface.get("virtual-interface"), "type", 12) + flavor_epa_interface["dedicated"] = "no" if iface["virtual-interface"]["type"] == "SR-IOV" \ + else "yes" + flavor_epa_interfaces.append(flavor_epa_interface) else: raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{}]':'vdu[{}]':'interface':'virtual" "-interface':'type':'{}'. Interface type is not supported".format( - str(vnfd["id"])[:255], str(vdu["id"])[:255], - iface.get("virtual-interface").get("type")), + vnfd_id, vdu_id, iface.get("virtual-interface").get("type")), HTTP_Bad_Request) if iface.get("external-connection-point-ref"): @@ -1035,9 +1092,19 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): db_interface["external_name"] = get_str(cp, "name", 255) cp_name2iface_uuid[db_interface["external_name"]] = iface_uuid cp_name2vm_uuid[db_interface["external_name"]] = vm_uuid + cp_name2db_interface[db_interface["external_name"]] = db_interface for cp_descriptor in vnfd_descriptor["connection-point"]: if cp_descriptor["name"] == db_interface["external_name"]: break + else: + raise KeyError() + + if vdu_id in vdu_id2cp_name: + vdu_id2cp_name[vdu_id] = None # more than two connecdtion point for this VDU + else: + vdu_id2cp_name[vdu_id] = db_interface["external_name"] + + # port security if str(cp_descriptor.get("port-security-enabled")).lower() == "false": db_interface["port_security"] = 0 elif str(cp_descriptor.get("port-security-enabled")).lower() == "true": @@ -1046,34 +1113,119 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'vdu[{vdu}]':" "'interface[{iface}]':'vnfd-connection-point-ref':'{cp}' is not present" " at connection-point".format( - vnf=vnfd["id"], vdu=vdu["id"], iface=iface["name"], + vnf=vnfd_id, vdu=vdu_id, iface=iface["name"], cp=iface.get("vnfd-connection-point-ref")), HTTP_Bad_Request) elif iface.get("internal-connection-point-ref"): try: + for icp_descriptor in vdu_descriptor["internal-connection-point"]: + if icp_descriptor["id"] == str(iface.get("internal-connection-point-ref")): + break + else: + raise KeyError("does not exist at vdu:internal-connection-point") + icp = None + icp_vld = None for vld in vnfd.get("internal-vld").itervalues(): for cp in vld.get("internal-connection-point").itervalues(): if cp.get("id-ref") == iface.get("internal-connection-point-ref"): - db_interface["net_id"] = net_id2uuid[vld.get("id")] - for cp_descriptor in vnfd_descriptor["connection-point"]: - if cp_descriptor["name"] == db_interface["external_name"]: - break - if str(cp_descriptor.get("port-security-enabled")).lower() == "false": - db_interface["port_security"] = 0 - elif str(cp_descriptor.get("port-security-enabled")).lower() == "true": - db_interface["port_security"] = 1 - break - except KeyError: + if icp: + raise KeyError("is referenced by more than one 'internal-vld'") + icp = cp + icp_vld = vld + if not icp: + raise KeyError("is not referenced by any 'internal-vld'") + + db_interface["net_id"] = net_id2uuid[icp_vld.get("id")] + if str(icp_descriptor.get("port-security-enabled")).lower() == "false": + db_interface["port_security"] = 0 + elif str(icp_descriptor.get("port-security-enabled")).lower() == "true": + db_interface["port_security"] = 1 + if icp.get("ip-address"): + if not icp_vld.get("ip-profile-ref"): + raise NfvoException + db_interface["ip_address"] = str(icp.get("ip-address")) + except KeyError as e: raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'vdu[{vdu}]':" - "'interface[{iface}]':'vdu-internal-connection-point-ref':'{cp}' is not" - " referenced by any internal-vld".format( - vnf=vnfd["id"], vdu=vdu["id"], iface=iface["name"], - cp=iface.get("vdu-internal-connection-point-ref")), + "'interface[{iface}]':'internal-connection-point-ref':'{cp}'" + " {msg}".format( + vnf=vnfd_id, vdu=vdu_id, iface=iface["name"], + cp=iface.get("internal-connection-point-ref"), msg=str(e)), HTTP_Bad_Request) - if iface.get("position") is not None: - db_interface["created_at"] = int(iface.get("position")) - 1000 + if iface.get("position"): + db_interface["created_at"] = int(iface.get("position")) * 50 + if iface.get("mac-address"): + db_interface["mac"] = str(iface.get("mac-address")) db_interfaces.append(db_interface) + # table flavors + db_flavor = { + "name": get_str(vdu, "name", 250) + "-flv", + "vcpus": int(vdu["vm-flavor"].get("vcpu-count", 1)), + "ram": int(vdu["vm-flavor"].get("memory-mb", 1)), + "disk": int(vdu["vm-flavor"].get("storage-gb", 0)), + } + # TODO revise the case of several numa-node-policy node + extended = {} + numa = {} + if devices: + extended["devices"] = devices + if flavor_epa_interfaces: + numa["interfaces"] = flavor_epa_interfaces + if vdu.get("guest-epa"): # TODO or dedicated_int: + epa_vcpu_set = False + if vdu["guest-epa"].get("numa-node-policy"): # TODO or dedicated_int: + numa_node_policy = vdu["guest-epa"].get("numa-node-policy") + if numa_node_policy.get("node"): + numa_node = numa_node_policy["node"].values()[0] + if numa_node.get("num-cores"): + numa["cores"] = numa_node["num-cores"] + epa_vcpu_set = True + if numa_node.get("paired-threads"): + if numa_node["paired-threads"].get("num-paired-threads"): + numa["paired-threads"] = int(numa_node["paired-threads"]["num-paired-threads"]) + epa_vcpu_set = True + if len(numa_node["paired-threads"].get("paired-thread-ids")): + numa["paired-threads-id"] = [] + for pair in numa_node["paired-threads"]["paired-thread-ids"].itervalues(): + numa["paired-threads-id"].append( + (str(pair["thread-a"]), str(pair["thread-b"])) + ) + if numa_node.get("num-threads"): + numa["threads"] = int(numa_node["num-threads"]) + epa_vcpu_set = True + if numa_node.get("memory-mb"): + numa["memory"] = max(int(numa_node["memory-mb"] / 1024), 1) + if vdu["guest-epa"].get("mempage-size"): + if vdu["guest-epa"]["mempage-size"] != "SMALL": + numa["memory"] = max(int(db_flavor["ram"] / 1024), 1) + if vdu["guest-epa"].get("cpu-pinning-policy") and not epa_vcpu_set: + if vdu["guest-epa"]["cpu-pinning-policy"] == "DEDICATED": + if vdu["guest-epa"].get("cpu-thread-pinning-policy") and \ + vdu["guest-epa"]["cpu-thread-pinning-policy"] != "PREFER": + numa["cores"] = max(db_flavor["vcpus"], 1) + else: + numa["threads"] = max(db_flavor["vcpus"], 1) + if numa: + extended["numas"] = [numa] + if extended: + extended_text = yaml.safe_dump(extended, default_flow_style=True, width=256) + db_flavor["extended"] = extended_text + # look if flavor exist + temp_flavor_dict = {'disk': db_flavor.get('disk', 0), + 'ram': db_flavor.get('ram'), + 'vcpus': db_flavor.get('vcpus'), + 'extended': db_flavor.get('extended') + } + existing_flavors = mydb.get_rows(FROM="flavors", WHERE=temp_flavor_dict) + if existing_flavors: + flavor_uuid = existing_flavors[0]["uuid"] + else: + flavor_uuid = str(uuid4()) + uuid_list.append(flavor_uuid) + db_flavor["uuid"] = flavor_uuid + db_flavors.append(db_flavor) + db_vm["flavor_id"] = flavor_uuid + # VNF affinity and antiaffinity for pg in vnfd.get("placement-groups").itervalues(): pg_name = get_str(pg, "name", 255) @@ -1082,7 +1234,7 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): if vdu_id not in vdu_id2db_table_index: raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'placement-groups[{pg}]':" "'member-vdus':'{vdu}'. Reference to a non-existing vdu".format( - vnf=vnfd["id"], pg=pg_name, vdu=vdu_id), + vnf=vnfd_id, pg=pg_name, vdu=vdu_id), HTTP_Bad_Request) db_vms[vdu_id2db_table_index[vdu_id]]["availability_zone"] = pg_name # TODO consider the case of isolation and not colocation @@ -1091,22 +1243,30 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): # VNF mgmt configuration mgmt_access = {} if vnfd["mgmt-interface"].get("vdu-id"): - if vnfd["mgmt-interface"]["vdu-id"] not in vdu_id2uuid: + mgmt_vdu_id = get_str(vnfd["mgmt-interface"], "vdu-id", 255) + if mgmt_vdu_id not in vdu_id2uuid: raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'mgmt-interface':'vdu-id':" "'{vdu}'. Reference to a non-existing vdu".format( - vnf=vnfd["id"], vdu=vnfd["mgmt-interface"]["vdu-id"]), + vnf=vnfd_id, vdu=mgmt_vdu_id), HTTP_Bad_Request) mgmt_access["vm_id"] = vdu_id2uuid[vnfd["mgmt-interface"]["vdu-id"]] + # if only one cp is defined by this VDU, mark this interface as of type "mgmt" + if vdu_id2cp_name.get(mgmt_vdu_id): + cp_name2db_interface[vdu_id2cp_name[mgmt_vdu_id]]["type"] = "mgmt" + if vnfd["mgmt-interface"].get("ip-address"): mgmt_access["ip-address"] = str(vnfd["mgmt-interface"].get("ip-address")) if vnfd["mgmt-interface"].get("cp"): if vnfd["mgmt-interface"]["cp"] not in cp_name2iface_uuid: raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'mgmt-interface':'cp':'{cp}'. " "Reference to a non-existing connection-point".format( - vnf=vnfd["id"], cp=vnfd["mgmt-interface"]["cp"]), + vnf=vnfd_id, cp=vnfd["mgmt-interface"]["cp"]), HTTP_Bad_Request) mgmt_access["vm_id"] = cp_name2vm_uuid[vnfd["mgmt-interface"]["cp"]] mgmt_access["interface_id"] = cp_name2iface_uuid[vnfd["mgmt-interface"]["cp"]] + # mark this interface as of type mgmt + cp_name2db_interface[vnfd["mgmt-interface"]["cp"]]["type"] = "mgmt" + default_user = get_str(vnfd.get("vnf-configuration", {}).get("config-access", {}).get("ssh-access", {}), "default-user", 64) @@ -1120,14 +1280,13 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): if mgmt_access: db_vnf["mgmt_access"] = yaml.safe_dump(mgmt_access, default_flow_style=True, width=256) - - db_vnfs.append(db_vnf) db_tables=[ {"vnfs": db_vnfs}, {"nets": db_nets}, {"images": db_images}, {"flavors": db_flavors}, + {"ip_profiles": db_ip_profiles}, {"vms": db_vms}, {"interfaces": db_interfaces}, ] @@ -1160,7 +1319,7 @@ def new_vnf(mydb, tenant_id, vnf_descriptor): vnf_descriptor['vnf']['tenant_id'] = tenant_id # Step 3. Get the URL of the VIM from the nfvo_tenant and the datacenter if global_config["auto_push_VNF_to_VIMs"]: - vims = get_vim(mydb, tenant_id) + vims = get_vim(mydb, tenant_id, ignore_errors=True) # Step 4. Review the descriptor and add missing fields #print vnf_descriptor @@ -1193,7 +1352,7 @@ def new_vnf(mydb, tenant_id, vnf_descriptor): myflavorDict["description"] = VNFCitem["description"] myflavorDict["ram"] = vnfc.get("ram", 0) myflavorDict["vcpus"] = vnfc.get("vcpus", 0) - myflavorDict["disk"] = vnfc.get("disk", 1) + myflavorDict["disk"] = vnfc.get("disk", 0) myflavorDict["extended"] = {} devices = vnfc.get("devices") @@ -1297,7 +1456,7 @@ def new_vnf_v02(mydb, tenant_id, vnf_descriptor): vnf_descriptor['vnf']['tenant_id'] = tenant_id # Step 3. Get the URL of the VIM from the nfvo_tenant and the datacenter if global_config["auto_push_VNF_to_VIMs"]: - vims = get_vim(mydb, tenant_id) + vims = get_vim(mydb, tenant_id, ignore_errors=True) # Step 4. Review the descriptor and add missing fields #print vnf_descriptor @@ -1329,7 +1488,7 @@ def new_vnf_v02(mydb, tenant_id, vnf_descriptor): myflavorDict["description"] = VNFCitem["description"] myflavorDict["ram"] = vnfc.get("ram", 0) myflavorDict["vcpus"] = vnfc.get("vcpus", 0) - myflavorDict["disk"] = vnfc.get("disk", 1) + myflavorDict["disk"] = vnfc.get("disk", 0) myflavorDict["extended"] = {} devices = vnfc.get("devices") @@ -1469,8 +1628,7 @@ def get_vnf_id(mydb, tenant_id, vnf_id): content = mydb.get_rows(FROM='vnfs join vms on vnfs.uuid=vms.vnf_id join interfaces on vms.uuid=interfaces.vm_id',\ SELECT=('interfaces.uuid as uuid','interfaces.external_name as external_name', 'vms.name as vm_name', 'interfaces.vm_id as vm_id', \ 'interfaces.internal_name as internal_name', 'interfaces.type as type', 'interfaces.vpci as vpci','interfaces.bw as bw'),\ - WHERE={'vnfs.uuid': vnf_id}, - WHERE_NOT={'interfaces.external_name': None} ) + WHERE={'vnfs.uuid': vnf_id, 'interfaces.external_name<>': None} ) #print content data['vnf']['external-connections'] = content @@ -1482,7 +1640,7 @@ def delete_vnf(mydb,tenant_id,vnf_id,datacenter=None,vim_tenant=None): if tenant_id != "any": check_tenant(mydb, tenant_id) # Get the URL of the VIM from the nfvo_tenant and the datacenter - vims = get_vim(mydb, tenant_id) + vims = get_vim(mydb, tenant_id, ignore_errors=True) else: vims={} @@ -1517,32 +1675,39 @@ def delete_vnf(mydb,tenant_id,vnf_id,datacenter=None,vim_tenant=None): continue #flavor not used, must be deleted #delelte at VIM - c = mydb.get_rows(FROM='datacenters_flavors', WHERE={'flavor_id':flavor}) + c = mydb.get_rows(FROM='datacenters_flavors', WHERE={'flavor_id': flavor}) for flavor_vim in c: - if flavor_vim["datacenter_vim_id"] not in vims: # TODO change to datacenter_tenant_id + if not flavor_vim['created']: # skip this flavor because not created by openmano continue - if flavor_vim['created']=='false': #skip this flavor because not created by openmano + # look for vim + myvim = None + for vim in vims.values(): + if vim["config"]["datacenter_tenant_id"] == flavor_vim["datacenter_vim_id"]: + myvim = vim + break + if not myvim: continue - myvim=vims[ flavor_vim["datacenter_id"] ] try: myvim.delete_flavor(flavor_vim["vim_id"]) - except vimconn.vimconnNotFoundException as e: - logger.warn("VIM flavor %s not exist at datacenter %s", flavor_vim["vim_id"], flavor_vim["datacenter_id"] ) + except vimconn.vimconnNotFoundException: + logger.warn("VIM flavor %s not exist at datacenter %s", flavor_vim["vim_id"], + flavor_vim["datacenter_vim_id"] ) except vimconn.vimconnException as e: logger.error("Not possible to delete VIM flavor %s from datacenter %s: %s %s", - flavor_vim["vim_id"], flavor_vim["datacenter_id"], type(e).__name__, str(e)) - undeletedItems.append("flavor {} from VIM {}".format(flavor_vim["vim_id"], flavor_vim["datacenter_id"] )) - #delete flavor from Database, using table flavors and with cascade foreign key also at datacenters_flavors + flavor_vim["vim_id"], flavor_vim["datacenter_vim_id"], type(e).__name__, str(e)) + undeletedItems.append("flavor {} from VIM {}".format(flavor_vim["vim_id"], + flavor_vim["datacenter_vim_id"])) + # delete flavor from Database, using table flavors and with cascade foreign key also at datacenters_flavors mydb.delete_row_by_id('flavors', flavor) except db_base_Exception as e: logger.error("delete_vnf_error. Not possible to get flavor details and delete '%s'. %s", flavor, str(e)) - undeletedItems.append("flavor %s" % flavor) + undeletedItems.append("flavor {}".format(flavor)) for image in imageList: try: #check if image is used by other vnf - c = mydb.get_rows(FROM='vms', WHERE={'image_id':image} ) + c = mydb.get_rows(FROM='vms', WHERE=[{'image_id': image}, {'image_list LIKE ': '%' + image + '%'}]) if len(c) > 0: logger.debug("Image '%s' not deleted because it is being used by another VNF", image) continue @@ -1657,8 +1822,7 @@ def new_scenario(mydb, tenant_id, topo): #1.2: Check that VNF are present at database table vnfs. Insert uuid, description and external interfaces for name,vnf in vnfs.items(): - where={} - where_or={"tenant_id": tenant_id, 'public': "true"} + where = {"OR": {"tenant_id": tenant_id, 'public': "true"}} error_text = "" error_pos = "'topology':'nodes':'" + name + "'" if 'vnf_id' in vnf: @@ -1667,14 +1831,12 @@ def new_scenario(mydb, tenant_id, topo): if 'VNF model' in vnf: error_text += " 'VNF model' " + vnf['VNF model'] where['name'] = vnf['VNF model'] - if len(where) == 0: + if len(where) == 1: raise NfvoException("Descriptor need a 'vnf_id' or 'VNF model' field at " + error_pos, HTTP_Bad_Request) vnf_db = mydb.get_rows(SELECT=('uuid','name','description'), FROM='vnfs', - WHERE=where, - WHERE_OR=where_or, - WHERE_AND_OR="AND") + WHERE=where) if len(vnf_db)==0: raise NfvoException("unknown" + error_text + " at " + error_pos, HTTP_Not_Found) elif len(vnf_db)>1: @@ -1684,7 +1846,7 @@ def new_scenario(mydb, tenant_id, topo): #get external interfaces ext_ifaces = mydb.get_rows(SELECT=('external_name as name','i.uuid as iface_uuid', 'i.type as type'), FROM='vnfs join vms on vnfs.uuid=vms.vnf_id join interfaces as i on vms.uuid=i.vm_id', - WHERE={'vnfs.uuid':vnf['uuid']}, WHERE_NOT={'external_name':None} ) + WHERE={'vnfs.uuid':vnf['uuid'], 'external_name<>': None} ) for ext_iface in ext_ifaces: vnf['ifaces'][ ext_iface['name'] ] = {'uuid':ext_iface['iface_uuid'], 'type':ext_iface['type']} @@ -1923,8 +2085,7 @@ def new_scenario_v02(mydb, tenant_id, scenario_dict, version): # 1: Check that VNF are present at database table vnfs and update content into scenario dict for name,vnf in scenario["vnfs"].iteritems(): - where={} - where_or={"tenant_id": tenant_id, 'public': "true"} + where = {"OR": {"tenant_id": tenant_id, 'public': "true"}} error_text = "" error_pos = "'scenario':'vnfs':'" + name + "'" if 'vnf_id' in vnf: @@ -1933,13 +2094,11 @@ def new_scenario_v02(mydb, tenant_id, scenario_dict, version): if 'vnf_name' in vnf: error_text += " 'vnf_name' " + vnf['vnf_name'] where['name'] = vnf['vnf_name'] - if len(where) == 0: + if len(where) == 1: raise NfvoException("Needed a 'vnf_id' or 'vnf_name' at " + error_pos, HTTP_Bad_Request) vnf_db = mydb.get_rows(SELECT=('uuid', 'name', 'description'), FROM='vnfs', - WHERE=where, - WHERE_OR=where_or, - WHERE_AND_OR="AND") + WHERE=where) if len(vnf_db) == 0: raise NfvoException("Unknown" + error_text + " at " + error_pos, HTTP_Not_Found) elif len(vnf_db) > 1: @@ -1950,7 +2109,7 @@ def new_scenario_v02(mydb, tenant_id, scenario_dict, version): # get external interfaces ext_ifaces = mydb.get_rows(SELECT=('external_name as name', 'i.uuid as iface_uuid', 'i.type as type'), FROM='vnfs join vms on vnfs.uuid=vms.vnf_id join interfaces as i on vms.uuid=i.vm_id', - WHERE={'vnfs.uuid':vnf['uuid']}, WHERE_NOT={'external_name': None} ) + WHERE={'vnfs.uuid':vnf['uuid'], 'external_name<>': None} ) for ext_iface in ext_ifaces: vnf['ifaces'][ ext_iface['name'] ] = {'uuid':ext_iface['iface_uuid'], 'type': ext_iface['type']} # TODO? get internal-connections from db.nets and their profiles, and update scenario[vnfs][internal-connections] accordingly @@ -2038,7 +2197,7 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): :param mydb: :param tenant_id: :param nsd_descriptor: - :return: The list of cretated NSD ids + :return: The list of created NSD ids """ try: mynsd = nsd_catalog.nsd() @@ -2050,6 +2209,11 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): db_sce_nets = [] db_sce_vnfs = [] db_sce_interfaces = [] + db_sce_vnffgs = [] + db_sce_rsps = [] + db_sce_rsp_hops = [] + db_sce_classifiers = [] + db_sce_classifier_matches = [] db_ip_profiles = [] db_ip_profiles_index = 0 uuid_list = [] @@ -2057,7 +2221,7 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): for nsd_yang in mynsd.nsd_catalog.nsd.itervalues(): nsd = nsd_yang.get() - # table sceanrios + # table scenarios scenario_uuid = str(uuid4()) uuid_list.append(scenario_uuid) nsd_uuid_list.append(scenario_uuid) @@ -2089,13 +2253,14 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): db_sce_vnf = { "uuid": sce_vnf_uuid, "scenario_id": scenario_uuid, - "name": existing_vnf[0]["name"][:200] + "." + get_str(vnf, "member-vnf-index", 5), + # "name": get_str(vnf, "member-vnf-index", 255), + "name": existing_vnf[0]["name"][:200] + "." + get_str(vnf, "member-vnf-index", 50), "vnf_id": existing_vnf[0]["uuid"], - "member_vnf_index": int(vnf["member-vnf-index"]), + "member_vnf_index": str(vnf["member-vnf-index"]), # TODO 'start-by-default': True } - vnf_index2scevnf_uuid[int(vnf['member-vnf-index'])] = sce_vnf_uuid - vnf_index2vnf_uuid[int(vnf['member-vnf-index'])] = existing_vnf[0]["uuid"] + vnf_index2scevnf_uuid[str(vnf['member-vnf-index'])] = sce_vnf_uuid + vnf_index2vnf_uuid[str(vnf['member-vnf-index'])] = existing_vnf[0]["uuid"] db_sce_vnfs.append(db_sce_vnf) # table ip_profiles (ip-profiles) @@ -2129,6 +2294,7 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): "scenario_id": scenario_uuid, # "type": #TODO "multipoint": not vld.get("type") == "ELINE", + "osm_id": get_str(vld, "id", 255), # "external": #TODO "description": get_str(vld, "description", 255), } @@ -2139,7 +2305,8 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): elif vld.get("provider-network").get("overlay-type") == "VLAN": db_sce_net["type"] = "data" else: - db_sce_net["type"] = "bridge" + # later on it will be fixed to bridge or data depending on the type of interfaces attached to it + db_sce_net["type"] = None db_sce_nets.append(db_sce_net) # ip-profile, link db_ip_profile with db_sce_net @@ -2151,10 +2318,12 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): str(nsd["id"]), str(vld["id"]), str(vld["ip-profile-ref"])), HTTP_Bad_Request) db_ip_profiles[ip_profile_name2db_table_index[ip_profile_name]]["sce_net_id"] = sce_net_uuid + elif vld.get("vim-network-name"): + db_sce_net["vim_network_name"] = get_str(vld, "vim-network-name", 255) # table sce_interfaces (vld:vnfd-connection-point-ref) for iface in vld.get("vnfd-connection-point-ref").itervalues(): - vnf_index = int(iface['member-vnf-index-ref']) + vnf_index = str(iface['member-vnf-index-ref']) # check correct parameters if vnf_index not in vnf_index2vnf_uuid: raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'vld[{}]':'vnfd-connection-point" @@ -2163,7 +2332,7 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): str(nsd["id"]), str(vld["id"]), str(iface["member-vnf-index-ref"])), HTTP_Bad_Request) - existing_ifaces = mydb.get_rows(SELECT=('i.uuid as uuid',), + existing_ifaces = mydb.get_rows(SELECT=('i.uuid as uuid', 'i.type as iface_type'), FROM="interfaces as i join vms on i.vm_id=vms.uuid", WHERE={'vnf_id': vnf_index2vnf_uuid[vnf_index], 'external_name': get_str(iface, "vnfd-connection-point-ref", @@ -2176,16 +2345,144 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): str(iface.get("vnfd-id-ref"))[:255]), HTTP_Bad_Request) interface_uuid = existing_ifaces[0]["uuid"] + if existing_ifaces[0]["iface_type"] == "data" and not db_sce_net["type"]: + db_sce_net["type"] = "data" sce_interface_uuid = str(uuid4()) uuid_list.append(sce_net_uuid) + iface_ip_address = None + if iface.get("ip-address"): + iface_ip_address = str(iface.get("ip-address")) db_sce_interface = { "uuid": sce_interface_uuid, "sce_vnf_id": vnf_index2scevnf_uuid[vnf_index], "sce_net_id": sce_net_uuid, "interface_id": interface_uuid, - # "ip_address": #TODO + "ip_address": iface_ip_address, } db_sce_interfaces.append(db_sce_interface) + if not db_sce_net["type"]: + db_sce_net["type"] = "bridge" + + # table sce_vnffgs (vnffgd) + for vnffg in nsd.get("vnffgd").itervalues(): + sce_vnffg_uuid = str(uuid4()) + uuid_list.append(sce_vnffg_uuid) + db_sce_vnffg = { + "uuid": sce_vnffg_uuid, + "name": get_str(vnffg, "name", 255), + "scenario_id": scenario_uuid, + "vendor": get_str(vnffg, "vendor", 255), + "description": get_str(vld, "description", 255), + } + db_sce_vnffgs.append(db_sce_vnffg) + + # deal with rsps + db_sce_rsps = [] + for rsp in vnffg.get("rsp").itervalues(): + sce_rsp_uuid = str(uuid4()) + uuid_list.append(sce_rsp_uuid) + db_sce_rsp = { + "uuid": sce_rsp_uuid, + "name": get_str(rsp, "name", 255), + "sce_vnffg_id": sce_vnffg_uuid, + "id": get_str(rsp, "id", 255), # only useful to link with classifiers; will be removed later in the code + } + db_sce_rsps.append(db_sce_rsp) + db_sce_rsp_hops = [] + for iface in rsp.get("vnfd-connection-point-ref").itervalues(): + vnf_index = str(iface['member-vnf-index-ref']) + if_order = int(iface['order']) + # check correct parameters + if vnf_index not in vnf_index2vnf_uuid: + raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'rsp[{}]':'vnfd-connection-point" + "-ref':'member-vnf-index-ref':'{}'. Reference to a non-existing index at " + "'nsd':'constituent-vnfd'".format( + str(nsd["id"]), str(rsp["id"]), str(iface["member-vnf-index-ref"])), + HTTP_Bad_Request) + + existing_ifaces = mydb.get_rows(SELECT=('i.uuid as uuid',), + FROM="interfaces as i join vms on i.vm_id=vms.uuid", + WHERE={'vnf_id': vnf_index2vnf_uuid[vnf_index], + 'external_name': get_str(iface, "vnfd-connection-point-ref", + 255)}) + if not existing_ifaces: + raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'rsp[{}]':'vnfd-connection-point" + "-ref':'vnfd-connection-point-ref':'{}'. Reference to a non-existing " + "connection-point name at VNFD '{}'".format( + str(nsd["id"]), str(rsp["id"]), str(iface["vnfd-connection-point-ref"]), + str(iface.get("vnfd-id-ref"))[:255]), + HTTP_Bad_Request) + interface_uuid = existing_ifaces[0]["uuid"] + sce_rsp_hop_uuid = str(uuid4()) + uuid_list.append(sce_rsp_hop_uuid) + db_sce_rsp_hop = { + "uuid": sce_rsp_hop_uuid, + "if_order": if_order, + "interface_id": interface_uuid, + "sce_vnf_id": vnf_index2scevnf_uuid[vnf_index], + "sce_rsp_id": sce_rsp_uuid, + } + db_sce_rsp_hops.append(db_sce_rsp_hop) + + # deal with classifiers + db_sce_classifiers = [] + for classifier in vnffg.get("classifier").itervalues(): + sce_classifier_uuid = str(uuid4()) + uuid_list.append(sce_classifier_uuid) + + # source VNF + vnf_index = str(classifier['member-vnf-index-ref']) + if vnf_index not in vnf_index2vnf_uuid: + raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'classifier[{}]':'vnfd-connection-point" + "-ref':'member-vnf-index-ref':'{}'. Reference to a non-existing index at " + "'nsd':'constituent-vnfd'".format( + str(nsd["id"]), str(classifier["id"]), str(classifier["member-vnf-index-ref"])), + HTTP_Bad_Request) + existing_ifaces = mydb.get_rows(SELECT=('i.uuid as uuid',), + FROM="interfaces as i join vms on i.vm_id=vms.uuid", + WHERE={'vnf_id': vnf_index2vnf_uuid[vnf_index], + 'external_name': get_str(classifier, "vnfd-connection-point-ref", + 255)}) + if not existing_ifaces: + raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'rsp[{}]':'vnfd-connection-point" + "-ref':'vnfd-connection-point-ref':'{}'. Reference to a non-existing " + "connection-point name at VNFD '{}'".format( + str(nsd["id"]), str(rsp["id"]), str(iface["vnfd-connection-point-ref"]), + str(iface.get("vnfd-id-ref"))[:255]), + HTTP_Bad_Request) + interface_uuid = existing_ifaces[0]["uuid"] + + db_sce_classifier = { + "uuid": sce_classifier_uuid, + "name": get_str(classifier, "name", 255), + "sce_vnffg_id": sce_vnffg_uuid, + "sce_vnf_id": vnf_index2scevnf_uuid[vnf_index], + "interface_id": interface_uuid, + } + rsp_id = get_str(classifier, "rsp-id-ref", 255) + rsp = next((item for item in db_sce_rsps if item["id"] == rsp_id), None) + db_sce_classifier["sce_rsp_id"] = rsp["uuid"] + db_sce_classifiers.append(db_sce_classifier) + + db_sce_classifier_matches = [] + for match in classifier.get("match-attributes").itervalues(): + sce_classifier_match_uuid = str(uuid4()) + uuid_list.append(sce_classifier_match_uuid) + db_sce_classifier_match = { + "uuid": sce_classifier_match_uuid, + "ip_proto": get_str(match, "ip-proto", 2), + "source_ip": get_str(match, "source-ip-address", 16), + "destination_ip": get_str(match, "destination-ip-address", 16), + "source_port": get_str(match, "source-port", 5), + "destination_port": get_str(match, "destination-port", 5), + "sce_classifier_id": sce_classifier_uuid, + } + db_sce_classifier_matches.append(db_sce_classifier_match) + # TODO: vnf/cp keys + + # remove unneeded id's in sce_rsps + for rsp in db_sce_rsps: + rsp.pop('id') db_tables = [ {"scenarios": db_scenarios}, @@ -2193,9 +2490,14 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): {"ip_profiles": db_ip_profiles}, {"sce_vnfs": db_sce_vnfs}, {"sce_interfaces": db_sce_interfaces}, + {"sce_vnffgs": db_sce_vnffgs}, + {"sce_rsps": db_sce_rsps}, + {"sce_rsp_hops": db_sce_rsp_hops}, + {"sce_classifiers": db_sce_classifiers}, + {"sce_classifier_matches": db_sce_classifier_matches}, ] - logger.debug("create_vnf Deployment done vnfDict: %s", + logger.debug("new_nsd_v3 done: %s", yaml.safe_dump(db_tables, indent=4, default_flow_style=False) ) mydb.new_rows(db_tables, uuid_list) return nsd_uuid_list @@ -2404,7 +2706,7 @@ def start_scenario(mydb, tenant_id, scenario_id, instance_scenario_name, instanc else: av_index = None - vm_id = myvim.new_vminstance(myVMDict['name'], myVMDict['description'], myVMDict.get('start', None), + vm_id, _ = myvim.new_vminstance(myVMDict['name'], myVMDict['description'], myVMDict.get('start', None), myVMDict['imageRef'], myVMDict['flavorRef'], myVMDict['networks'], availability_zone_index=av_index, availability_zone_list=vnf_availability_zones) @@ -2606,6 +2908,7 @@ def update(d, u): d[k] = u[k] return d + def create_instance(mydb, tenant_id, instance_dict): # print "Checking that nfvo_tenant_id exists and getting the VIM URI and the VIM tenant_id" # logger.debug("Creating instance...") @@ -2620,8 +2923,7 @@ def create_instance(mydb, tenant_id, instance_dict): myvim_threads_id[default_datacenter_id], _ = get_vim_thread(mydb, tenant_id, default_datacenter_id) tenant = mydb.get_rows_by_id('nfvo_tenants', tenant_id) # myvim_tenant = myvim['tenant_id'] - - rollbackList=[] + rollbackList = [] # print "Checking that the scenario exists and getting the scenario dictionary" scenarioDict = mydb.get_scenario(scenario, tenant_id, datacenter_vim_id=myvim_threads_id[default_datacenter_id], @@ -2634,6 +2936,10 @@ def create_instance(mydb, tenant_id, instance_dict): db_instance_vnfs = [] db_instance_vms = [] db_instance_interfaces = [] + db_instance_sfis = [] + db_instance_sfs = [] + db_instance_classifications = [] + db_instance_sfps = [] db_ip_profiles = [] db_vim_actions = [] uuid_list = [] @@ -2662,22 +2968,38 @@ def create_instance(mydb, tenant_id, instance_dict): } # Auxiliary dictionaries from x to y - vnf_net2instance = {} sce_net2instance = {} net2task_id = {'scenario': {}} + def ip_profile_IM2RO(ip_profile_im): + # translate from input format to database format + ip_profile_ro = {} + if 'subnet-address' in ip_profile_im: + ip_profile_ro['subnet_address'] = ip_profile_im['subnet-address'] + if 'ip-version' in ip_profile_im: + ip_profile_ro['ip_version'] = ip_profile_im['ip-version'] + if 'gateway-address' in ip_profile_im: + ip_profile_ro['gateway_address'] = ip_profile_im['gateway-address'] + if 'dns-address' in ip_profile_im: + ip_profile_ro['dns_address'] = ip_profile_im['dns-address'] + if isinstance(ip_profile_ro['dns_address'], (list, tuple)): + ip_profile_ro['dns_address'] = ";".join(ip_profile_ro['dns_address']) + if 'dhcp' in ip_profile_im: + ip_profile_ro['dhcp_start_address'] = ip_profile_im['dhcp'].get('start-address') + ip_profile_ro['dhcp_enabled'] = ip_profile_im['dhcp'].get('enabled', True) + ip_profile_ro['dhcp_count'] = ip_profile_im['dhcp'].get('count') + return ip_profile_ro + # logger.debug("Creating instance from scenario-dict:\n%s", # yaml.safe_dump(scenarioDict, indent=4, default_flow_style=False)) try: # 0 check correct parameters for net_name, net_instance_desc in instance_dict.get("networks", {}).iteritems(): - found = False for scenario_net in scenarioDict['nets']: - if net_name == scenario_net["name"]: - found = True + if net_name == scenario_net.get("name") or net_name == scenario_net.get("osm_id") or net_name == scenario_net.get("uuid"): break - if not found: - raise NfvoException("Invalid scenario network name '{}' at instance:networks".format(net_name), + else: + raise NfvoException("Invalid scenario network name or id '{}' at instance:networks".format(net_name), HTTP_Bad_Request) if "sites" not in net_instance_desc: net_instance_desc["sites"] = [ {} ] @@ -2699,13 +3021,11 @@ def create_instance(mydb, tenant_id, instance_dict): site["datacenter"] = default_datacenter_id # change name to id for vnf_name, vnf_instance_desc in instance_dict.get("vnfs",{}).iteritems(): - found = False for scenario_vnf in scenarioDict['vnfs']: - if vnf_name == scenario_vnf['name']: - found = True + if vnf_name == scenario_vnf['member_vnf_index'] or vnf_name == scenario_vnf['uuid'] or vnf_name == scenario_vnf['name']: break - if not found: - raise NfvoException("Invalid vnf name '{}' at instance:vnfs".format(vnf_instance_desc), HTTP_Bad_Request) + else: + raise NfvoException("Invalid vnf name '{}' at instance:vnfs".format(vnf_name), HTTP_Bad_Request) if "datacenter" in vnf_instance_desc: # Add this datacenter to myvims vnf_instance_desc["datacenter"] = get_datacenter_uuid(mydb, tenant_id, vnf_instance_desc["datacenter"]) @@ -2715,67 +3035,104 @@ def create_instance(mydb, tenant_id, instance_dict): myvim_threads_id[d], _ = get_vim_thread(mydb, tenant_id, vnf_instance_desc["datacenter"]) scenario_vnf["datacenter"] = vnf_instance_desc["datacenter"] + for net_id, net_instance_desc in vnf_instance_desc.get("networks", {}).iteritems(): + for scenario_net in scenario_vnf['nets']: + if net_id == scenario_net['osm_id'] or net_id == scenario_net['uuid'] or net_id == scenario_net["name"]: + break + else: + raise NfvoException("Invalid net id or name '{}' at instance:vnfs:networks".format(net_id), HTTP_Bad_Request) + if net_instance_desc.get("vim-network-name"): + scenario_net["vim-network-name"] = net_instance_desc["vim-network-name"] + if net_instance_desc.get("name"): + scenario_net["name"] = net_instance_desc["name"] + if 'ip-profile' in net_instance_desc: + ipprofile_db = ip_profile_IM2RO(net_instance_desc['ip-profile']) + if 'ip_profile' not in scenario_net: + scenario_net['ip_profile'] = ipprofile_db + else: + update(scenario_net['ip_profile'], ipprofile_db) + + for vdu_id, vdu_instance_desc in vnf_instance_desc.get("vdus", {}).iteritems(): + for scenario_vm in scenario_vnf['vms']: + if vdu_id == scenario_vm['osm_id'] or vdu_id == scenario_vm["name"]: + break + else: + raise NfvoException("Invalid vdu id or name '{}' at instance:vnfs:vdus".format(vdu_id), HTTP_Bad_Request) + scenario_vm["instance_parameters"] = vdu_instance_desc + for iface_id, iface_instance_desc in vdu_instance_desc.get("interfaces", {}).iteritems(): + for scenario_interface in scenario_vm['interfaces']: + if iface_id == scenario_interface['internal_name'] or iface_id == scenario_interface["external_name"]: + scenario_interface.update(iface_instance_desc) + break + else: + raise NfvoException("Invalid vdu id or name '{}' at instance:vnfs:vdus".format(vdu_id), HTTP_Bad_Request) + # 0.1 parse cloud-config parameters cloud_config = unify_cloud_config(instance_dict.get("cloud-config"), scenarioDict.get("cloud-config")) - # We add the RO key to cloud_config - if tenant[0].get('RO_pub_key'): - RO_key = {"key-pairs": [tenant[0]['RO_pub_key']]} - cloud_config = unify_cloud_config(cloud_config, RO_key) # 0.2 merge instance information into scenario # Ideally, the operation should be as simple as: update(scenarioDict,instance_dict) # However, this is not possible yet. - for net_name, net_instance_desc in instance_dict.get("networks",{}).iteritems(): + for net_name, net_instance_desc in instance_dict.get("networks", {}).iteritems(): for scenario_net in scenarioDict['nets']: if net_name == scenario_net["name"]: if 'ip-profile' in net_instance_desc: - # translate from input format to database format - ipprofile_in = net_instance_desc['ip-profile'] - ipprofile_db = {} - ipprofile_db['subnet_address'] = ipprofile_in.get('subnet-address') - ipprofile_db['ip_version'] = ipprofile_in.get('ip-version', 'IPv4') - ipprofile_db['gateway_address'] = ipprofile_in.get('gateway-address') - ipprofile_db['dns_address'] = ipprofile_in.get('dns-address') - if isinstance(ipprofile_db['dns_address'], (list, tuple)): - ipprofile_db['dns_address'] = ";".join(ipprofile_db['dns_address']) - if 'dhcp' in ipprofile_in: - ipprofile_db['dhcp_start_address'] = ipprofile_in['dhcp'].get('start-address') - ipprofile_db['dhcp_enabled'] = ipprofile_in['dhcp'].get('enabled', True) - ipprofile_db['dhcp_count'] = ipprofile_in['dhcp'].get('count' ) + ipprofile_db = ip_profile_IM2RO(net_instance_desc['ip-profile']) if 'ip_profile' not in scenario_net: scenario_net['ip_profile'] = ipprofile_db else: update(scenario_net['ip_profile'], ipprofile_db) - for interface in net_instance_desc.get('interfaces', () ): + for interface in net_instance_desc.get('interfaces', ()): if 'ip_address' in interface: for vnf in scenarioDict['vnfs']: if interface['vnf'] == vnf['name']: for vnf_interface in vnf['interfaces']: if interface['vnf_interface'] == vnf_interface['external_name']: - vnf_interface['ip_address']=interface['ip_address'] + vnf_interface['ip_address'] = interface['ip_address'] # logger.debug(">>>>>>>> Merged dictionary") # logger.debug("Creating instance scenario-dict MERGED:\n%s", # yaml.safe_dump(scenarioDict, indent=4, default_flow_style=False)) # 1. Creating new nets (sce_nets) in the VIM" + number_mgmt_networks = 0 db_instance_nets = [] for sce_net in scenarioDict['nets']: - descriptor_net = instance_dict.get("networks", {}).get(sce_net["name"], {}) + # get involved datacenters where this network need to be created + involved_datacenters = [] + for sce_vnf in scenarioDict.get("vnfs"): + vnf_datacenter = sce_vnf.get("datacenter", default_datacenter_id) + if vnf_datacenter in involved_datacenters: + continue + if sce_vnf.get("interfaces"): + for sce_vnf_ifaces in sce_vnf["interfaces"]: + if sce_vnf_ifaces.get("sce_net_id") == sce_net["uuid"]: + involved_datacenters.append(vnf_datacenter) + break + + descriptor_net = {} + if instance_dict.get("networks") and instance_dict["networks"].get(sce_net["name"]): + descriptor_net = instance_dict["networks"][sce_net["name"]] net_name = descriptor_net.get("vim-network-name") sce_net2instance[sce_net['uuid']] = {} net2task_id['scenario'][sce_net['uuid']] = {} - sites = descriptor_net.get("sites", [ {} ]) - for site in sites: - if site.get("datacenter"): - vim = myvims[ site["datacenter"] ] - datacenter_id = site["datacenter"] - myvim_thread_id = myvim_threads_id[ site["datacenter"] ] - else: - vim = myvims[ default_datacenter_id ] - datacenter_id = default_datacenter_id - myvim_thread_id = myvim_threads_id[default_datacenter_id] + if sce_net["external"]: + number_mgmt_networks += 1 + + for datacenter_id in involved_datacenters: + netmap_use = None + netmap_create = None + if descriptor_net.get("sites"): + for site in descriptor_net["sites"]: + if site.get("datacenter") == datacenter_id: + netmap_use = site.get("netmap-use") + netmap_create = site.get("netmap-create") + break + + vim = myvims[datacenter_id] + myvim_thread_id = myvim_threads_id[datacenter_id] + net_type = sce_net['type'] lookfor_filter = {'admin_state_up': True, 'status': 'ACTIVE'} # 'shared': True @@ -2783,59 +3140,64 @@ def create_instance(mydb, tenant_id, instance_dict): if sce_net["external"]: net_name = sce_net["name"] else: - net_name = "{}.{}".format(instance_name, sce_net["name"]) + net_name = "{}-{}".format(instance_name, sce_net["name"]) net_name = net_name[:255] # limit length - if "netmap-use" in site or "netmap-create" in site: + if netmap_use or netmap_create: create_network = False lookfor_network = False - if "netmap-use" in site: + if netmap_use: lookfor_network = True - if utils.check_valid_uuid(site["netmap-use"]): - filter_text = "scenario id '%s'" % site["netmap-use"] - lookfor_filter["id"] = site["netmap-use"] + if utils.check_valid_uuid(netmap_use): + lookfor_filter["id"] = netmap_use else: - filter_text = "scenario name '%s'" % site["netmap-use"] - lookfor_filter["name"] = site["netmap-use"] - if "netmap-create" in site: + lookfor_filter["name"] = netmap_use + if netmap_create: create_network = True net_vim_name = net_name - if site["netmap-create"]: - net_vim_name = site["netmap-create"] + if isinstance(netmap_create, str): + net_vim_name = netmap_create + elif sce_net.get("vim_network_name"): + create_network = False + lookfor_network = True + lookfor_filter["name"] = sce_net.get("vim_network_name") elif sce_net["external"]: - if sce_net['vim_id'] != None: + if sce_net['vim_id'] is not None: # there is a netmap at datacenter_nets database # TODO REVISE!!!! create_network = False lookfor_network = True lookfor_filter["id"] = sce_net['vim_id'] - filter_text = "vim_id '{}' datacenter_netmap name '{}'. Try to reload vims with "\ - "datacenter-net-update".format(sce_net['vim_id'], sce_net["name"]) - # look for network at datacenter and return error + elif vim["config"].get("management_network_id") or vim["config"].get("management_network_name"): + if number_mgmt_networks > 1: + raise NfvoException("Found several VLD of type mgmt. " + "You must concrete what vim-network must be use for each one", + HTTP_Bad_Request) + create_network = False + lookfor_network = True + if vim["config"].get("management_network_id"): + lookfor_filter["id"] = vim["config"]["management_network_id"] + else: + lookfor_filter["name"] = vim["config"]["management_network_name"] else: # There is not a netmap, look at datacenter for a net with this name and create if not found create_network = True lookfor_network = True lookfor_filter["name"] = sce_net["name"] net_vim_name = sce_net["name"] - filter_text = "scenario name '%s'" % sce_net["name"] else: net_vim_name = net_name create_network = True lookfor_network = False - if lookfor_network and create_network: - # TODO create two tasks FIND + CREATE with their relationship - task_action = "FIND" - task_params = (lookfor_filter,) - # task_action = "CREATE" - # task_params = (net_vim_name, net_type, sce_net.get('ip_profile', None)) - # task + task_extra = {} + if create_network: + task_action = "CREATE" + task_extra["params"] = (net_vim_name, net_type, sce_net.get('ip_profile', None)) + if lookfor_network: + task_extra["find"] = (lookfor_filter,) elif lookfor_network: task_action = "FIND" - task_params = (lookfor_filter,) - elif create_network: - task_action = "CREATE" - task_params = (net_vim_name, net_type, sce_net.get('ip_profile', None)) + task_extra["params"] = (lookfor_filter,) # fill database content net_uuid = str(uuid4()) @@ -2849,7 +3211,7 @@ def create_instance(mydb, tenant_id, instance_dict): "created": create_network, 'datacenter_id': datacenter_id, 'datacenter_tenant_id': myvim_thread_id, - 'status': 'BUILD' if create_network else "ACTIVE" + 'status': 'BUILD' # if create_network else "ACTIVE" } db_instance_nets.append(db_net) db_vim_action = { @@ -2860,7 +3222,7 @@ def create_instance(mydb, tenant_id, instance_dict): "action": task_action, "item": "instance_nets", "item_id": net_uuid, - "extra": yaml.safe_dump({"params": task_params}, default_flow_style=True, width=256) + "extra": yaml.safe_dump(task_extra, default_flow_style=True, width=256) } net2task_id['scenario'][sce_net['uuid']][datacenter_id] = task_index task_index += 1 @@ -2879,297 +3241,187 @@ def create_instance(mydb, tenant_id, instance_dict): } db_ip_profiles.append(db_ip_profile) - # 2. Creating new nets (vnf internal nets) in the VIM" - # For each vnf net, we create it and we add it to instanceNetlist. - for sce_vnf in scenarioDict['vnfs']: - for net in sce_vnf['nets']: - if sce_vnf.get("datacenter"): - datacenter_id = sce_vnf["datacenter"] - myvim_thread_id = myvim_threads_id[sce_vnf["datacenter"]] - else: - datacenter_id = default_datacenter_id - myvim_thread_id = myvim_threads_id[default_datacenter_id] - descriptor_net = instance_dict.get("vnfs", {}).get(sce_vnf["name"], {}) - net_name = descriptor_net.get("name") - if not net_name: - net_name = "{}.{}".format(instance_name, net["name"]) - net_name = net_name[:255] # limit length - net_type = net['type'] - - if sce_vnf['uuid'] not in vnf_net2instance: - vnf_net2instance[sce_vnf['uuid']] = {} - if sce_vnf['uuid'] not in net2task_id: - net2task_id[sce_vnf['uuid']] = {} - net2task_id[sce_vnf['uuid']][net['uuid']] = task_index - - # fill database content - net_uuid = str(uuid4()) - uuid_list.append(net_uuid) - vnf_net2instance[sce_vnf['uuid']][net['uuid']] = net_uuid - db_net = { - "uuid": net_uuid, - 'vim_net_id': None, + # Create VNFs + vnf_params = { + "default_datacenter_id": default_datacenter_id, + "myvim_threads_id": myvim_threads_id, + "instance_uuid": instance_uuid, + "instance_name": instance_name, + "instance_action_id": instance_action_id, + "myvims": myvims, + "cloud_config": cloud_config, + "RO_pub_key": tenant[0].get('RO_pub_key'), + } + vnf_params_out = { + "task_index": task_index, + "uuid_list": uuid_list, + "db_instance_nets": db_instance_nets, + "db_vim_actions": db_vim_actions, + "db_ip_profiles": db_ip_profiles, + "db_instance_vnfs": db_instance_vnfs, + "db_instance_vms": db_instance_vms, + "db_instance_interfaces": db_instance_interfaces, + "net2task_id": net2task_id, + "sce_net2instance": sce_net2instance, + } + # sce_vnf_list = sorted(scenarioDict['vnfs'], key=lambda k: k['name']) + for sce_vnf in scenarioDict['vnfs']: # sce_vnf_list: + instantiate_vnf(mydb, sce_vnf, vnf_params, vnf_params_out, rollbackList) + task_index = vnf_params_out["task_index"] + uuid_list = vnf_params_out["uuid_list"] + + # Create VNFFGs + # task_depends_on = [] + for vnffg in scenarioDict['vnffgs']: + for rsp in vnffg['rsps']: + sfs_created = [] + for cp in rsp['connection_points']: + count = mydb.get_rows( + SELECT=('vms.count'), + FROM="vms join interfaces on vms.uuid=interfaces.vm_id join sce_rsp_hops as h on interfaces.uuid=h.interface_id", + WHERE={'h.uuid': cp['uuid']})[0]['count'] + instance_vnf = next((item for item in db_instance_vnfs if item['sce_vnf_id'] == cp['sce_vnf_id']), None) + instance_vms = [item for item in db_instance_vms if item['instance_vnf_id'] == instance_vnf['uuid']] + dependencies = [] + for instance_vm in instance_vms: + action = next((item for item in db_vim_actions if item['item_id'] == instance_vm['uuid']), None) + if action: + dependencies.append(action['task_index']) + # TODO: throw exception if count != len(instance_vms) + # TODO: and action shouldn't ever be None + sfis_created = [] + for i in range(count): + # create sfis + sfi_uuid = str(uuid4()) + uuid_list.append(sfi_uuid) + db_sfi = { + "uuid": sfi_uuid, + "instance_scenario_id": instance_uuid, + 'sce_rsp_hop_id': cp['uuid'], + 'datacenter_id': datacenter_id, + 'datacenter_tenant_id': myvim_thread_id, + "vim_sfi_id": None, # vim thread will populate + } + db_instance_sfis.append(db_sfi) + db_vim_action = { + "instance_action_id": instance_action_id, + "task_index": task_index, + "datacenter_vim_id": myvim_thread_id, + "action": "CREATE", + "status": "SCHEDULED", + "item": "instance_sfis", + "item_id": sfi_uuid, + "extra": yaml.safe_dump({"params": "", "depends_on": [dependencies[i]]}, + default_flow_style=True, width=256) + } + sfis_created.append(task_index) + task_index += 1 + db_vim_actions.append(db_vim_action) + # create sfs + sf_uuid = str(uuid4()) + uuid_list.append(sf_uuid) + db_sf = { + "uuid": sf_uuid, + "instance_scenario_id": instance_uuid, + 'sce_rsp_hop_id': cp['uuid'], + 'datacenter_id': datacenter_id, + 'datacenter_tenant_id': myvim_thread_id, + "vim_sf_id": None, # vim thread will populate + } + db_instance_sfs.append(db_sf) + db_vim_action = { + "instance_action_id": instance_action_id, + "task_index": task_index, + "datacenter_vim_id": myvim_thread_id, + "action": "CREATE", + "status": "SCHEDULED", + "item": "instance_sfs", + "item_id": sf_uuid, + "extra": yaml.safe_dump({"params": "", "depends_on": sfis_created}, + default_flow_style=True, width=256) + } + sfs_created.append(task_index) + task_index += 1 + db_vim_actions.append(db_vim_action) + classifier = rsp['classifier'] + + # TODO the following ~13 lines can be reused for the sfi case + count = mydb.get_rows( + SELECT=('vms.count'), + FROM="vms join interfaces on vms.uuid=interfaces.vm_id join sce_classifiers as c on interfaces.uuid=c.interface_id", + WHERE={'c.uuid': classifier['uuid']})[0]['count'] + instance_vnf = next((item for item in db_instance_vnfs if item['sce_vnf_id'] == classifier['sce_vnf_id']), None) + instance_vms = [item for item in db_instance_vms if item['instance_vnf_id'] == instance_vnf['uuid']] + dependencies = [] + for instance_vm in instance_vms: + action = next((item for item in db_vim_actions if item['item_id'] == instance_vm['uuid']), None) + if action: + dependencies.append(action['task_index']) + # TODO: throw exception if count != len(instance_vms) + # TODO: and action shouldn't ever be None + classifications_created = [] + for i in range(count): + for match in classifier['matches']: + # create classifications + classification_uuid = str(uuid4()) + uuid_list.append(classification_uuid) + db_classification = { + "uuid": classification_uuid, + "instance_scenario_id": instance_uuid, + 'sce_classifier_match_id': match['uuid'], + 'datacenter_id': datacenter_id, + 'datacenter_tenant_id': myvim_thread_id, + "vim_classification_id": None, # vim thread will populate + } + db_instance_classifications.append(db_classification) + classification_params = { + "ip_proto": match["ip_proto"], + "source_ip": match["source_ip"], + "destination_ip": match["destination_ip"], + "source_port": match["source_port"], + "destination_port": match["destination_port"] + } + db_vim_action = { + "instance_action_id": instance_action_id, + "task_index": task_index, + "datacenter_vim_id": myvim_thread_id, + "action": "CREATE", + "status": "SCHEDULED", + "item": "instance_classifications", + "item_id": classification_uuid, + "extra": yaml.safe_dump({"params": classification_params, "depends_on": [dependencies[i]]}, + default_flow_style=True, width=256) + } + classifications_created.append(task_index) + task_index += 1 + db_vim_actions.append(db_vim_action) + + # create sfps + sfp_uuid = str(uuid4()) + uuid_list.append(sfp_uuid) + db_sfp = { + "uuid": sfp_uuid, "instance_scenario_id": instance_uuid, - "net_id": net["uuid"], - "created": True, + 'sce_rsp_id': rsp['uuid'], 'datacenter_id': datacenter_id, 'datacenter_tenant_id': myvim_thread_id, + "vim_sfp_id": None, # vim thread will populate } - db_instance_nets.append(db_net) - + db_instance_sfps.append(db_sfp) db_vim_action = { "instance_action_id": instance_action_id, "task_index": task_index, "datacenter_vim_id": myvim_thread_id, - "status": "SCHEDULED", "action": "CREATE", - "item": "instance_nets", - "item_id": net_uuid, - "extra": yaml.safe_dump({"params": (net_name, net_type, net.get('ip_profile',None))}, + "status": "SCHEDULED", + "item": "instance_sfps", + "item_id": sfp_uuid, + "extra": yaml.safe_dump({"params": "", "depends_on": sfs_created + classifications_created}, default_flow_style=True, width=256) } task_index += 1 db_vim_actions.append(db_vim_action) - if 'ip_profile' in net: - db_ip_profile = { - 'instance_net_id': net_uuid, - 'ip_version': net['ip_profile']['ip_version'], - 'subnet_address': net['ip_profile']['subnet_address'], - 'gateway_address': net['ip_profile']['gateway_address'], - 'dns_address': net['ip_profile']['dns_address'], - 'dhcp_enabled': net['ip_profile']['dhcp_enabled'], - 'dhcp_start_address': net['ip_profile']['dhcp_start_address'], - 'dhcp_count': net['ip_profile']['dhcp_count'], - } - db_ip_profiles.append(db_ip_profile) - - # print "vnf_net2instance:" - # print yaml.safe_dump(vnf_net2instance, indent=4, default_flow_style=False) - - # 3. Creating new vm instances in the VIM - # myvim.new_vminstance(self,vimURI,tenant_id,name,description,image_id,flavor_id,net_dict) - sce_vnf_list = sorted(scenarioDict['vnfs'], key=lambda k: k['name']) - for sce_vnf in sce_vnf_list: - vnf_availability_zones = [] - for vm in sce_vnf['vms']: - vm_av = vm.get('availability_zone') - if vm_av and vm_av not in vnf_availability_zones: - vnf_availability_zones.append(vm_av) - - # check if there is enough availability zones available at vim level. - if myvims[datacenter_id].availability_zone and vnf_availability_zones: - if len(vnf_availability_zones) > len(myvims[datacenter_id].availability_zone): - raise NfvoException('No enough availability zones at VIM for this deployment', HTTP_Bad_Request) - - if sce_vnf.get("datacenter"): - vim = myvims[ sce_vnf["datacenter"] ] - myvim_thread_id = myvim_threads_id[ sce_vnf["datacenter"] ] - datacenter_id = sce_vnf["datacenter"] - else: - vim = myvims[ default_datacenter_id ] - myvim_thread_id = myvim_threads_id[ default_datacenter_id ] - datacenter_id = default_datacenter_id - sce_vnf["datacenter_id"] = datacenter_id - i = 0 - - vnf_uuid = str(uuid4()) - uuid_list.append(vnf_uuid) - db_instance_vnf = { - 'uuid': vnf_uuid, - 'instance_scenario_id': instance_uuid, - 'vnf_id': sce_vnf['vnf_id'], - 'sce_vnf_id': sce_vnf['uuid'], - 'datacenter_id': datacenter_id, - 'datacenter_tenant_id': myvim_thread_id, - } - db_instance_vnfs.append(db_instance_vnf) - - for vm in sce_vnf['vms']: - myVMDict = {} - myVMDict['name'] = "{}.{}.{}".format(instance_name[:64], sce_vnf['name'][:64], vm["name"][:64]) - myVMDict['description'] = myVMDict['name'][0:99] -# if not startvms: -# myVMDict['start'] = "no" - myVMDict['name'] = myVMDict['name'][0:255] # limit name length - #create image at vim in case it not exist - image_dict = mydb.get_table_by_uuid_name("images", vm['image_id']) - image_id = create_or_use_image(mydb, {datacenter_id: vim}, image_dict, [], True) - vm['vim_image_id'] = image_id - - # create flavor at vim in case it not exist - flavor_dict = mydb.get_table_by_uuid_name("flavors", vm['flavor_id']) - if flavor_dict['extended']!=None: - flavor_dict['extended'] = yaml.load(flavor_dict['extended']) - flavor_id = create_or_use_flavor(mydb, {datacenter_id: vim}, flavor_dict, rollbackList, True) - - # Obtain information for additional disks - extended_flavor_dict = mydb.get_rows(FROM='datacenters_flavors', SELECT=('extended',), WHERE={'vim_id': flavor_id}) - if not extended_flavor_dict: - raise NfvoException("flavor '{}' not found".format(flavor_id), HTTP_Not_Found) - return - - # extended_flavor_dict_yaml = yaml.load(extended_flavor_dict[0]) - myVMDict['disks'] = None - extended_info = extended_flavor_dict[0]['extended'] - if extended_info != None: - extended_flavor_dict_yaml = yaml.load(extended_info) - if 'disks' in extended_flavor_dict_yaml: - myVMDict['disks'] = extended_flavor_dict_yaml['disks'] - - vm['vim_flavor_id'] = flavor_id - myVMDict['imageRef'] = vm['vim_image_id'] - myVMDict['flavorRef'] = vm['vim_flavor_id'] - myVMDict['availability_zone'] = vm.get('availability_zone') - myVMDict['networks'] = [] - task_depends_on = [] - # TODO ALF. connect_mgmt_interfaces. Connect management interfaces if this is true - db_vm_ifaces = [] - for iface in vm['interfaces']: - netDict = {} - if iface['type']=="data": - netDict['type'] = iface['model'] - elif "model" in iface and iface["model"]!=None: - netDict['model']=iface['model'] - # TODO in future, remove this because mac_address will not be set, and the type of PV,VF - # is obtained from iterface table model - # discover type of interface looking at flavor - for numa in flavor_dict.get('extended',{}).get('numas',[]): - for flavor_iface in numa.get('interfaces',[]): - if flavor_iface.get('name') == iface['internal_name']: - if flavor_iface['dedicated'] == 'yes': - netDict['type']="PF" #passthrough - elif flavor_iface['dedicated'] == 'no': - netDict['type']="VF" #siov - elif flavor_iface['dedicated'] == 'yes:sriov': - netDict['type']="VFnotShared" #sriov but only one sriov on the PF - netDict["mac_address"] = flavor_iface.get("mac_address") - break; - netDict["use"]=iface['type'] - if netDict["use"]=="data" and not netDict.get("type"): - #print "netDict", netDict - #print "iface", iface - e_text = "Cannot determine the interface type PF or VF of VNF '%s' VM '%s' iface '%s'" %(sce_vnf['name'], vm['name'], iface['internal_name']) - if flavor_dict.get('extended')==None: - raise NfvoException(e_text + "After database migration some information is not available. \ - Try to delete and create the scenarios and VNFs again", HTTP_Conflict) - else: - raise NfvoException(e_text, HTTP_Internal_Server_Error) - if netDict["use"]=="mgmt" or netDict["use"]=="bridge": - netDict["type"]="virtual" - if "vpci" in iface and iface["vpci"] is not None: - netDict['vpci'] = iface['vpci'] - if "mac" in iface and iface["mac"] is not None: - netDict['mac_address'] = iface['mac'] - if "port-security" in iface and iface["port-security"] is not None: - netDict['port_security'] = iface['port-security'] - if "floating-ip" in iface and iface["floating-ip"] is not None: - netDict['floating_ip'] = iface['floating-ip'] - netDict['name'] = iface['internal_name'] - if iface['net_id'] is None: - for vnf_iface in sce_vnf["interfaces"]: - # print iface - # print vnf_iface - if vnf_iface['interface_id']==iface['uuid']: - netDict['net_id'] = "TASK-{}".format(net2task_id['scenario'][ vnf_iface['sce_net_id'] ][datacenter_id]) - instance_net_id = sce_net2instance[ vnf_iface['sce_net_id'] ][datacenter_id] - task_depends_on.append(net2task_id['scenario'][ vnf_iface['sce_net_id'] ][datacenter_id]) - break - else: - netDict['net_id'] = "TASK-{}".format(net2task_id[ sce_vnf['uuid'] ][ iface['net_id'] ]) - instance_net_id = vnf_net2instance[ sce_vnf['uuid'] ][ iface['net_id'] ] - task_depends_on.append(net2task_id[sce_vnf['uuid'] ][ iface['net_id']]) - # skip bridge ifaces not connected to any net - if 'net_id' not in netDict or netDict['net_id']==None: - continue - myVMDict['networks'].append(netDict) - db_vm_iface={ - # "uuid" - # 'instance_vm_id': instance_vm_uuid, - "instance_net_id": instance_net_id, - 'interface_id': iface['uuid'], - # 'vim_interface_id': , - 'type': 'external' if iface['external_name'] is not None else 'internal', - 'ip_address': iface.get('ip_address'), - 'floating_ip': int(iface.get('floating-ip', False)), - 'port_security': int(iface.get('port-security', True)) - } - db_vm_ifaces.append(db_vm_iface) - # print ">>>>>>>>>>>>>>>>>>>>>>>>>>>" - # print myVMDict['name'] - # print "networks", yaml.safe_dump(myVMDict['networks'], indent=4, default_flow_style=False) - # print "interfaces", yaml.safe_dump(vm['interfaces'], indent=4, default_flow_style=False) - # print ">>>>>>>>>>>>>>>>>>>>>>>>>>>" - if vm.get("boot_data"): - cloud_config_vm = unify_cloud_config(vm["boot_data"], cloud_config) - else: - cloud_config_vm = cloud_config - if myVMDict.get('availability_zone'): - av_index = vnf_availability_zones.index(myVMDict['availability_zone']) - else: - av_index = None - for vm_index in range(0, vm.get('count', 1)): - vm_index_name = "" - if vm.get('count', 1) > 1: - vm_index_name += "." + chr(97 + vm_index) - task_params = (myVMDict['name']+vm_index_name, myVMDict['description'], myVMDict.get('start', None), - myVMDict['imageRef'], myVMDict['flavorRef'], myVMDict['networks'], cloud_config_vm, - myVMDict['disks'], av_index, vnf_availability_zones) - # put interface uuid back to scenario[vnfs][vms[[interfaces] - for net in myVMDict['networks']: - if "vim_id" in net: - for iface in vm['interfaces']: - if net["name"]==iface["internal_name"]: - iface["vim_id"]=net["vim_id"] - break - vm_uuid = str(uuid4()) - uuid_list.append(vm_uuid) - db_vm = { - "uuid": vm_uuid, - 'instance_vnf_id': vnf_uuid, - #TODO delete "vim_vm_id": vm_id, - "vm_id": vm["uuid"], - # "status": - } - db_instance_vms.append(db_vm) - - iface_index = 0 - for db_vm_iface in db_vm_ifaces: - iface_uuid = str(uuid4()) - uuid_list.append(iface_uuid) - db_vm_iface_instance = { - "uuid": iface_uuid, - "instance_vm_id": vm_uuid - } - db_vm_iface_instance.update(db_vm_iface) - if db_vm_iface_instance.get("ip_address"): # increment ip_address - ip = db_vm_iface_instance.get("ip_address") - i = ip.rfind(".") - if i > 0: - try: - i += 1 - ip = ip[i:] + str(int(ip[:i]) +1) - db_vm_iface_instance["ip_address"] = ip - except: - db_vm_iface_instance["ip_address"] = None - db_instance_interfaces.append(db_vm_iface_instance) - myVMDict['networks'][iface_index]["uuid"] = iface_uuid - iface_index += 1 - - db_vim_action = { - "instance_action_id": instance_action_id, - "task_index": task_index, - "datacenter_vim_id": myvim_thread_id, - "action": "CREATE", - "status": "SCHEDULED", - "item": "instance_vms", - "item_id": vm_uuid, - "extra": yaml.safe_dump({"params": task_params, "depends_on": task_depends_on}, - default_flow_style=True, width=256) - } - task_index += 1 - db_vim_actions.append(db_vim_action) - scenarioDict["datacenter2tenant"] = myvim_threads_id db_instance_action["number_tasks"] = task_index @@ -3183,6 +3435,10 @@ def create_instance(mydb, tenant_id, instance_dict): {"instance_vms": db_instance_vms}, {"instance_interfaces": db_instance_interfaces}, {"instance_actions": db_instance_action}, + {"instance_sfis": db_instance_sfis}, + {"instance_sfs": db_instance_sfs}, + {"instance_classifications": db_instance_classifications}, + {"instance_sfps": db_instance_sfps}, {"vim_actions": db_vim_actions} ] @@ -3208,13 +3464,360 @@ def create_instance(mydb, tenant_id, instance_dict): raise NfvoException(error_text, e.http_code) +def instantiate_vnf(mydb, sce_vnf, params, params_out, rollbackList): + default_datacenter_id = params["default_datacenter_id"] + myvim_threads_id = params["myvim_threads_id"] + instance_uuid = params["instance_uuid"] + instance_name = params["instance_name"] + instance_action_id = params["instance_action_id"] + myvims = params["myvims"] + cloud_config = params["cloud_config"] + RO_pub_key = params["RO_pub_key"] + + task_index = params_out["task_index"] + uuid_list = params_out["uuid_list"] + db_instance_nets = params_out["db_instance_nets"] + db_vim_actions = params_out["db_vim_actions"] + db_ip_profiles = params_out["db_ip_profiles"] + db_instance_vnfs = params_out["db_instance_vnfs"] + db_instance_vms = params_out["db_instance_vms"] + db_instance_interfaces = params_out["db_instance_interfaces"] + net2task_id = params_out["net2task_id"] + sce_net2instance = params_out["sce_net2instance"] + + vnf_net2instance = {} + + # 2. Creating new nets (vnf internal nets) in the VIM" + # For each vnf net, we create it and we add it to instanceNetlist. + if sce_vnf.get("datacenter"): + datacenter_id = sce_vnf["datacenter"] + myvim_thread_id = myvim_threads_id[sce_vnf["datacenter"]] + else: + datacenter_id = default_datacenter_id + myvim_thread_id = myvim_threads_id[default_datacenter_id] + for net in sce_vnf['nets']: + # TODO revis + # descriptor_net = instance_dict.get("vnfs", {}).get(sce_vnf["name"], {}) + # net_name = descriptor_net.get("name") + net_name = None + if not net_name: + net_name = "{}-{}".format(instance_name, net["name"]) + net_name = net_name[:255] # limit length + net_type = net['type'] + + if sce_vnf['uuid'] not in vnf_net2instance: + vnf_net2instance[sce_vnf['uuid']] = {} + if sce_vnf['uuid'] not in net2task_id: + net2task_id[sce_vnf['uuid']] = {} + net2task_id[sce_vnf['uuid']][net['uuid']] = task_index + + # fill database content + net_uuid = str(uuid4()) + uuid_list.append(net_uuid) + vnf_net2instance[sce_vnf['uuid']][net['uuid']] = net_uuid + db_net = { + "uuid": net_uuid, + 'vim_net_id': None, + "instance_scenario_id": instance_uuid, + "net_id": net["uuid"], + "created": True, + 'datacenter_id': datacenter_id, + 'datacenter_tenant_id': myvim_thread_id, + } + db_instance_nets.append(db_net) + + if net.get("vim-network-name"): + lookfor_filter = {"name": net["vim-network-name"]} + task_action = "FIND" + task_extra = {"params": (lookfor_filter,)} + else: + task_action = "CREATE" + task_extra = {"params": (net_name, net_type, net.get('ip_profile', None))} + + db_vim_action = { + "instance_action_id": instance_action_id, + "task_index": task_index, + "datacenter_vim_id": myvim_thread_id, + "status": "SCHEDULED", + "action": task_action, + "item": "instance_nets", + "item_id": net_uuid, + "extra": yaml.safe_dump(task_extra, default_flow_style=True, width=256) + } + task_index += 1 + db_vim_actions.append(db_vim_action) + + if 'ip_profile' in net: + db_ip_profile = { + 'instance_net_id': net_uuid, + 'ip_version': net['ip_profile']['ip_version'], + 'subnet_address': net['ip_profile']['subnet_address'], + 'gateway_address': net['ip_profile']['gateway_address'], + 'dns_address': net['ip_profile']['dns_address'], + 'dhcp_enabled': net['ip_profile']['dhcp_enabled'], + 'dhcp_start_address': net['ip_profile']['dhcp_start_address'], + 'dhcp_count': net['ip_profile']['dhcp_count'], + } + db_ip_profiles.append(db_ip_profile) + + # print "vnf_net2instance:" + # print yaml.safe_dump(vnf_net2instance, indent=4, default_flow_style=False) + + # 3. Creating new vm instances in the VIM + # myvim.new_vminstance(self,vimURI,tenant_id,name,description,image_id,flavor_id,net_dict) + ssh_access = None + if sce_vnf.get('mgmt_access'): + ssh_access = sce_vnf['mgmt_access'].get('config-access', {}).get('ssh-access') + vnf_availability_zones = [] + for vm in sce_vnf['vms']: + vm_av = vm.get('availability_zone') + if vm_av and vm_av not in vnf_availability_zones: + vnf_availability_zones.append(vm_av) + + # check if there is enough availability zones available at vim level. + if myvims[datacenter_id].availability_zone and vnf_availability_zones: + if len(vnf_availability_zones) > len(myvims[datacenter_id].availability_zone): + raise NfvoException('No enough availability zones at VIM for this deployment', HTTP_Bad_Request) + + if sce_vnf.get("datacenter"): + vim = myvims[sce_vnf["datacenter"]] + myvim_thread_id = myvim_threads_id[sce_vnf["datacenter"]] + datacenter_id = sce_vnf["datacenter"] + else: + vim = myvims[default_datacenter_id] + myvim_thread_id = myvim_threads_id[default_datacenter_id] + datacenter_id = default_datacenter_id + sce_vnf["datacenter_id"] = datacenter_id + i = 0 + + vnf_uuid = str(uuid4()) + uuid_list.append(vnf_uuid) + db_instance_vnf = { + 'uuid': vnf_uuid, + 'instance_scenario_id': instance_uuid, + 'vnf_id': sce_vnf['vnf_id'], + 'sce_vnf_id': sce_vnf['uuid'], + 'datacenter_id': datacenter_id, + 'datacenter_tenant_id': myvim_thread_id, + } + db_instance_vnfs.append(db_instance_vnf) + + for vm in sce_vnf['vms']: + myVMDict = {} + sce_vnf_name = sce_vnf['member_vnf_index'] if sce_vnf['member_vnf_index'] else sce_vnf['name'] + myVMDict['name'] = "{}-{}-{}".format(instance_name[:64], sce_vnf_name[:64], vm["name"][:64]) + myVMDict['description'] = myVMDict['name'][0:99] + # if not startvms: + # myVMDict['start'] = "no" + if vm.get("instance_parameters") and vm["instance_parameters"].get("name"): + myVMDict['name'] = vm["instance_parameters"].get("name") + myVMDict['name'] = myVMDict['name'][0:255] # limit name length + # create image at vim in case it not exist + image_uuid = vm['image_id'] + if vm.get("image_list"): + for alternative_image in vm["image_list"]: + if alternative_image["vim_type"] == vim["config"]["_vim_type_internal"]: + image_uuid = alternative_image['image_id'] + break + image_dict = mydb.get_table_by_uuid_name("images", image_uuid) + image_id = create_or_use_image(mydb, {datacenter_id: vim}, image_dict, [], True) + vm['vim_image_id'] = image_id + + # create flavor at vim in case it not exist + flavor_dict = mydb.get_table_by_uuid_name("flavors", vm['flavor_id']) + if flavor_dict['extended'] != None: + flavor_dict['extended'] = yaml.load(flavor_dict['extended']) + flavor_id = create_or_use_flavor(mydb, {datacenter_id: vim}, flavor_dict, rollbackList, True) + + # Obtain information for additional disks + extended_flavor_dict = mydb.get_rows(FROM='datacenters_flavors', SELECT=('extended',), + WHERE={'vim_id': flavor_id}) + if not extended_flavor_dict: + raise NfvoException("flavor '{}' not found".format(flavor_id), HTTP_Not_Found) + + # extended_flavor_dict_yaml = yaml.load(extended_flavor_dict[0]) + myVMDict['disks'] = None + extended_info = extended_flavor_dict[0]['extended'] + if extended_info != None: + extended_flavor_dict_yaml = yaml.load(extended_info) + if 'disks' in extended_flavor_dict_yaml: + myVMDict['disks'] = extended_flavor_dict_yaml['disks'] + if vm.get("instance_parameters") and vm["instance_parameters"].get("devices"): + for disk in myVMDict['disks']: + if disk.get("name") in vm["instance_parameters"]["devices"]: + disk.update(vm["instance_parameters"]["devices"][disk.get("name")]) + + vm['vim_flavor_id'] = flavor_id + myVMDict['imageRef'] = vm['vim_image_id'] + myVMDict['flavorRef'] = vm['vim_flavor_id'] + myVMDict['availability_zone'] = vm.get('availability_zone') + myVMDict['networks'] = [] + task_depends_on = [] + # TODO ALF. connect_mgmt_interfaces. Connect management interfaces if this is true + db_vm_ifaces = [] + for iface in vm['interfaces']: + netDict = {} + if iface['type'] == "data": + netDict['type'] = iface['model'] + elif "model" in iface and iface["model"] != None: + netDict['model'] = iface['model'] + # TODO in future, remove this because mac_address will not be set, and the type of PV,VF + # is obtained from iterface table model + # discover type of interface looking at flavor + for numa in flavor_dict.get('extended', {}).get('numas', []): + for flavor_iface in numa.get('interfaces', []): + if flavor_iface.get('name') == iface['internal_name']: + if flavor_iface['dedicated'] == 'yes': + netDict['type'] = "PF" # passthrough + elif flavor_iface['dedicated'] == 'no': + netDict['type'] = "VF" # siov + elif flavor_iface['dedicated'] == 'yes:sriov': + netDict['type'] = "VFnotShared" # sriov but only one sriov on the PF + netDict["mac_address"] = flavor_iface.get("mac_address") + break + netDict["use"] = iface['type'] + if netDict["use"] == "data" and not netDict.get("type"): + # print "netDict", netDict + # print "iface", iface + e_text = "Cannot determine the interface type PF or VF of VNF '{}' VM '{}' iface '{}'".fromat( + sce_vnf['name'], vm['name'], iface['internal_name']) + if flavor_dict.get('extended') == None: + raise NfvoException(e_text + "After database migration some information is not available. \ + Try to delete and create the scenarios and VNFs again", HTTP_Conflict) + else: + raise NfvoException(e_text, HTTP_Internal_Server_Error) + if netDict["use"] == "mgmt" or netDict["use"] == "bridge": + netDict["type"] = "virtual" + if iface.get("vpci"): + netDict['vpci'] = iface['vpci'] + if iface.get("mac"): + netDict['mac_address'] = iface['mac'] + if iface.get("ip_address"): + netDict['ip_address'] = iface['ip_address'] + if iface.get("port-security") is not None: + netDict['port_security'] = iface['port-security'] + if iface.get("floating-ip") is not None: + netDict['floating_ip'] = iface['floating-ip'] + netDict['name'] = iface['internal_name'] + if iface['net_id'] is None: + for vnf_iface in sce_vnf["interfaces"]: + # print iface + # print vnf_iface + if vnf_iface['interface_id'] == iface['uuid']: + netDict['net_id'] = "TASK-{}".format( + net2task_id['scenario'][vnf_iface['sce_net_id']][datacenter_id]) + instance_net_id = sce_net2instance[vnf_iface['sce_net_id']][datacenter_id] + task_depends_on.append(net2task_id['scenario'][vnf_iface['sce_net_id']][datacenter_id]) + break + else: + netDict['net_id'] = "TASK-{}".format(net2task_id[sce_vnf['uuid']][iface['net_id']]) + instance_net_id = vnf_net2instance[sce_vnf['uuid']][iface['net_id']] + task_depends_on.append(net2task_id[sce_vnf['uuid']][iface['net_id']]) + # skip bridge ifaces not connected to any net + if 'net_id' not in netDict or netDict['net_id'] == None: + continue + myVMDict['networks'].append(netDict) + db_vm_iface = { + # "uuid" + # 'instance_vm_id': instance_vm_uuid, + "instance_net_id": instance_net_id, + 'interface_id': iface['uuid'], + # 'vim_interface_id': , + 'type': 'external' if iface['external_name'] is not None else 'internal', + 'ip_address': iface.get('ip_address'), + 'mac_address': iface.get('mac'), + 'floating_ip': int(iface.get('floating-ip', False)), + 'port_security': int(iface.get('port-security', True)) + } + db_vm_ifaces.append(db_vm_iface) + # print ">>>>>>>>>>>>>>>>>>>>>>>>>>>" + # print myVMDict['name'] + # print "networks", yaml.safe_dump(myVMDict['networks'], indent=4, default_flow_style=False) + # print "interfaces", yaml.safe_dump(vm['interfaces'], indent=4, default_flow_style=False) + # print ">>>>>>>>>>>>>>>>>>>>>>>>>>>" + + # We add the RO key to cloud_config if vnf will need ssh access + cloud_config_vm = cloud_config + if ssh_access and ssh_access['required'] and ssh_access['default-user'] and tenant[0].get('RO_pub_key'): + RO_key = {"key-pairs": [tenant[0]['RO_pub_key']]} + cloud_config_vm = unify_cloud_config(cloud_config_vm, RO_key) + if vm.get("boot_data"): + cloud_config_vm = unify_cloud_config(vm["boot_data"], cloud_config_vm) + + if myVMDict.get('availability_zone'): + av_index = vnf_availability_zones.index(myVMDict['availability_zone']) + else: + av_index = None + for vm_index in range(0, vm.get('count', 1)): + vm_name = myVMDict['name'] + "-" + str(vm_index+1) + task_params = (vm_name, myVMDict['description'], myVMDict.get('start', None), + myVMDict['imageRef'], myVMDict['flavorRef'], myVMDict['networks'], cloud_config_vm, + myVMDict['disks'], av_index, vnf_availability_zones) + # put interface uuid back to scenario[vnfs][vms[[interfaces] + for net in myVMDict['networks']: + if "vim_id" in net: + for iface in vm['interfaces']: + if net["name"] == iface["internal_name"]: + iface["vim_id"] = net["vim_id"] + break + vm_uuid = str(uuid4()) + uuid_list.append(vm_uuid) + db_vm = { + "uuid": vm_uuid, + 'instance_vnf_id': vnf_uuid, + # TODO delete "vim_vm_id": vm_id, + "vm_id": vm["uuid"], + "vim_name": vm_name, + # "status": + } + db_instance_vms.append(db_vm) + + iface_index = 0 + for db_vm_iface in db_vm_ifaces: + iface_uuid = str(uuid4()) + uuid_list.append(iface_uuid) + db_vm_iface_instance = { + "uuid": iface_uuid, + "instance_vm_id": vm_uuid + } + db_vm_iface_instance.update(db_vm_iface) + if db_vm_iface_instance.get("ip_address"): # increment ip_address + ip = db_vm_iface_instance.get("ip_address") + i = ip.rfind(".") + if i > 0: + try: + i += 1 + ip = ip[i:] + str(int(ip[:i]) + 1) + db_vm_iface_instance["ip_address"] = ip + except: + db_vm_iface_instance["ip_address"] = None + db_instance_interfaces.append(db_vm_iface_instance) + myVMDict['networks'][iface_index]["uuid"] = iface_uuid + iface_index += 1 + + db_vim_action = { + "instance_action_id": instance_action_id, + "task_index": task_index, + "datacenter_vim_id": myvim_thread_id, + "action": "CREATE", + "status": "SCHEDULED", + "item": "instance_vms", + "item_id": vm_uuid, + "extra": yaml.safe_dump({"params": task_params, "depends_on": task_depends_on}, + default_flow_style=True, width=256) + } + task_index += 1 + db_vim_actions.append(db_vim_action) + params_out["task_index"] = task_index + params_out["uuid_list"] = uuid_list + + def delete_instance(mydb, tenant_id, instance_id): # print "Checking that the instance_id exists and getting the instance dictionary" instanceDict = mydb.get_instance_scenario(instance_id, tenant_id) # print yaml.safe_dump(instanceDict, indent=4, default_flow_style=False) tenant_id = instanceDict["tenant_id"] # print "Checking that nfvo_tenant_id exists and getting the VIM URI and the VIM tenant_id" - # 1. Delete from Database message = mydb.delete_instance_scenario(instance_id, tenant_id) @@ -3223,6 +3826,7 @@ def delete_instance(mydb, tenant_id, instance_id): myvims = {} myvim_threads = {} vimthread_affected = {} + net2vm_dependencies = {} task_index = 0 instance_action_id = get_task_id() @@ -3261,36 +3865,29 @@ def delete_instance(mydb, tenant_id, instance_id): if not myvim: error_msg += "\n VM id={} cannot be deleted because datacenter={} not found".format(vm['vim_vm_id'], sce_vnf["datacenter_id"]) continue - try: - db_vim_action = { - "instance_action_id": instance_action_id, - "task_index": task_index, - "datacenter_vim_id": sce_vnf["datacenter_tenant_id"], - "action": "DELETE", - "status": "SCHEDULED", - "item": "instance_vms", - "item_id": vm["uuid"], - "extra": yaml.safe_dump({"params": vm["interfaces"]}, - default_flow_style=True, width=256) - } - task_index += 1 - db_vim_actions.append(db_vim_action) - - except vimconn.vimconnNotFoundException as e: - error_msg+="\n VM VIM_id={} not found at datacenter={}".format(vm['vim_vm_id'], sce_vnf["datacenter_id"]) - logger.warn("VM instance '%s'uuid '%s', VIM id '%s', from VNF_id '%s' not found", - vm['name'], vm['uuid'], vm['vim_vm_id'], sce_vnf['vnf_id']) - except vimconn.vimconnException as e: - error_msg+="\n VM VIM_id={} at datacenter={} Error: {} {}".format(vm['vim_vm_id'], sce_vnf["datacenter_id"], e.http_code, str(e)) - logger.error("Error %d deleting VM instance '%s'uuid '%s', VIM_id '%s', from VNF_id '%s': %s", - e.http_code, vm['name'], vm['uuid'], vm['vim_vm_id'], sce_vnf['vnf_id'], str(e)) + db_vim_action = { + "instance_action_id": instance_action_id, + "task_index": task_index, + "datacenter_vim_id": sce_vnf["datacenter_tenant_id"], + "action": "DELETE", + "status": "SCHEDULED", + "item": "instance_vms", + "item_id": vm["uuid"], + "extra": yaml.safe_dump({"params": vm["interfaces"]}, + default_flow_style=True, width=256) + } + db_vim_actions.append(db_vim_action) + for interface in vm["interfaces"]: + if not interface.get("instance_net_id"): + continue + if interface["instance_net_id"] not in net2vm_dependencies: + net2vm_dependencies[interface["instance_net_id"]] = [] + net2vm_dependencies[interface["instance_net_id"]].append(task_index) + task_index += 1 # 2.2 deleting NETS # net_fail_list=[] for net in instanceDict['nets']: - # TODO if not net['created']: - # TODO continue #skip not created nets - vimthread_affected[net["datacenter_tenant_id"]] = None datacenter_key = (net["datacenter_id"], net["datacenter_tenant_id"]) if datacenter_key not in myvims: @@ -3313,31 +3910,171 @@ def delete_instance(mydb, tenant_id, instance_id): if not myvim: error_msg += "\n Net VIM_id={} cannot be deleted because datacenter={} not found".format(net['vim_net_id'], net["datacenter_id"]) continue - try: - db_vim_action = { - "instance_action_id": instance_action_id, - "task_index": task_index, - "datacenter_vim_id": net["datacenter_tenant_id"], - "action": "DELETE", - "status": "SCHEDULED", - "item": "instance_nets", - "item_id": net["uuid"], - "extra": yaml.safe_dump({"params": (net['vim_net_id'], net['sdn_net_id'])}, - default_flow_style=True, width=256) - } - task_index += 1 - db_vim_actions.append(db_vim_action) + extra = {"params": (net['vim_net_id'], net['sdn_net_id'])} + if net2vm_dependencies.get(net["uuid"]): + extra["depends_on"] = net2vm_dependencies[net["uuid"]] + db_vim_action = { + "instance_action_id": instance_action_id, + "task_index": task_index, + "datacenter_vim_id": net["datacenter_tenant_id"], + "action": "DELETE", + "status": "SCHEDULED", + "item": "instance_nets", + "item_id": net["uuid"], + "extra": yaml.safe_dump(extra, default_flow_style=True, width=256) + } + task_index += 1 + db_vim_actions.append(db_vim_action) - except vimconn.vimconnNotFoundException as e: - error_msg += "\n NET VIM_id={} not found at datacenter={}".format(net['vim_net_id'], net["datacenter_id"]) - logger.warn("NET '%s', VIM_id '%s', from VNF_net_id '%s' not found", - net['uuid'], net['vim_net_id'], str(net['vnf_net_id'])) - except vimconn.vimconnException as e: - error_msg += "\n NET VIM_id={} at datacenter={} Error: {} {}".format(net['vim_net_id'], - net["datacenter_id"], - e.http_code, str(e)) - logger.error("Error %d deleting NET '%s', VIM_id '%s', from VNF_net_id '%s': %s", - e.http_code, net['uuid'], net['vim_net_id'], str(net['vnf_net_id']), str(e)) + # 2.3 deleting VNFFGs + + for sfp in instanceDict.get('sfps', ()): + vimthread_affected[sfp["datacenter_tenant_id"]] = None + datacenter_key = (sfp["datacenter_id"], sfp["datacenter_tenant_id"]) + if datacenter_key not in myvims: + try: + _,myvim_thread = get_vim_thread(mydb, tenant_id, sfp["datacenter_id"], sfp["datacenter_tenant_id"]) + except NfvoException as e: + logger.error(str(e)) + myvim_thread = None + myvim_threads[datacenter_key] = myvim_thread + vims = get_vim(mydb, tenant_id, datacenter_id=sfp["datacenter_id"], + datacenter_tenant_id=sfp["datacenter_tenant_id"]) + if len(vims) == 0: + logger.error("datacenter '{}' with datacenter_tenant_id '{}' not found".format(sfp["datacenter_id"], sfp["datacenter_tenant_id"])) + myvims[datacenter_key] = None + else: + myvims[datacenter_key] = vims.values()[0] + myvim = myvims[datacenter_key] + myvim_thread = myvim_threads[datacenter_key] + + if not myvim: + error_msg += "\n vim_sfp_id={} cannot be deleted because datacenter={} not found".format(sfp['vim_sfp_id'], sfp["datacenter_id"]) + continue + extra = {"params": (sfp['vim_sfp_id'])} + db_vim_action = { + "instance_action_id": instance_action_id, + "task_index": task_index, + "datacenter_vim_id": sfp["datacenter_tenant_id"], + "action": "DELETE", + "status": "SCHEDULED", + "item": "instance_sfps", + "item_id": sfp["uuid"], + "extra": yaml.safe_dump(extra, default_flow_style=True, width=256) + } + task_index += 1 + db_vim_actions.append(db_vim_action) + + for sf in instanceDict.get('sfs', ()): + vimthread_affected[sf["datacenter_tenant_id"]] = None + datacenter_key = (sf["datacenter_id"], sf["datacenter_tenant_id"]) + if datacenter_key not in myvims: + try: + _,myvim_thread = get_vim_thread(mydb, tenant_id, sf["datacenter_id"], sf["datacenter_tenant_id"]) + except NfvoException as e: + logger.error(str(e)) + myvim_thread = None + myvim_threads[datacenter_key] = myvim_thread + vims = get_vim(mydb, tenant_id, datacenter_id=sf["datacenter_id"], + datacenter_tenant_id=sf["datacenter_tenant_id"]) + if len(vims) == 0: + logger.error("datacenter '{}' with datacenter_tenant_id '{}' not found".format(sf["datacenter_id"], sf["datacenter_tenant_id"])) + myvims[datacenter_key] = None + else: + myvims[datacenter_key] = vims.values()[0] + myvim = myvims[datacenter_key] + myvim_thread = myvim_threads[datacenter_key] + + if not myvim: + error_msg += "\n vim_sf_id={} cannot be deleted because datacenter={} not found".format(sf['vim_sf_id'], sf["datacenter_id"]) + continue + extra = {"params": (sf['vim_sf_id'])} + db_vim_action = { + "instance_action_id": instance_action_id, + "task_index": task_index, + "datacenter_vim_id": sf["datacenter_tenant_id"], + "action": "DELETE", + "status": "SCHEDULED", + "item": "instance_sfs", + "item_id": sf["uuid"], + "extra": yaml.safe_dump(extra, default_flow_style=True, width=256) + } + task_index += 1 + db_vim_actions.append(db_vim_action) + + for sfi in instanceDict.get('sfis', ()): + vimthread_affected[sfi["datacenter_tenant_id"]] = None + datacenter_key = (sfi["datacenter_id"], sfi["datacenter_tenant_id"]) + if datacenter_key not in myvims: + try: + _,myvim_thread = get_vim_thread(mydb, tenant_id, sfi["datacenter_id"], sfi["datacenter_tenant_id"]) + except NfvoException as e: + logger.error(str(e)) + myvim_thread = None + myvim_threads[datacenter_key] = myvim_thread + vims = get_vim(mydb, tenant_id, datacenter_id=sfi["datacenter_id"], + datacenter_tenant_id=sfi["datacenter_tenant_id"]) + if len(vims) == 0: + logger.error("datacenter '{}' with datacenter_tenant_id '{}' not found".format(sfi["datacenter_id"], sfi["datacenter_tenant_id"])) + myvims[datacenter_key] = None + else: + myvims[datacenter_key] = vims.values()[0] + myvim = myvims[datacenter_key] + myvim_thread = myvim_threads[datacenter_key] + + if not myvim: + error_msg += "\n vim_sfi_id={} cannot be deleted because datacenter={} not found".format(sfi['vim_sfi_id'], sfi["datacenter_id"]) + continue + extra = {"params": (sfi['vim_sfi_id'])} + db_vim_action = { + "instance_action_id": instance_action_id, + "task_index": task_index, + "datacenter_vim_id": sfi["datacenter_tenant_id"], + "action": "DELETE", + "status": "SCHEDULED", + "item": "instance_sfis", + "item_id": sfi["uuid"], + "extra": yaml.safe_dump(extra, default_flow_style=True, width=256) + } + task_index += 1 + db_vim_actions.append(db_vim_action) + + for classification in instanceDict['classifications']: + vimthread_affected[classification["datacenter_tenant_id"]] = None + datacenter_key = (classification["datacenter_id"], classification["datacenter_tenant_id"]) + if datacenter_key not in myvims: + try: + _,myvim_thread = get_vim_thread(mydb, tenant_id, classification["datacenter_id"], classification["datacenter_tenant_id"]) + except NfvoException as e: + logger.error(str(e)) + myvim_thread = None + myvim_threads[datacenter_key] = myvim_thread + vims = get_vim(mydb, tenant_id, datacenter_id=classification["datacenter_id"], + datacenter_tenant_id=classification["datacenter_tenant_id"]) + if len(vims) == 0: + logger.error("datacenter '{}' with datacenter_tenant_id '{}' not found".format(classification["datacenter_id"], classification["datacenter_tenant_id"])) + myvims[datacenter_key] = None + else: + myvims[datacenter_key] = vims.values()[0] + myvim = myvims[datacenter_key] + myvim_thread = myvim_threads[datacenter_key] + + if not myvim: + error_msg += "\n vim_classification_id={} cannot be deleted because datacenter={} not found".format(classification['vim_classification_id'], classification["datacenter_id"]) + continue + extra = {"params": (classification['vim_classification_id'])} + db_vim_action = { + "instance_action_id": instance_action_id, + "task_index": task_index, + "datacenter_vim_id": classification["datacenter_tenant_id"], + "action": "DELETE", + "status": "SCHEDULED", + "item": "instance_classifications", + "item_id": classification["uuid"], + "extra": yaml.safe_dump(extra, default_flow_style=True, width=256) + } + task_index += 1 + db_vim_actions.append(db_vim_action) db_instance_action["number_tasks"] = task_index db_tables = [ @@ -3357,6 +4094,26 @@ def delete_instance(mydb, tenant_id, instance_id): else: return "action_id={} instance {} deleted".format(instance_action_id, message) +def get_instance_id(mydb, tenant_id, instance_id): + global ovim + #check valid tenant_id + check_tenant(mydb, tenant_id) + #obtain data + + instance_dict = mydb.get_instance_scenario(instance_id, tenant_id, verbose=True) + for net in instance_dict["nets"]: + if net.get("sdn_net_id"): + net_sdn = ovim.show_network(net["sdn_net_id"]) + net["sdn_info"] = { + "admin_state_up": net_sdn.get("admin_state_up"), + "flows": net_sdn.get("flows"), + "last_error": net_sdn.get("last_error"), + "ports": net_sdn.get("ports"), + "type": net_sdn.get("type"), + "status": net_sdn.get("status"), + "vlan": net_sdn.get("vlan"), + } + return instance_dict def refresh_instance(mydb, nfvo_tenant, instanceDict, datacenter=None, vim_tenant=None): '''Refreshes a scenario instance. It modifies instanceDict''' @@ -3532,27 +4289,202 @@ def instance_action(mydb,nfvo_tenant,instance_id, action_dict): if len(vims) == 0: raise NfvoException("datacenter '{}' not found".format(str(instanceDict['datacenter_id'])), HTTP_Not_Found) myvim = vims.values()[0] + vm_result = {} + vm_error = 0 + vm_ok = 0 - if action_dict.get("create-vdu"): - for vdu in action_dict["create-vdu"]: + myvim_threads_id = {} + if action_dict.get("vdu-scaling"): + db_instance_vms = [] + db_vim_actions = [] + db_instance_interfaces = [] + instance_action_id = get_task_id() + db_instance_action = { + "uuid": instance_action_id, # same uuid for the instance and the action on create + "tenant_id": nfvo_tenant, + "instance_id": instance_id, + "description": "SCALE", + } + vm_result["instance_action_id"] = instance_action_id + task_index = 0 + for vdu in action_dict["vdu-scaling"]: vdu_id = vdu.get("vdu-id") + osm_vdu_id = vdu.get("osm_vdu_id") + member_vnf_index = vdu.get("member-vnf-index") vdu_count = vdu.get("count", 1) - # get from database TODO - # insert tasks TODO - pass + if vdu_id: + target_vm = mydb.get_rows( + FROM="instance_vms as vms join instance_vnfs as vnfs on vms.instance_vnf_id=vnfs.uuid", + WHERE={"vms.uuid": vdu_id}, + ORDER_BY="vms.created_at" + ) + if not target_vm: + raise NfvoException("Cannot find the vdu with id {}".format(vdu_id), HTTP_Not_Found) + else: + if not osm_vdu_id and not member_vnf_index: + raise NfvoException("Invalid imput vdu parameters. Must supply either 'vdu-id' of 'osm_vdu_id','member-vnf-index'") + target_vm = mydb.get_rows( + # SELECT=("ivms.uuid", "ivnfs.datacenter_id", "ivnfs.datacenter_tenant_id"), + FROM="instance_vms as ivms join instance_vnfs as ivnfs on ivms.instance_vnf_id=ivnfs.uuid"\ + " join sce_vnfs as svnfs on ivnfs.sce_vnf_id=svnfs.uuid"\ + " join vms on ivms.vm_id=vms.uuid", + WHERE={"vms.osm_id": osm_vdu_id, "svnfs.member_vnf_index": member_vnf_index}, + ORDER_BY="ivms.created_at" + ) + if not target_vm: + raise NfvoException("Cannot find the vdu with osm_vdu_id {} and member-vnf-index {}".format(osm_vdu_id, member_vnf_index), HTTP_Not_Found) + vdu_id = target_vm[-1]["uuid"] + vm_result[vdu_id] = {"created": [], "deleted": [], "description": "scheduled"} + target_vm = target_vm[-1] + datacenter = target_vm["datacenter_id"] + myvim_threads_id[datacenter], _ = get_vim_thread(mydb, nfvo_tenant, datacenter) + if vdu["type"] == "delete": + # look for nm + vm_interfaces = None + for sce_vnf in instanceDict['vnfs']: + for vm in sce_vnf['vms']: + if vm["uuid"] == vdu_id: + vm_interfaces = vm["interfaces"] + break + + db_vim_action = { + "instance_action_id": instance_action_id, + "task_index": task_index, + "datacenter_vim_id": target_vm["datacenter_tenant_id"], + "action": "DELETE", + "status": "SCHEDULED", + "item": "instance_vms", + "item_id": target_vm["uuid"], + "extra": yaml.safe_dump({"params": vm_interfaces}, + default_flow_style=True, width=256) + } + task_index += 1 + db_vim_actions.append(db_vim_action) + vm_result[vdu_id]["deleted"].append(vdu_id) + # delete from database + db_instance_vms.append({"TO-DELETE": vdu_id}) + + else: # vdu["type"] == "create": + iface2iface = {} + where = {"item": "instance_vms", "item_id": target_vm["uuid"], "action": "CREATE"} + + vim_action_to_clone = mydb.get_rows(FROM="vim_actions", WHERE=where) + if not vim_action_to_clone: + raise NfvoException("Cannot find the vim_action at database with {}".format(where), HTTP_Internal_Server_Error) + vim_action_to_clone = vim_action_to_clone[0] + extra = yaml.safe_load(vim_action_to_clone["extra"]) + + # generate a new depends_on. Convert format TASK-Y into new format TASK-ACTION-XXXX.XXXX.Y + # TODO do the same for flavor and image when available + task_depends_on = [] + task_params = extra["params"] + task_params_networks = deepcopy(task_params[5]) + for iface in task_params[5]: + if iface["net_id"].startswith("TASK-"): + if "." not in iface["net_id"]: + task_depends_on.append("{}.{}".format(vim_action_to_clone["instance_action_id"], + iface["net_id"][5:])) + iface["net_id"] = "TASK-{}.{}".format(vim_action_to_clone["instance_action_id"], + iface["net_id"][5:]) + else: + task_depends_on.append(iface["net_id"][5:]) + if "mac_address" in iface: + del iface["mac_address"] + + vm_ifaces_to_clone = mydb.get_rows(FROM="instance_interfaces", WHERE={"instance_vm_id": target_vm["uuid"]}) + for index in range(0, vdu_count): + vm_uuid = str(uuid4()) + vm_name = target_vm.get('vim_name') + try: + suffix = vm_name.rfind("-") + vm_name = vm_name[:suffix+1] + str(1 + int(vm_name[suffix+1:])) + except Exception: + pass + db_instance_vm = { + "uuid": vm_uuid, + 'instance_vnf_id': target_vm['instance_vnf_id'], + 'vm_id': target_vm['vm_id'], + 'vim_name': vm_name + } + db_instance_vms.append(db_instance_vm) + + for vm_iface in vm_ifaces_to_clone: + iface_uuid = str(uuid4()) + iface2iface[vm_iface["uuid"]] = iface_uuid + db_vm_iface = { + "uuid": iface_uuid, + 'instance_vm_id': vm_uuid, + "instance_net_id": vm_iface["instance_net_id"], + 'interface_id': vm_iface['interface_id'], + 'type': vm_iface['type'], + 'floating_ip': vm_iface['floating_ip'], + 'port_security': vm_iface['port_security'] + } + db_instance_interfaces.append(db_vm_iface) + task_params_copy = deepcopy(task_params) + for iface in task_params_copy[5]: + iface["uuid"] = iface2iface[iface["uuid"]] + # increment ip_address + if "ip_address" in iface: + ip = iface.get("ip_address") + i = ip.rfind(".") + if i > 0: + try: + i += 1 + ip = ip[i:] + str(int(ip[:i]) + 1) + iface["ip_address"] = ip + except: + iface["ip_address"] = None + if vm_name: + task_params_copy[0] = vm_name + db_vim_action = { + "instance_action_id": instance_action_id, + "task_index": task_index, + "datacenter_vim_id": vim_action_to_clone["datacenter_vim_id"], + "action": "CREATE", + "status": "SCHEDULED", + "item": "instance_vms", + "item_id": vm_uuid, + # ALF + # ALF + # TODO examinar parametros, quitar MAC o incrementar. Incrementar IP y colocar las dependencias con ACTION-asdfasd. + # ALF + # ALF + "extra": yaml.safe_dump({"params": task_params_copy, "depends_on": task_depends_on}, default_flow_style=True, width=256) + } + task_index += 1 + db_vim_actions.append(db_vim_action) + vm_result[vdu_id]["created"].append(vm_uuid) + + db_instance_action["number_tasks"] = task_index + db_tables = [ + {"instance_vms": db_instance_vms}, + {"instance_interfaces": db_instance_interfaces}, + {"instance_actions": db_instance_action}, + # TODO revise sfps + # {"instance_sfis": db_instance_sfis}, + # {"instance_sfs": db_instance_sfs}, + # {"instance_classifications": db_instance_classifications}, + # {"instance_sfps": db_instance_sfps}, + {"vim_actions": db_vim_actions} + ] + logger.debug("create_vdu done DB tables: %s", + yaml.safe_dump(db_tables, indent=4, default_flow_style=False)) + mydb.new_rows(db_tables, []) + for myvim_thread in myvim_threads_id.values(): + vim_threads["running"][myvim_thread].insert_task(db_vim_actions) + + return vm_result input_vnfs = action_dict.pop("vnfs", []) input_vms = action_dict.pop("vms", []) - action_over_all = True if len(input_vnfs)==0 and len (input_vms)==0 else False - vm_result = {} - vm_error = 0 - vm_ok = 0 + action_over_all = True if not input_vnfs and not input_vms else False for sce_vnf in instanceDict['vnfs']: for vm in sce_vnf['vms']: - if not action_over_all: - if sce_vnf['uuid'] not in input_vnfs and sce_vnf['vnf_name'] not in input_vnfs and \ - vm['uuid'] not in input_vms and vm['name'] not in input_vms: - continue + if not action_over_all and sce_vnf['uuid'] not in input_vnfs and sce_vnf['vnf_name'] not in input_vnfs and \ + sce_vnf['member_vnf_index'] not in input_vnfs and \ + vm['uuid'] not in input_vms and vm['name'] not in input_vms: + continue try: if "add_public_key" in action_dict: mgmt_access = {} @@ -3627,7 +4559,7 @@ def instance_action(mydb,nfvo_tenant,instance_id, action_dict): return vm_result def instance_action_get(mydb, nfvo_tenant, instance_id, action_id): - filter={} + filter = {} if nfvo_tenant and nfvo_tenant != "any": filter["tenant_id"] = nfvo_tenant if instance_id and instance_id != "any": @@ -3635,9 +4567,12 @@ def instance_action_get(mydb, nfvo_tenant, instance_id, action_id): if action_id: filter["uuid"] = action_id rows = mydb.get_rows(FROM="instance_actions", WHERE=filter) - if not rows and action_id: - raise NfvoException("Not found any action with this criteria", HTTP_Not_Found) - return {"ations": rows} + if action_id: + if not rows: + raise NfvoException("Not found any action with this criteria", HTTP_Not_Found) + vim_actions = mydb.get_rows(FROM="vim_actions", WHERE={"instance_action_id": action_id}) + rows[0]["vim_actions"] = vim_actions + return {"actions": rows} def create_or_use_console_proxy_thread(console_server, console_port): @@ -3682,7 +4617,7 @@ def new_tenant(mydb, tenant_dict): tenant_dict['encrypted_RO_priv_key'] = priv_key mydb.new_row("nfvo_tenants", tenant_dict, confidential_data=True) except db_base_Exception as e: - raise NfvoException("Error creating the new tenant: {} ".format(tenant_dict['name']) + str(e), HTTP_Internal_Server_Error) + raise NfvoException("Error creating the new tenant: {} ".format(tenant_dict['name']) + str(e), e.http_code) return tenant_uuid def delete_tenant(mydb, tenant): @@ -3694,22 +4629,33 @@ def delete_tenant(mydb, tenant): def new_datacenter(mydb, datacenter_descriptor): + sdn_port_mapping = None if "config" in datacenter_descriptor: - datacenter_descriptor["config"]=yaml.safe_dump(datacenter_descriptor["config"],default_flow_style=True,width=256) - #Check that datacenter-type is correct + sdn_port_mapping = datacenter_descriptor["config"].pop("sdn-port-mapping", None) + datacenter_descriptor["config"] = yaml.safe_dump(datacenter_descriptor["config"], default_flow_style=True, + width=256) + # Check that datacenter-type is correct datacenter_type = datacenter_descriptor.get("type", "openvim"); - module_info = None + # module_info = None try: module = "vimconn_" + datacenter_type pkg = __import__("osm_ro." + module) - vim_conn = getattr(pkg, module) + # vim_conn = getattr(pkg, module) # module_info = imp.find_module(module, [__file__[:__file__.rfind("/")]]) except (IOError, ImportError): # if module_info and module_info[0]: # file.close(module_info[0]) - raise NfvoException("Incorrect datacenter type '{}'. Plugin '{}'.py not installed".format(datacenter_type, module), HTTP_Bad_Request) + raise NfvoException("Incorrect datacenter type '{}'. Plugin '{}.py' not installed".format(datacenter_type, + module), + HTTP_Bad_Request) datacenter_id = mydb.new_row("datacenters", datacenter_descriptor, add_uuid=True, confidential_data=True) + if sdn_port_mapping: + try: + datacenter_sdn_port_mapping_set(mydb, None, datacenter_id, sdn_port_mapping) + except Exception as e: + mydb.delete_row_by_id("datacenters", datacenter_id) # Rollback + raise e return datacenter_id @@ -3721,10 +4667,14 @@ def edit_datacenter(mydb, datacenter_id_name, datacenter_descriptor): datacenter_id = datacenter['uuid'] where={'uuid': datacenter['uuid']} remove_port_mapping = False + new_sdn_port_mapping = None if "config" in datacenter_descriptor: if datacenter_descriptor['config'] != None: try: new_config_dict = datacenter_descriptor["config"] + if "sdn-port-mapping" in new_config_dict: + remove_port_mapping = True + new_sdn_port_mapping = new_config_dict.pop("sdn-port-mapping") #delete null fields to_delete=[] for k in new_config_dict: @@ -3754,6 +4704,11 @@ def edit_datacenter(mydb, datacenter_id_name, datacenter_descriptor): logger.error("Error deleting datacenter-port-mapping " + str(e)) mydb.update_rows('datacenters', datacenter_descriptor, where) + if new_sdn_port_mapping: + try: + datacenter_sdn_port_mapping_set(mydb, None, datacenter_id, new_sdn_port_mapping) + except ovimException as e: + logger.error("Error adding datacenter-port-mapping " + str(e)) return datacenter_id @@ -3770,75 +4725,78 @@ def delete_datacenter(mydb, datacenter): def associate_datacenter_to_tenant(mydb, nfvo_tenant, datacenter, vim_tenant_id=None, vim_tenant_name=None, vim_username=None, vim_password=None, config=None): # get datacenter info - datacenter_id = get_datacenter_uuid(mydb, None, datacenter) - - create_vim_tenant = True if not vim_tenant_id and not vim_tenant_name else False - - # get nfvo_tenant info - tenant_dict = mydb.get_table_by_uuid_name('nfvo_tenants', nfvo_tenant) - if vim_tenant_name==None: - vim_tenant_name=tenant_dict['name'] - - #check that this association does not exist before - tenants_datacenter_dict={"nfvo_tenant_id":tenant_dict['uuid'], "datacenter_id":datacenter_id } - tenants_datacenters = mydb.get_rows(FROM='tenants_datacenters', WHERE=tenants_datacenter_dict) - if len(tenants_datacenters)>0: - raise NfvoException("datacenter '{}' and tenant'{}' are already attached".format(datacenter_id, tenant_dict['uuid']), HTTP_Conflict) - - vim_tenant_id_exist_atdb=False - if not create_vim_tenant: - where_={"datacenter_id": datacenter_id} - if vim_tenant_id!=None: - where_["vim_tenant_id"] = vim_tenant_id - if vim_tenant_name!=None: - where_["vim_tenant_name"] = vim_tenant_name - #check if vim_tenant_id is already at database - datacenter_tenants_dict = mydb.get_rows(FROM='datacenter_tenants', WHERE=where_) - if len(datacenter_tenants_dict)>=1: - datacenter_tenants_dict = datacenter_tenants_dict[0] - vim_tenant_id_exist_atdb=True - #TODO check if a field has changed and edit entry at datacenter_tenants at DB - else: #result=0 + try: + datacenter_id = get_datacenter_uuid(mydb, None, datacenter) + + create_vim_tenant = True if not vim_tenant_id and not vim_tenant_name else False + + # get nfvo_tenant info + tenant_dict = mydb.get_table_by_uuid_name('nfvo_tenants', nfvo_tenant) + if vim_tenant_name==None: + vim_tenant_name=tenant_dict['name'] + + #check that this association does not exist before + tenants_datacenter_dict={"nfvo_tenant_id":tenant_dict['uuid'], "datacenter_id":datacenter_id } + tenants_datacenters = mydb.get_rows(FROM='tenants_datacenters', WHERE=tenants_datacenter_dict) + if len(tenants_datacenters)>0: + raise NfvoException("datacenter '{}' and tenant'{}' are already attached".format(datacenter_id, tenant_dict['uuid']), HTTP_Conflict) + + vim_tenant_id_exist_atdb=False + if not create_vim_tenant: + where_={"datacenter_id": datacenter_id} + if vim_tenant_id!=None: + where_["vim_tenant_id"] = vim_tenant_id + if vim_tenant_name!=None: + where_["vim_tenant_name"] = vim_tenant_name + #check if vim_tenant_id is already at database + datacenter_tenants_dict = mydb.get_rows(FROM='datacenter_tenants', WHERE=where_) + if len(datacenter_tenants_dict)>=1: + datacenter_tenants_dict = datacenter_tenants_dict[0] + vim_tenant_id_exist_atdb=True + #TODO check if a field has changed and edit entry at datacenter_tenants at DB + else: #result=0 + datacenter_tenants_dict = {} + #insert at table datacenter_tenants + else: #if vim_tenant_id==None: + #create tenant at VIM if not provided + try: + _, myvim = get_datacenter_by_name_uuid(mydb, None, datacenter, vim_user=vim_username, + vim_passwd=vim_password) + datacenter_name = myvim["name"] + vim_tenant_id = myvim.new_tenant(vim_tenant_name, "created by openmano for datacenter "+datacenter_name) + except vimconn.vimconnException as e: + raise NfvoException("Not possible to create vim_tenant {} at VIM: {}".format(vim_tenant_id, str(e)), HTTP_Internal_Server_Error) datacenter_tenants_dict = {} - #insert at table datacenter_tenants - else: #if vim_tenant_id==None: - #create tenant at VIM if not provided - try: - _, myvim = get_datacenter_by_name_uuid(mydb, None, datacenter, vim_user=vim_username, - vim_passwd=vim_password) - datacenter_name = myvim["name"] - vim_tenant_id = myvim.new_tenant(vim_tenant_name, "created by openmano for datacenter "+datacenter_name) - except vimconn.vimconnException as e: - raise NfvoException("Not possible to create vim_tenant {} at VIM: {}".format(vim_tenant_id, str(e)), HTTP_Internal_Server_Error) - datacenter_tenants_dict = {} - datacenter_tenants_dict["created"]="true" - - #fill datacenter_tenants table - if not vim_tenant_id_exist_atdb: - datacenter_tenants_dict["vim_tenant_id"] = vim_tenant_id - datacenter_tenants_dict["vim_tenant_name"] = vim_tenant_name - datacenter_tenants_dict["user"] = vim_username - datacenter_tenants_dict["passwd"] = vim_password - datacenter_tenants_dict["datacenter_id"] = datacenter_id - if config: - datacenter_tenants_dict["config"] = yaml.safe_dump(config, default_flow_style=True, width=256) - id_ = mydb.new_row('datacenter_tenants', datacenter_tenants_dict, add_uuid=True, confidential_data=True) - datacenter_tenants_dict["uuid"] = id_ - - #fill tenants_datacenters table - datacenter_tenant_id = datacenter_tenants_dict["uuid"] - tenants_datacenter_dict["datacenter_tenant_id"] = datacenter_tenant_id - mydb.new_row('tenants_datacenters', tenants_datacenter_dict) - # create thread - datacenter_id, myvim = get_datacenter_by_name_uuid(mydb, tenant_dict['uuid'], datacenter_id) # reload data - datacenter_name = myvim["name"] - thread_name = get_non_used_vim_name(datacenter_name, datacenter_id, tenant_dict['name'], tenant_dict['uuid']) - new_thread = vim_thread.vim_thread(myvim, task_lock, thread_name, datacenter_name, datacenter_tenant_id, - db=db, db_lock=db_lock, ovim=ovim) - new_thread.start() - thread_id = datacenter_tenants_dict["uuid"] - vim_threads["running"][thread_id] = new_thread - return datacenter_id + datacenter_tenants_dict["created"]="true" + + #fill datacenter_tenants table + if not vim_tenant_id_exist_atdb: + datacenter_tenants_dict["vim_tenant_id"] = vim_tenant_id + datacenter_tenants_dict["vim_tenant_name"] = vim_tenant_name + datacenter_tenants_dict["user"] = vim_username + datacenter_tenants_dict["passwd"] = vim_password + datacenter_tenants_dict["datacenter_id"] = datacenter_id + if config: + datacenter_tenants_dict["config"] = yaml.safe_dump(config, default_flow_style=True, width=256) + id_ = mydb.new_row('datacenter_tenants', datacenter_tenants_dict, add_uuid=True, confidential_data=True) + datacenter_tenants_dict["uuid"] = id_ + + #fill tenants_datacenters table + datacenter_tenant_id = datacenter_tenants_dict["uuid"] + tenants_datacenter_dict["datacenter_tenant_id"] = datacenter_tenant_id + mydb.new_row('tenants_datacenters', tenants_datacenter_dict) + # create thread + datacenter_id, myvim = get_datacenter_by_name_uuid(mydb, tenant_dict['uuid'], datacenter_id) # reload data + datacenter_name = myvim["name"] + thread_name = get_non_used_vim_name(datacenter_name, datacenter_id, tenant_dict['name'], tenant_dict['uuid']) + new_thread = vim_thread.vim_thread(myvim, task_lock, thread_name, datacenter_name, datacenter_tenant_id, + db=db, db_lock=db_lock, ovim=ovim) + new_thread.start() + thread_id = datacenter_tenants_dict["uuid"] + vim_threads["running"][thread_id] = new_thread + return datacenter_id + except vimconn.vimconnException as e: + raise NfvoException(str(e), HTTP_Bad_Request) def edit_datacenter_to_tenant(mydb, nfvo_tenant, datacenter_id, vim_tenant_id=None, vim_tenant_name=None, @@ -3880,9 +4838,6 @@ def edit_datacenter_to_tenant(mydb, nfvo_tenant, datacenter_id, vim_tenant_id=No return datacenter_id def deassociate_datacenter_to_tenant(mydb, tenant_id, datacenter, vim_tenant_id=None): - #get datacenter info - datacenter_id, myvim = get_datacenter_by_name_uuid(mydb, tenant_id, datacenter) - #get nfvo_tenant info if not tenant_id or tenant_id=="any": tenant_uuid = None @@ -3890,8 +4845,9 @@ def deassociate_datacenter_to_tenant(mydb, tenant_id, datacenter, vim_tenant_id= tenant_dict = mydb.get_table_by_uuid_name('nfvo_tenants', tenant_id) tenant_uuid = tenant_dict['uuid'] + datacenter_id = get_datacenter_uuid(mydb, tenant_uuid, datacenter) #check that this association exist before - tenants_datacenter_dict={"datacenter_id":datacenter_id } + tenants_datacenter_dict={"datacenter_id": datacenter_id } if tenant_uuid: tenants_datacenter_dict["nfvo_tenant_id"] = tenant_uuid tenant_datacenter_list = mydb.get_rows(FROM='tenants_datacenters', WHERE=tenants_datacenter_dict) @@ -3911,6 +4867,7 @@ def deassociate_datacenter_to_tenant(mydb, tenant_id, datacenter, vim_tenant_id= if vim_tenant_dict['created']=='true': #delete tenant at VIM if created by NFVO try: + datacenter_id, myvim = get_datacenter_by_name_uuid(mydb, tenant_id, datacenter) myvim.delete_tenant(vim_tenant_dict['vim_tenant_id']) except vimconn.vimconnException as e: warning = "Not possible to delete vim_tenant_id {} from VIM: {} ".format(vim_tenant_dict['vim_tenant_id'], str(e)) @@ -3919,9 +4876,10 @@ def deassociate_datacenter_to_tenant(mydb, tenant_id, datacenter, vim_tenant_id= logger.error("Cannot delete datacenter_tenants " + str(e)) pass # the error will be caused because dependencies, vim_tenant can not be deleted thread_id = tenant_datacenter_item["datacenter_tenant_id"] - thread = vim_threads["running"][thread_id] - thread.insert_task("exit") - vim_threads["deleting"][thread_id] = thread + thread = vim_threads["running"].get(thread_id) + if thread: + thread.insert_task("exit") + vim_threads["deleting"][thread_id] = thread return "datacenter {} detached. {}".format(datacenter_id, warning) @@ -4063,7 +5021,7 @@ def get_sdn_net_id(mydb, tenant_id, datacenter, network_id): result = mydb.get_rows(SELECT=('sdn_net_id',), FROM='instance_nets', WHERE=search_dict) except db_base_Exception as e: raise NfvoException("db_base_Exception obtaining SDN network to associated to vim network {}".format( - network_id) + str(e), HTTP_Internal_Server_Error) + network_id) + str(e), e.http_code) sdn_net_counter = 0 for net in result: @@ -4120,7 +5078,7 @@ def vim_net_sdn_attach(mydb, tenant_id, datacenter, network_id, descriptor): sdn_network_id, network_id) + str(e), HTTP_Internal_Server_Error) except db_base_Exception as e: raise NfvoException("db_base_Exception attaching SDN network to vim network {}".format( - network_id) + str(e), HTTP_Internal_Server_Error) + network_id) + str(e), e.http_code) return 'Port uuid: '+ result @@ -4250,7 +5208,7 @@ def vim_action_delete(mydb, tenant_id, datacenter, item, name): mydb.delete_row(FROM='instance_nets', WHERE={'instance_scenario_id': None, 'sdn_net_id': sdn_network_id, 'vim_net_id': item_id}) except db_base_Exception as e: raise NfvoException("Error deleting correspondence for VIM/SDN dataplane networks{}: ".format(correspondence) + - str(e), HTTP_Internal_Server_Error) + str(e), e.http_code) #Delete the SDN network try: @@ -4292,29 +5250,35 @@ def vim_action_create(mydb, tenant_id, datacenter, item, descriptor): #If the datacenter has a SDN controller defined and the network is of dataplane type, then create the sdn network if get_sdn_controller_id(mydb, datacenter) != None and (net_type == 'data' or net_type == 'ptp'): + #obtain datacenter_tenant_id + datacenter_tenant_id = mydb.get_rows(SELECT=('uuid',), + FROM='datacenter_tenants', + WHERE={'datacenter_id': datacenter})[0]['uuid'] try: sdn_network = {} sdn_network['vlan'] = net_vlan sdn_network['type'] = net_type sdn_network['name'] = net_name + sdn_network['region'] = datacenter_tenant_id ovim_content = ovim.new_network(sdn_network) except ovimException as e: - self.logger.error("ovimException creating SDN network={} ".format( + logger.error("ovimException creating SDN network={} ".format( sdn_network) + str(e), exc_info=True) raise NfvoException("ovimException creating SDN network={} ".format(sdn_network) + str(e), HTTP_Internal_Server_Error) # Save entry in in dabase mano_db in table instance_nets to stablish a dictionary vim_net_id <->sdn_net_id # use instance_scenario_id=None to distinguish from real instaces of nets - correspondence = {'instance_scenario_id': None, 'sdn_net_id': ovim_content, 'vim_net_id': content} - #obtain datacenter_tenant_id - correspondence['datacenter_tenant_id'] = mydb.get_rows(SELECT=('uuid',), FROM='datacenter_tenants', WHERE={'datacenter_id': datacenter})[0]['uuid'] - + correspondence = {'instance_scenario_id': None, + 'sdn_net_id': ovim_content, + 'vim_net_id': content, + 'datacenter_tenant_id': datacenter_tenant_id + } try: mydb.new_row('instance_nets', correspondence, add_uuid=True) except db_base_Exception as e: - raise NfvoException("Error saving correspondence for VIM/SDN dataplane networks{}: ".format(correspondence) + - str(e), HTTP_Internal_Server_Error) + raise NfvoException("Error saving correspondence for VIM/SDN dataplane networks{}: {}".format( + correspondence, e), e.http_code) elif item=="tenants": tenant = descriptor["tenant"] content = myvim.new_tenant(tenant["name"], tenant.get("description")) @@ -4379,13 +5343,15 @@ def datacenter_sdn_port_mapping_set(mydb, tenant_id, datacenter_id, sdn_port_map element = dict() element["compute_node"] = compute_node["compute_node"] for port in compute_node["ports"]: - element["pci"] = port.get("pci") + pci = port.get("pci") element["switch_port"] = port.get("switch_port") element["switch_mac"] = port.get("switch_mac") - if not element["pci"] or not (element["switch_port"] or element["switch_mac"]): + if not pci or not (element["switch_port"] or element["switch_mac"]): raise NfvoException ("The mapping must contain the 'pci' and at least one of the elements 'switch_port'" " or 'switch_mac'", HTTP_Bad_Request) - maps.append(dict(element)) + for pci_expanded in utils.expand_brackets(pci): + element["pci"] = pci_expanded + maps.append(dict(element)) return ovim.set_of_port_mapping(maps, ofc_id=sdn_controller_id, switch_dpid=switch_dpid, region=datacenter_id)