X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2FRO.git;a=blobdiff_plain;f=RO%2Fosm_ro%2Fnfvo.py;h=2efd61043ca2839417f6cbdc6f708617ac4769fd;hp=36109cb817c98f15e64679008c2a1bf4161a102f;hb=0fd60da673bd914ed150c83f8673569307dcc28f;hpb=4126d05e24ada55226bb13a9d556655811cedadc diff --git a/RO/osm_ro/nfvo.py b/RO/osm_ro/nfvo.py index 36109cb8..2efd6104 100644 --- a/RO/osm_ro/nfvo.py +++ b/RO/osm_ro/nfvo.py @@ -29,7 +29,9 @@ __date__ ="$16-sep-2014 22:05:01$" # import imp import json +import string import yaml +from random import choice as random_choice from osm_ro import utils from osm_ro.utils import deprecated from osm_ro.vim_thread import vim_thread @@ -63,7 +65,7 @@ from pkg_resources import iter_entry_points # WIM from .wim import sdnconn -from .wim.wimconn_fake import FakeConnector +from .wim.wimconn_dummy import DummyConnector from .wim.failing_connector import FailingConnector from .http_tools import errors as httperrors from .wim.engine import WimEngine @@ -100,6 +102,7 @@ last_task_id = 0.0 db = None db_lock = Lock() +worker_id = None class NfvoException(httperrors.HttpMappedError): """Common Class for NFVO errors""" @@ -144,20 +147,29 @@ def new_task(name, params, depends=None): def is_task_id(id): return True if id[:5] == "TASK-" else False - -def get_non_used_vim_name(datacenter_name, datacenter_id, tenant_name, tenant_id): - name = datacenter_name[:16] - if name not in vim_threads["names"]: - vim_threads["names"].append(name) - return name - if tenant_name: - name = datacenter_name[:16] + "." + tenant_name[:16] - if name not in vim_threads["names"]: - vim_threads["names"].append(name) - return name - name = datacenter_id - vim_threads["names"].append(name) - return name +def get_process_id(): + """ + Obtain a unique ID for this process. If running from inside docker, it will get docker ID. If not it + will provide a random one + :return: Obtained ID + """ + # Try getting docker id. If fails, get pid + try: + with open("/proc/self/cgroup", "r") as f: + text_id_ = f.readline() + _, _, text_id = text_id_.rpartition("/") + text_id = text_id.replace("\n", "")[:12] + if text_id: + return text_id + except Exception: + pass + # Return a random id + return "".join(random_choice("0123456789abcdef") for _ in range(12)) + +def get_non_used_vim_name(datacenter_name, datacenter_id): + return "{}:{}:{}".format( + worker_id[:12], datacenter_id.replace("-", "")[:32], datacenter_name[:16] + ) # -- Move def get_non_used_wim_name(wim_name, wim_id, tenant_name, tenant_id): @@ -175,7 +187,7 @@ def get_non_used_wim_name(wim_name, wim_id, tenant_name, tenant_id): def start_service(mydb, persistence=None, wim=None): - global db, global_config, plugins, ovim + global db, global_config, plugins, ovim, worker_id db = nfvo_db.nfvo_db(lock=db_lock) mydb.lock = db_lock db.connect(global_config['db_host'], global_config['db_user'], global_config['db_passwd'], global_config['db_name']) @@ -183,8 +195,9 @@ def start_service(mydb, persistence=None, wim=None): persistence = persistence or WimPersistence(db) try: - if "rosdn_fake" not in plugins: - plugins["rosdn_fake"] = FakeConnector + worker_id = get_process_id() + if "rosdn_dummy" not in plugins: + plugins["rosdn_dummy"] = DummyConnector # starts ovim library ovim = Sdn(db, plugins) @@ -237,8 +250,7 @@ def start_service(mydb, persistence=None, wim=None): vim['datacenter_id'], e)) # raise NfvoException("Error at VIM {}; {}: {}".format(vim["type"], type(e).__name__, e), # httperrors.Internal_Server_Error) - thread_name = get_non_used_vim_name(vim['datacenter_name'], vim['datacenter_id'], vim['vim_tenant_name'], - vim['vim_tenant_id']) + thread_name = get_non_used_vim_name(vim['datacenter_name'], vim['datacenter_id']) new_thread = vim_thread(task_lock, plugins, thread_name, None, vim['datacenter_tenant_id'], db=db) new_thread.start() @@ -252,7 +264,7 @@ def start_service(mydb, persistence=None, wim=None): _load_plugin(plugin_name, type="sdn") thread_id = wim['uuid'] - thread_name = get_non_used_vim_name(wim['name'], wim['uuid'], wim['uuid'], None) + thread_name = get_non_used_vim_name(wim['name'], wim['uuid']) new_thread = vim_thread(task_lock, plugins, thread_name, wim['uuid'], None, db=db) new_thread.start() vim_threads["running"][thread_id] = new_thread @@ -977,6 +989,7 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): # table nets (internal-vld) net_id2uuid = {} # for mapping interface with network + net_id2index = {} # for mapping interface with network for vld in vnfd.get("internal-vld").values(): net_uuid = str(uuid4()) uuid_list.append(net_uuid) @@ -989,6 +1002,7 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): "type": "bridge", # TODO adjust depending on connection point type } net_id2uuid[vld.get("id")] = net_uuid + net_id2index[vld.get("id")] = len(db_nets) db_nets.append(db_net) # ip-profile, link db_ip_profile with db_sce_net if vld.get("ip-profile-ref"): @@ -1181,7 +1195,7 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): raise KeyError() if vdu_id in vdu_id2cp_name: - vdu_id2cp_name[vdu_id] = None # more than two connecdtion point for this VDU + vdu_id2cp_name[vdu_id] = None # more than two connection point for this VDU else: vdu_id2cp_name[vdu_id] = db_interface["external_name"] @@ -1216,6 +1230,10 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): if not icp: raise KeyError("is not referenced by any 'internal-vld'") + # set network type as data + if iface.get("virtual-interface") and iface["virtual-interface"].get("type") in \ + ("SR-IOV", "PCI-PASSTHROUGH"): + db_nets[net_id2index[icp_vld.get("id")]]["type"] = "data" db_interface["net_id"] = net_id2uuid[icp_vld.get("id")] if str(icp_descriptor.get("port-security-enabled")).lower() == "false": db_interface["port_security"] = 0 @@ -2433,7 +2451,6 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): elif vld.get("vim-network-name"): db_sce_net["vim_network_name"] = get_str(vld, "vim-network-name", 255) - # table sce_interfaces (vld:vnfd-connection-point-ref) for iface in vld.get("vnfd-connection-point-ref").values(): # Check if there are VDUs in the descriptor @@ -2447,7 +2464,7 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): "'nsd':'constituent-vnfd'".format( str(nsd["id"]), str(vld["id"]), str(iface["member-vnf-index-ref"])), httperrors.Bad_Request) - + existing_ifaces = mydb.get_rows(SELECT=('i.uuid as uuid', 'i.type as iface_type'), FROM="interfaces as i join vms on i.vm_id=vms.uuid", WHERE={'vnf_id': vnf_index2vnf_uuid[vnf_index], @@ -2474,7 +2491,7 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): "sce_net_id": sce_net_uuid, "interface_id": interface_uuid, "ip_address": iface_ip_address, - } + } db_sce_interfaces.append(db_sce_interface) if not db_sce_net["type"]: db_sce_net["type"] = "bridge" @@ -2963,7 +2980,7 @@ def get_vim_thread(mydb, tenant_id, datacenter_id_name=None, datacenter_tenant_i if datacenter_tenant_id: thread_id = datacenter_tenant_id thread = vim_threads["running"].get(datacenter_tenant_id) - else: + if not thread: where_={"td.nfvo_tenant_id": tenant_id} if datacenter_id_name: if utils.check_valid_uuid(datacenter_id_name): @@ -2975,7 +2992,7 @@ def get_vim_thread(mydb, tenant_id, datacenter_id_name=None, datacenter_tenant_i if datacenter_tenant_id: where_["dt.uuid"] = datacenter_tenant_id datacenters = mydb.get_rows( - SELECT=("dt.uuid as datacenter_tenant_id",), + SELECT=("dt.uuid as datacenter_tenant_id, d.name as datacenter_name",), FROM="datacenter_tenants as dt join tenants_datacenters as td on dt.uuid=td.datacenter_tenant_id " "join datacenters as d on d.uuid=dt.datacenter_id", WHERE=where_) @@ -2983,7 +3000,14 @@ def get_vim_thread(mydb, tenant_id, datacenter_id_name=None, datacenter_tenant_i raise NfvoException("More than one datacenters found, try to identify with uuid", httperrors.Conflict) elif datacenters: thread_id = datacenters[0]["datacenter_tenant_id"] + datacenter_name = datacenters[0]["datacenter_name"] thread = vim_threads["running"].get(thread_id) + if not thread: + thread_name = get_non_used_vim_name(datacenter_name, datacenter_id) + thread = vim_thread(task_lock, plugins, thread_name, None, + thread_id, db=mydb) + thread.start() + vim_threads["running"][thread_id] = thread if not thread: raise NfvoException("datacenter '{}' not found".format(str(datacenter_id_name)), httperrors.Not_Found) return thread_id, thread @@ -3205,12 +3229,12 @@ def create_instance(mydb, tenant_id, instance_dict): else: update(scenario_net['ip_profile'], ipprofile_db) - if 'provider-network' in net_instance_desc: - provider_network_db = net_instance_desc['provider-network'] - if 'provider-network' not in scenario_net: - scenario_net['provider-network'] = provider_network_db - else: - update(scenario_net['provider-network'], provider_network_db) + if net_instance_desc.get('provider-network'): + provider_network_db = net_instance_desc['provider-network'] + if 'provider_network' not in scenario_net: + scenario_net['provider_network'] = provider_network_db + else: + update(scenario_net['provider_network'], provider_network_db) for vdu_id, vdu_instance_desc in vnf_instance_desc.get("vdus", {}).items(): for scenario_vm in scenario_vnf['vms']: @@ -3287,28 +3311,6 @@ def create_instance(mydb, tenant_id, instance_dict): involved_datacenters.append(default_datacenter_id) target_wim_account = sce_net.get("wim_account", default_wim_account) - # --> WIM - # TODO: use this information during network creation - wim_account_id = wim_account_name = None - if len(involved_datacenters) > 1 and 'uuid' in sce_net: - if target_wim_account is None or target_wim_account is True: # automatic selection of WIM - # OBS: sce_net without uuid are used internally to VNFs - # and the assumption is that VNFs will not be split among - # different datacenters - wim_account = wim_engine.find_suitable_wim_account( - involved_datacenters, tenant_id) - wim_account_id = wim_account['uuid'] - wim_account_name = wim_account['name'] - wim_usage[sce_net['uuid']] = wim_account_id - elif isinstance(target_wim_account, str): # manual selection of WIM - wim_account.persist.get_wim_account_by(target_wim_account, tenant_id) - wim_account_id = wim_account['uuid'] - wim_account_name = wim_account['name'] - wim_usage[sce_net['uuid']] = wim_account_id - else: # not WIM usage - wim_usage[sce_net['uuid']] = False - # <-- WIM - descriptor_net = {} if instance_dict.get("networks"): if sce_net.get("uuid") in instance_dict["networks"]: @@ -3341,14 +3343,39 @@ def create_instance(mydb, tenant_id, instance_dict): ) if not target_instance_nets: raise NfvoException( - "Cannot find the target network at instance:networks[{}]:use-network".format(descriptor_net_name), - httperrors.Bad_Request) + "Cannot find the target network at instance:networks[{}]:use-network".format( + descriptor_net_name), httperrors.Bad_Request) else: use_network = target_instance_nets[0]["related"] if sce_net["external"]: number_mgmt_networks += 1 + # --> WIM + # TODO: use this information during network creation + wim_account_id = wim_account_name = None + if len(involved_datacenters) > 1 and 'uuid' in sce_net: + urls = [myvims[v].url for v in involved_datacenters] + if len(set(urls)) < 2: + wim_usage[sce_net['uuid']] = False + elif target_wim_account is None or target_wim_account is True: # automatic selection of WIM + # OBS: sce_net without uuid are used internally to VNFs + # and the assumption is that VNFs will not be split among + # different datacenters + wim_account = wim_engine.find_suitable_wim_account( + involved_datacenters, tenant_id) + wim_account_id = wim_account['uuid'] + wim_account_name = wim_account['name'] + wim_usage[sce_net['uuid']] = wim_account_id + elif isinstance(target_wim_account, str): # manual selection of WIM + wim_account.persist.get_wim_account_by(target_wim_account, tenant_id) + wim_account_id = wim_account['uuid'] + wim_account_name = wim_account['name'] + wim_usage[sce_net['uuid']] = wim_account_id + else: # not WIM usage + wim_usage[sce_net['uuid']] = False + # <-- WIM + for datacenter_id in involved_datacenters: netmap_use = None netmap_create = None @@ -3457,7 +3484,11 @@ def create_instance(mydb, tenant_id, instance_dict): "created": create_network, # TODO py3 "sdn": True, }) + task_wim_extra = {"params": [net_type, wim_account_name]} + # add sdn interfaces + if sce_net.get('provider_network') and sce_net['provider_network'].get("sdn-ports"): + task_wim_extra["sdn-ports"] = sce_net['provider_network'].get("sdn-ports") db_vim_action = { "instance_action_id": instance_action_id, "status": "SCHEDULED", @@ -3793,6 +3824,7 @@ def instantiate_vnf(mydb, sce_vnf, params, params_out, rollbackList): sce_net2wim_instance = params_out["sce_net2wim_instance"] vnf_net2instance = {} + vnf_net2wim_instance = {} # 2. Creating new nets (vnf internal nets) in the VIM" # For each vnf net, we create it and we add it to instanceNetlist. @@ -3840,6 +3872,7 @@ def instantiate_vnf(mydb, sce_vnf, params, params_out, rollbackList): "created": True, # TODO py3 "sdn": True, }) + vnf_net2wim_instance[net_uuid] = sdn_net_id db_net = { "uuid": net_uuid, @@ -4076,7 +4109,7 @@ def instantiate_vnf(mydb, sce_vnf, params, params_out, rollbackList): else: netDict['net_id'] = "TASK-{}".format(net2task_id[sce_vnf['uuid']][iface['net_id']]) instance_net_id = vnf_net2instance[sce_vnf['uuid']][iface['net_id']] - instance_wim_net_id = None + instance_wim_net_id = vnf_net2wim_instance.get(instance_net_id) task_depends_on.append(net2task_id[sce_vnf['uuid']][iface['net_id']]) # skip bridge ifaces not connected to any net if 'net_id' not in netDict or netDict['net_id'] == None: @@ -4158,14 +4191,14 @@ def instantiate_vnf(mydb, sce_vnf, params, params_out, rollbackList): db_vm_iface_instance.update(db_vm_iface) if db_vm_iface_instance.get("ip_address"): # increment ip_address ip = db_vm_iface_instance.get("ip_address") - i = ip.rfind(".") - if i > 0: - try: + try: + i = ip.rfind(".") + if i > 0: i += 1 ip = ip[i:] + str(int(ip[:i]) + 1) db_vm_iface_instance["ip_address"] = ip - except: - db_vm_iface_instance["ip_address"] = None + except: + db_vm_iface_instance["ip_address"] = None db_instance_interfaces.append(db_vm_iface_instance) myVMDict['networks'][iface_index]["uuid"] = iface_uuid iface_index += 1 @@ -4795,6 +4828,16 @@ def instance_action(mydb,nfvo_tenant,instance_id, action_dict): "extra": yaml.safe_dump({"params": vm_interfaces}, default_flow_style=True, width=256) } + # get affected instance_interfaces (deleted on cascade) to check if a wim_network must be updated + deleted_interfaces = mydb.get_rows( + SELECT=("instance_wim_net_id", ), + FROM="instance_interfaces", + WHERE={"instance_vm_id": vdu_id, "instance_wim_net_id<>": None}, + ) + for deleted_interface in deleted_interfaces: + db_vim_actions.append({"TO-UPDATE": {}, "WHERE": { + "item": "instance_wim_nets", "item_id": deleted_interface["instance_wim_net_id"]}}) + task_index += 1 db_vim_actions.append(db_vim_action) vm_result["deleted"].append(vdu_id) @@ -4853,26 +4896,31 @@ def instance_action(mydb,nfvo_tenant,instance_id, action_dict): "uuid": iface_uuid, 'instance_vm_id': vm_uuid, "instance_net_id": vm_iface["instance_net_id"], + "instance_wim_net_id": vm_iface["instance_wim_net_id"], 'interface_id': vm_iface['interface_id'], 'type': vm_iface['type'], + 'model': vm_iface['model'], 'floating_ip': vm_iface['floating_ip'], 'port_security': vm_iface['port_security'] } db_instance_interfaces.append(db_vm_iface) + if db_vm_iface["instance_wim_net_id"]: + db_vim_actions.append({"TO-UPDATE": {}, "WHERE": { + "item": "instance_wim_nets", "item_id": db_vm_iface["instance_wim_net_id"]}}) task_params_copy = deepcopy(task_params) for iface in task_params_copy[5]: iface["uuid"] = iface2iface[iface["uuid"]] # increment ip_address - if "ip_address" in iface: - ip = iface.get("ip_address") - i = ip.rfind(".") - if i > 0: - try: + if iface.get("ip_address"): + try: + ip = iface["ip_address"] + i = ip.rfind(".") + if i > 0: i += 1 ip = ip[i:] + str(int(ip[:i]) + 1) iface["ip_address"] = ip - except: - iface["ip_address"] = None + except: + iface["ip_address"] = None if vm_name: task_params_copy[0] = vm_name db_vim_action = { @@ -5085,6 +5133,15 @@ def new_datacenter(mydb, datacenter_descriptor): datacenter_type = datacenter_descriptor.get("type", "openvim"); # module_info = None + for url_field in ('vim_url', 'vim_url_admin'): + # It is common that users copy and paste the URL from the VIM website + # (example OpenStack), therefore a common mistake is to include blank + # characters at the end of the URL. Let's remove it and just in case, + # lets remove trailing slash as well. + url = datacenter_descriptor.get(url_field) + if url: + datacenter_descriptor[url_field] = url.strip(string.whitespace + '/') + # load plugin plugin_name = "rovim_" + datacenter_type if plugin_name not in plugins: @@ -5208,12 +5265,13 @@ def create_vim_account(mydb, nfvo_tenant, datacenter_id, name=None, vim_id=None, else: #if vim_tenant==None: #create tenant at VIM if not provided try: - _, myvim = get_datacenter_by_name_uuid(mydb, None, datacenter, vim_user=vim_username, - vim_passwd=vim_password) + _, myvim = get_datacenter_by_name_uuid(mydb, None, datacenter_id, vim_user=vim_username, + vim_passwd=vim_password) datacenter_name = myvim["name"] vim_tenant = myvim.new_tenant(vim_tenant_name, "created by openmano for datacenter "+datacenter_name) except vimconn.vimconnException as e: - raise NfvoException("Not possible to create vim_tenant {} at VIM: {}".format(vim_tenant_id, str(e)), httperrors.Internal_Server_Error) + raise NfvoException("Not possible to create vim_tenant {} at VIM: {}".format(vim_tenant_name, e), + httperrors.Internal_Server_Error) datacenter_tenants_dict = {} datacenter_tenants_dict["created"]="true" @@ -5239,7 +5297,7 @@ def create_vim_account(mydb, nfvo_tenant, datacenter_id, name=None, vim_id=None, mydb.new_row('tenants_datacenters', tenants_datacenter_dict) # create thread - thread_name = get_non_used_vim_name(datacenter_name, datacenter_id, tenant_dict['name'], tenant_dict['uuid']) + thread_name = get_non_used_vim_name(datacenter_name, datacenter_id) new_thread = vim_thread(task_lock, plugins, thread_name, None, datacenter_tenant_id, db=db) new_thread.start() thread_id = datacenter_tenants_dict["uuid"] @@ -5271,7 +5329,7 @@ def edit_vim_account(mydb, nfvo_tenant, datacenter_tenant_id, datacenter_id=None if config: original_config_dict = yaml.load(original_config, Loader=yaml.Loader) original_config_dict.update(config) - update["config"] = yaml.safe_dump(original_config_dict, default_flow_style=True, width=256) + update_["config"] = yaml.safe_dump(original_config_dict, default_flow_style=True, width=256) if name: update_['name'] = name if vim_tenant: @@ -5759,7 +5817,13 @@ def sdn_controller_create(mydb, tenant_id, sdn_controller): try: wim_id = ovim.new_of_controller(sdn_controller) - thread_name = get_non_used_vim_name(sdn_controller['name'], wim_id, wim_id, None) + # Load plugin if not previously loaded + controller_type = sdn_controller.get("type") + plugin_name = "rosdn_" + controller_type + if plugin_name not in plugins: + _load_plugin(plugin_name, type="sdn") + + thread_name = get_non_used_vim_name(sdn_controller['name'], wim_id) new_thread = vim_thread(task_lock, plugins, thread_name, wim_id, None, db=db) new_thread.start() thread_id = wim_id @@ -5823,6 +5887,8 @@ def datacenter_sdn_port_mapping_set(mydb, tenant_id, datacenter_id, sdn_port_map pci = port.get("pci") element["switch_port"] = port.get("switch_port") element["switch_mac"] = port.get("switch_mac") + element["switch_dpid"] = port.get("switch_dpid") + element["switch_id"] = port.get("switch_id") if not element["switch_port"] and not element["switch_mac"]: raise NfvoException ("The mapping must contain 'switch_port' or 'switch_mac'", httperrors.Bad_Request) for pci_expanded in utils.expand_brackets(pci):