X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=models%2Fopenmano%2Fpython%2Frift%2Fopenmano%2Frift2openmano.py;h=aeda2894571a1da4f6eb211407c7028577e12ab9;hb=f314b4af9744068a7ed7a6a6314220c3aa857523;hp=6cc13a7e8125d166906e262b3b874e0c8ff94054;hpb=49868d2c71eb364cee9707515be6841a568dad40;p=osm%2FSO.git diff --git a/models/openmano/python/rift/openmano/rift2openmano.py b/models/openmano/python/rift/openmano/rift2openmano.py index 6cc13a7e..aeda2894 100755 --- a/models/openmano/python/rift/openmano/rift2openmano.py +++ b/models/openmano/python/rift/openmano/rift2openmano.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# +# # Copyright 2016 RIFT.IO Inc # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,13 +24,27 @@ import os import sys import tempfile import yaml +import ast +import json + +import gi +gi.require_version('RwYang', '1.0') +gi.require_version('RwProjectVnfdYang', '1.0') +gi.require_version('RwProjectNsdYang', '1.0') +gi.require_version('NsdYang', '1.0') +gi.require_version('VnfdYang', '1.0') from gi.repository import ( RwYang, - RwVnfdYang, - RwNsdYang, + RwProjectVnfdYang as RwVnfdYang, + RwProjectNsdYang as RwNsdYang, + NsdYang as NsdYang, + VnfdYang as VnfdYang ) +import rift.package.store +import rift.package.cloud_init + logger = logging.getLogger("rift2openmano.py") @@ -39,10 +53,9 @@ class VNFNotFoundError(Exception): class RiftNSD(object): - model = RwYang.Model.create_libncx() + model = RwYang.Model.create_libyang() model.load_module('nsd') - model.load_module('rw-nsd') - + def __init__(self, descriptor): self._nsd = descriptor @@ -65,6 +78,10 @@ class RiftNSD(object): def constituent_vnfds(self): return self._nsd.constituent_vnfd + @property + def scaling_group_descriptor(self): + return self._nsd.scaling_group_descriptor + @property def vlds(self): return self._nsd.vld @@ -80,28 +97,26 @@ class RiftNSD(object): @classmethod def from_xml_file_hdl(cls, hdl): hdl.seek(0) - descriptor = RwNsdYang.YangData_Nsd_NsdCatalog_Nsd() + descriptor = NsdYang.YangData_Nsd_NsdCatalog_Nsd() descriptor.from_xml_v2(RiftNSD.model, hdl.read()) return cls(descriptor) @classmethod def from_yaml_file_hdl(cls, hdl): hdl.seek(0) - descriptor = RwNsdYang.YangData_Nsd_NsdCatalog_Nsd() + descriptor = NsdYang.YangData_Nsd_NsdCatalog_Nsd() descriptor.from_yaml(RiftNSD.model, hdl.read()) return cls(descriptor) - @classmethod - def from_dict(cls, nsd_dict): - descriptor = RwNsdYang.YangData_Nsd_NsdCatalog_Nsd.from_dict(nsd_dict) - return cls(descriptor) + def from_dict(self): + descriptor = NsdYang.YangData_Nsd_NsdCatalog_Nsd.from_dict(self._nsd.as_dict(), ignore_missing_keys=True).to_json_without_namespace(RiftNSD.model) + return descriptor class RiftVNFD(object): - model = RwYang.Model.create_libncx() + model = RwYang.Model.create_libyang() model.load_module('vnfd') - model.load_module('rw-vnfd') - + def __init__(self, descriptor): self._vnfd = descriptor @@ -135,21 +150,20 @@ class RiftVNFD(object): @classmethod def from_xml_file_hdl(cls, hdl): hdl.seek(0) - descriptor = RwVnfdYang.YangData_Vnfd_VnfdCatalog_Vnfd() + descriptor = VnfdYang.YangData_Vnfd_VnfdCatalog_Vnfd() descriptor.from_xml_v2(RiftVNFD.model, hdl.read()) return cls(descriptor) @classmethod def from_yaml_file_hdl(cls, hdl): hdl.seek(0) - descriptor = RwVnfdYang.YangData_Vnfd_VnfdCatalog_Vnfd() + descriptor = VnfdYang.YangData_Vnfd_VnfdCatalog_Vnfd() descriptor.from_yaml(RiftVNFD.model, hdl.read()) return cls(descriptor) - @classmethod - def from_dict(cls, vnfd_dict): - descriptor = RwVnfdYang.YangData_Vnfd_VnfdCatalog_Vnfd.from_dict(vnfd_dict) - return cls(descriptor) + def from_dict(self): + descriptor = VnfdYang.YangData_Vnfd_VnfdCatalog_Vnfd.from_dict(self._vnfd.as_dict(), ignore_missing_keys=True).to_json_without_namespace(RiftVNFD.model) + return descriptor def is_writable_directory(dir_path): @@ -170,61 +184,39 @@ def is_writable_directory(dir_path): return True -def create_vnfd_from_xml_files(vnfd_file_hdls): - """ Create a list of RiftVNFD instances from xml file handles +def create_vnfd_from_files(vnfd_file_hdls): + """ Create a list of RiftVNFD instances from xml/yaml file handles Arguments: - vnfd_file_hdls - Rift VNFD XML file handles + vnfd_file_hdls - Rift VNFD XML/YAML file handles Returns: A list of RiftVNFD instances """ vnfd_dict = {} for vnfd_file_hdl in vnfd_file_hdls: - vnfd = RiftVNFD.from_xml_file_hdl(vnfd_file_hdl) - vnfd_dict[vnfd.id] = vnfd - - return vnfd_dict - -def create_vnfd_from_yaml_files(vnfd_file_hdls): - """ Create a list of RiftVNFD instances from xml file handles - - Arguments: - vnfd_file_hdls - Rift VNFD YAML file handles - - Returns: - A list of RiftVNFD instances - """ - vnfd_dict = {} - for vnfd_file_hdl in vnfd_file_hdls: - vnfd = RiftVNFD.from_yaml_file_hdl(vnfd_file_hdl) + if vnfd_file_hdl.name.endswith("yaml") or vnfd_file_hdl.name.endswith("yaml"): + vnfd = RiftVNFD.from_yaml_file_hdl(vnfd_file_hdl) + else: + vnfd = RiftVNFD.from_xml_file_hdl(vnfd_file_hdl) vnfd_dict[vnfd.id] = vnfd return vnfd_dict -def create_nsd_from_xml_file(nsd_file_hdl): - """ Create a list of RiftNSD instances from xml file handles +def create_nsd_from_file(nsd_file_hdl): + """ Create a list of RiftNSD instances from yaml/xml file handles Arguments: - nsd_file_hdls - Rift NSD XML file handles + nsd_file_hdls - Rift NSD XML/yaml file handles Returns: A list of RiftNSD instances """ - nsd = RiftNSD.from_xml_file_hdl(nsd_file_hdl) - return nsd - -def create_nsd_from_yaml_file(nsd_file_hdl): - """ Create a list of RiftNSD instances from yaml file handles - - Arguments: - nsd_file_hdls - Rift NSD XML file handles - - Returns: - A list of RiftNSD instances - """ - nsd = RiftNSD.from_yaml_file_hdl(nsd_file_hdl) + if nsd_file_hdl.name.endswith("yaml") or nsd_file_hdl.name.endswith("yaml"): + nsd = RiftNSD.from_yaml_file_hdl(nsd_file_hdl) + else: + nsd = RiftNSD.from_xml_file_hdl(nsd_file_hdl) return nsd @@ -235,258 +227,137 @@ def convert_vnfd_name(vnfd_name, member_idx): return vnfd_name + "__" + str(member_idx) -def rift2openmano_nsd(rift_nsd, rift_vnfds,openmano_vnfd_ids): - for vnfd_id in rift_nsd.vnfd_ids: - if vnfd_id not in rift_vnfds: - raise VNFNotFoundError("VNF id %s not provided" % vnfd_id) - - openmano = {} - openmano["name"] = rift_nsd.name - openmano["description"] = rift_nsd.description - topology = {} - openmano["topology"] = topology - - topology["nodes"] = {} - for vnfd in rift_nsd.constituent_vnfds: - vnfd_id = vnfd.vnfd_id_ref - rift_vnfd = rift_vnfds[vnfd_id] - member_idx = vnfd.member_vnf_index - openmano_vnfd_id = openmano_vnfd_ids.get(vnfd_id,None) - if openmano_vnfd_id: - topology["nodes"][rift_vnfd.name + "__" + str(member_idx)] = { - "type": "VNF", - "vnf_id": openmano_vnfd_id - } - else: - topology["nodes"][rift_vnfd.name + "__" + str(member_idx)] = { - "type": "VNF", - "VNF model": rift_vnfd.name - } - - for vld in rift_nsd.vlds: - # Openmano has both bridge_net and dataplane_net models for network types - # For now, since we are using openmano in developer mode lets just hardcode - # to bridge_net since it won't matter anyways. - # topology["nodes"][vld.name] = {"type": "network", "model": "bridge_net"} - pass - - topology["connections"] = {} - for vld in rift_nsd.vlds: - - # Create a connections entry for each external VLD - topology["connections"][vld.name] = {} - topology["connections"][vld.name]["nodes"] = [] - - #if vld.vim_network_name: - if True: - if vld.name not in topology["nodes"]: - topology["nodes"][vld.name] = { - "type": "external_network", - "model": vld.name, - } - - # Add the external network to the list of connection points - topology["connections"][vld.name]["nodes"].append( - {vld.name: "0"} - ) - elif vld.provider_network.has_field("physical_network"): - # Add the external datacenter network to the topology - # node list if it isn't already added - ext_net_name = vld.provider_network.physical_network - ext_net_name_with_seg = ext_net_name - if vld.provider_network.has_field("segmentation_id"): - ext_net_name_with_seg += ":{}".format(vld.provider_network.segmentation_id) - - if ext_net_name not in topology["nodes"]: - topology["nodes"][ext_net_name] = { - "type": "external_network", - "model": ext_net_name_with_seg, - } - - # Add the external network to the list of connection points - topology["connections"][vld.name]["nodes"].append( - {ext_net_name: "0"} - ) - - - for vnfd_cp in vld.vnfd_connection_point_ref: - - # Get the RIFT VNF for this external VLD connection point - vnfd = rift_vnfds[vnfd_cp.vnfd_id_ref] - - # For each VNF in this connection, use the same interface name - topology["connections"][vld.name]["type"] = "link" - # Vnf ref is the vnf name with the member_vnf_idx appended - member_idx = vnfd_cp.member_vnf_index_ref - vnf_ref = vnfd.name + "__" + str(member_idx) - topology["connections"][vld.name]["nodes"].append( - { - vnf_ref: vnfd_cp.vnfd_connection_point_ref - } - ) - - return openmano - - -def rift2openmano_vnfd(rift_vnfd): - openmano_vnf = {"vnf":{}} - vnf = openmano_vnf["vnf"] - - vnf["name"] = rift_vnfd.name - vnf["description"] = rift_vnfd.description - - vnf["external-connections"] = [] - - def find_vdu_and_ext_if_by_cp_ref(cp_ref_name): - for vdu in rift_vnfd.vdus: - for ext_if in vdu.external_interface: - if ext_if.vnfd_connection_point_ref == cp_ref_name: - return vdu, ext_if - - raise ValueError("External connection point reference %s not found" % cp_ref_name) - - def find_vdu_and_int_if_by_cp_ref(cp_ref_id): - for vdu in rift_vnfd.vdus: - for int_if in vdu.internal_interface: - if int_if.vdu_internal_connection_point_ref == cp_ref_id: - return vdu, int_if - - raise ValueError("Internal connection point reference %s not found" % cp_ref_id) - - def rift2openmano_if_type(rift_type): - if rift_type == "OM_MGMT": - return "mgmt" - elif rift_type == "VIRTIO": - return "bridge" - else: - return "data" - - # Add all external connections - for cp in rift_vnfd.cps: - # Find the VDU and and external interface for this connection point - vdu, ext_if = find_vdu_and_ext_if_by_cp_ref(cp.name) - connection = { - "name": cp.name, - "type": rift2openmano_if_type(ext_if.virtual_interface.type_yang), - "VNFC": vdu.name, - "local_iface_name": ext_if.name, - "description": "%s iface on VDU %s" % (ext_if.name, vdu.name), - } - - vnf["external-connections"].append(connection) - - # Add all internal networks - for vld in rift_vnfd.internal_vlds: - connection = { - "name": vld.name, - "description": vld.description, - "type": "data", - "elements": [], - } - - # Add the specific VDU connection points - for int_cp_ref in vld.internal_connection_point_ref: - vdu, int_if = find_vdu_and_int_if_by_cp_ref(int_cp_ref) - connection["elements"].append({ - "VNFC": vdu.name, - "local_iface_name": int_if.name, - }) - if "internal-connections" not in vnf: - vnf["internal-connections"] = [] - - vnf["internal-connections"].append(connection) - - # Add VDU's - vnf["VNFC"] = [] - for vdu in rift_vnfd.vdus: - vnfc = { - "name": vdu.name, - "description": vdu.name, - "VNFC image": vdu.image if os.path.isabs(vdu.image) else "/var/images/{}".format(vdu.image), - "numas": [{ - "memory": max(int(vdu.vm_flavor.memory_mb/1024), 1), - "interfaces":[], - }], - "bridge-ifaces": [], - } - - numa_node_policy = vdu.guest_epa.numa_node_policy - if numa_node_policy.has_field("node"): - numa_node = numa_node_policy.node[0] - - if numa_node.has_field("paired_threads"): - if numa_node.paired_threads.has_field("num_paired_threads"): - vnfc["numas"][0]["paired-threads"] = numa_node.paired_threads.num_paired_threads - if len(numa_node.paired_threads.paired_thread_ids) > 0: - vnfc["numas"][0]["paired-threads-id"] = [] - for pair in numa_node.paired_threads.paired_thread_ids: - vnfc["numas"][0]["paired-threads-id"].append( - [pair.thread_a, pair.thread_b] - ) - - else: - if vdu.vm_flavor.has_field("vcpu_count"): - vnfc["numas"][0]["cores"] = max(vdu.vm_flavor.vcpu_count, 1) - - if vdu.has_field("hypervisor_epa"): - vnfc["hypervisor"] = {} - if vdu.hypervisor_epa.has_field("type"): - if vdu.hypervisor_epa.type_yang == "REQUIRE_KVM": - vnfc["hypervisor"]["type"] = "QEMU-kvm" - - if vdu.hypervisor_epa.has_field("version"): - vnfc["hypervisor"]["version"] = vdu.hypervisor_epa.version - - if vdu.has_field("host_epa"): - vnfc["processor"] = {} - if vdu.host_epa.has_field("om_cpu_model_string"): - vnfc["processor"]["model"] = vdu.host_epa.om_cpu_model_string - if vdu.host_epa.has_field("om_cpu_feature"): - vnfc["processor"]["features"] = [] - for feature in vdu.host_epa.om_cpu_feature: - vnfc["processor"]["features"].append(feature) +def rift2openmano_nsd(rift_nsd, rift_vnfds, openmano_vnfd_ids, http_api, rift_vnfd_id=None): + try: + if rift_vnfd_id is None: + for vnfd_id in rift_nsd.vnfd_ids: + if vnfd_id not in rift_vnfds: + raise VNFNotFoundError("VNF id %s not provided" % vnfd_id) + + openmano_nsd_im_body = json.loads(rift_nsd.from_dict()) + openmano_nsd_api_format = { + "nsd:nsd-catalog": { + "nsd": [openmano_nsd_im_body['nsd-catalog']['nsd'][0]] + } + } + + openmano_nsd = http_api.post_nsd_v3(openmano_nsd_api_format) + + return openmano_nsd + + except Exception as e: + logger.error(e) + raise e + +def rift2openmano_vnfd_nsd(rift_nsd, rift_vnfds, openmano_vnfd_ids, http_api, rift_vnfd_id=None): + try: + if rift_vnfd_id not in rift_vnfds: + raise VNFNotFoundError("VNF id %s not provided" % rift_vnfd_id) + # This is the scaling NSD Descriptor. Can use the NSD IM Model. + openmano_nsd_im_body = json.loads(rift_nsd.from_dict()) - if vdu.vm_flavor.has_field("storage_gb"): - vnfc["disk"] = vdu.vm_flavor.storage_gb + openmano_nsd_api_format = { + "nsd:nsd-catalog": { + "nsd": [openmano_nsd_im_body['nsd-catalog']['nsd'][0]] + } + } - vnf["VNFC"].append(vnfc) + openmano_nsd = http_api.post_nsd_v3(openmano_nsd_im_body) + + return openmano_nsd - for int_if in list(vdu.internal_interface) + list(vdu.external_interface): - intf = { - "name": int_if.name, - } - if int_if.virtual_interface.has_field("vpci"): - intf["vpci"] = int_if.virtual_interface.vpci + except Exception as e: + logger.error(e) + raise e - if int_if.virtual_interface.type_yang in ["VIRTIO", "OM_MGMT"]: - vnfc["bridge-ifaces"].append(intf) - elif int_if.virtual_interface.type_yang == "SR-IOV": - intf["bandwidth"] = "10 Gbps" - intf["dedicated"] = "yes:sriov" - vnfc["numas"][0]["interfaces"].append(intf) +def cloud_init(rift_vnfd_id, vdu, project_name='default'): + """ Populate cloud_init with script from + either the inline contents or from the file provided + """ + vnfd_package_store = rift.package.store.VnfdPackageFilesystemStore(logger, project=project_name) + + cloud_init_msg = None + if 'cloud_init' in vdu: + logger.debug("cloud_init script provided inline %s", vdu['cloud_init']) + cloud_init_msg = vdu['cloud_init'] + elif 'cloud_init_file' in vdu: + # Get cloud-init script contents from the file provided in the cloud_init_file param + logger.debug("cloud_init script provided in file %s", vdu['cloud_init_file']) + filename = vdu['cloud_init_file'] + vnfd_package_store.refresh() + stored_package = vnfd_package_store.get_package(rift_vnfd_id) + cloud_init_extractor = rift.package.cloud_init.PackageCloudInitExtractor(logger) + try: + cloud_init_msg = cloud_init_extractor.read_script(stored_package, filename) + except rift.package.cloud_init.CloudInitExtractionError as e: + raise ValueError(e) + else: + logger.debug("VDU translation: cloud-init script not provided") + return - elif int_if.virtual_interface.type_yang == "PCI_PASSTHROUGH": - intf["bandwidth"] = "10 Gbps" - intf["dedicated"] = "yes" - if "interfaces" not in vnfc["numas"][0]: - vnfc["numas"][0]["interfaces"] = [] - vnfc["numas"][0]["interfaces"].append(intf) - else: - raise ValueError("Interface type %s not supported" % int_if.virtual_interface) + logger.debug("Current cloud init msg is {}".format(cloud_init_msg)) + return cloud_init_msg - if int_if.virtual_interface.has_field("bandwidth"): - if int_if.virtual_interface.bandwidth != 0: - bps = int_if.virtual_interface.bandwidth +def config_file_init(rift_vnfd_id, vdu, cfg_file, project_name='default'): + """ Populate config file init with file provided + """ + vnfd_package_store = rift.package.store.VnfdPackageFilesystemStore(logger, project=project_name) + + # Get script contents from the file provided in the cloud_init directory + logger.debug("config file script provided in file {}".format(cfg_file)) + filename = cfg_file + vnfd_package_store.refresh() + stored_package = vnfd_package_store.get_package(rift_vnfd_id) + cloud_init_extractor = rift.package.cloud_init.PackageCloudInitExtractor(logger) + try: + cfg_file_msg = cloud_init_extractor.read_script(stored_package, filename) + except rift.package.cloud_init.CloudInitExtractionError as e: + raise ValueError(e) - # Calculate the bits per second conversion - for x in [('M', 1000000), ('G', 1000000000)]: - if bps/x[1] >= 1: - intf["bandwidth"] = "{} {}bps".format(math.ceil(bps/x[1]), x[0]) + logger.debug("Current config file msg is {}".format(cfg_file_msg)) + return cfg_file_msg +def rift2openmano_vnfd(rift_vnfd, rift_nsd, http_api): + try: + openmano_vnfd_im_body = json.loads(rift_vnfd.from_dict()) + + # All type_yang leafs renamed to type + + + vnfd_dict = openmano_vnfd_im_body['vnfd-catalog']['vnfd'][0] + + if 'vdu' in vnfd_dict: + for vdu in vnfd_dict['vdu']: + if 'cloud_init_file' in vdu: + # Replacing the leaf with the actual contents of the file. + # The RO does not have the ability to read files yet. + vdu['cloud_init_file'] = cloud_init(openmano_vnfd_im_body.id, vdu) + elif 'cloud_init' in vdu: + vdu['cloud_init'] = cloud_init(openmano_vnfd_im_body.id, vdu) + + if 'supplemental_boot_data' in vdu: + if 'config_file' in vdu['supplemental_boot_data']: + for config_file in vdu['supplemental_boot_data']['config_file']: + # Replacing the leaf with the actual contents of the file. + # The RO does not have the ability to read files yet. + config_file['source'] = config_file_init(openmano_vnfd_im_body.id, vdu, config_file['source']) + + openmano_vnfd_api_format = { + "vnfd:vnfd-catalog": { + "vnfd": [vnfd_dict] + } + } + + openmano_vnfd = http_api.post_vnfd_v3(openmano_vnfd_api_format) + + return openmano_vnfd + + except Exception as e: + logger.error(e) + raise e - return openmano_vnf def parse_args(argv=sys.argv[1:]): @@ -507,14 +378,14 @@ def parse_args(argv=sys.argv[1:]): parser.add_argument( '-n', '--nsd-file-hdl', - metavar="nsd_xml_file", + metavar="nsd_file", type=argparse.FileType('r'), help="Rift NSD Descriptor File", ) parser.add_argument( '-v', '--vnfd-file-hdls', - metavar="vnfd_xml_file", + metavar="vnfd_file", action='append', type=argparse.FileType('r'), help="Rift VNFD Descriptor File", @@ -552,20 +423,25 @@ def write_yaml_to_file(name, outdir, desc_dict): def main(argv=sys.argv[1:]): args = parse_args(argv) - nsd = None + openmano_vnfr_ids = dict() + vnf_dict = None if args.vnfd_file_hdls is not None: - vnf_dict = create_vnfd_from_xml_files(args.vnfd_file_hdls) + vnf_dict = create_vnfd_from_files(args.vnfd_file_hdls) - if args.nsd_file_hdl is not None: - nsd = create_nsd_from_xml_file(args.nsd_file_hdl) + for vnfd in vnf_dict: + openmano_vnfr_ids[vnfd] = vnfd - openmano_nsd = rift2openmano_nsd(nsd, vnf_dict) + if args.nsd_file_hdl is not None: + nsd = create_nsd_from_file(args.nsd_file_hdl) + openmano_nsd = rift2openmano_nsd(nsd, vnf_dict, openmano_vnfr_ids) + vnfd_nsd = rift2openmano_vnfd_nsd(nsd, vnf_dict, openmano_vnfr_ids) write_yaml_to_file(openmano_nsd["name"], args.outdir, openmano_nsd) + write_yaml_to_file(vnfd_nsd["name"], args.outdir, vnfd_nsd) for vnf in vnf_dict.values(): - openmano_vnf = rift2openmano_vnfd(vnf) + openmano_vnf = rift2openmano_vnfd(vnf, nsd) write_yaml_to_file(openmano_vnf["vnf"]["name"], args.outdir, openmano_vnf)