Adapt SO to RO schema changes
[osm/SO.git] / models / openmano / python / rift / openmano / rift2openmano.py
1 #!/usr/bin/env python3
2
3 #
4 # Copyright 2016 RIFT.IO Inc
5 #
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at
9 #
10 # http://www.apache.org/licenses/LICENSE-2.0
11 #
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 #
18
19 import argparse
20 import collections
21 import logging
22 import math
23 import os
24 import sys
25 import tempfile
26 import yaml
27
28 import gi
29 gi.require_version('RwYang', '1.0')
30 gi.require_version('RwVnfdYang', '1.0')
31 gi.require_version('RwNsdYang', '1.0')
32
33 from gi.repository import (
34 RwYang,
35 RwVnfdYang,
36 RwNsdYang,
37 )
38
39 import rift.package.store
40 import rift.package.cloud_init
41
42 logger = logging.getLogger("rift2openmano.py")
43
44
45 class VNFNotFoundError(Exception):
46 pass
47
48
49 class RiftNSD(object):
50 model = RwYang.Model.create_libncx()
51 model.load_module('nsd')
52 model.load_module('rw-nsd')
53
54 def __init__(self, descriptor):
55 self._nsd = descriptor
56
57 def __str__(self):
58 return str(self._nsd)
59
60 @property
61 def name(self):
62 return self._nsd.name
63
64 @property
65 def id(self):
66 return self._nsd.id
67
68 @property
69 def vnfd_ids(self):
70 return [c.vnfd_id_ref for c in self._nsd.constituent_vnfd]
71
72 @property
73 def constituent_vnfds(self):
74 return self._nsd.constituent_vnfd
75
76 @property
77 def vlds(self):
78 return self._nsd.vld
79
80 @property
81 def cps(self):
82 return self._nsd.connection_point
83
84 @property
85 def description(self):
86 return self._nsd.description
87
88 @classmethod
89 def from_xml_file_hdl(cls, hdl):
90 hdl.seek(0)
91 descriptor = RwNsdYang.YangData_Nsd_NsdCatalog_Nsd()
92 descriptor.from_xml_v2(RiftNSD.model, hdl.read())
93 return cls(descriptor)
94
95 @classmethod
96 def from_yaml_file_hdl(cls, hdl):
97 hdl.seek(0)
98 descriptor = RwNsdYang.YangData_Nsd_NsdCatalog_Nsd()
99 descriptor.from_yaml(RiftNSD.model, hdl.read())
100 return cls(descriptor)
101
102 @classmethod
103 def from_dict(cls, nsd_dict):
104 descriptor = RwNsdYang.YangData_Nsd_NsdCatalog_Nsd.from_dict(nsd_dict)
105 return cls(descriptor)
106
107
108 class RiftVNFD(object):
109 model = RwYang.Model.create_libncx()
110 model.load_module('vnfd')
111 model.load_module('rw-vnfd')
112
113 def __init__(self, descriptor):
114 self._vnfd = descriptor
115
116 def __str__(self):
117 return str(self._vnfd)
118
119 @property
120 def id(self):
121 return self._vnfd.id
122
123 @property
124 def name(self):
125 return self._vnfd.name
126
127 @property
128 def description(self):
129 return self._vnfd.description
130
131 @property
132 def cps(self):
133 return self._vnfd.connection_point
134
135 @property
136 def vdus(self):
137 return self._vnfd.vdu
138
139 @property
140 def internal_vlds(self):
141 return self._vnfd.internal_vld
142
143 @classmethod
144 def from_xml_file_hdl(cls, hdl):
145 hdl.seek(0)
146 descriptor = RwVnfdYang.YangData_Vnfd_VnfdCatalog_Vnfd()
147 descriptor.from_xml_v2(RiftVNFD.model, hdl.read())
148 return cls(descriptor)
149
150 @classmethod
151 def from_yaml_file_hdl(cls, hdl):
152 hdl.seek(0)
153 descriptor = RwVnfdYang.YangData_Vnfd_VnfdCatalog_Vnfd()
154 descriptor.from_yaml(RiftVNFD.model, hdl.read())
155 return cls(descriptor)
156
157 @classmethod
158 def from_dict(cls, vnfd_dict):
159 descriptor = RwVnfdYang.YangData_Vnfd_VnfdCatalog_Vnfd.from_dict(vnfd_dict)
160 return cls(descriptor)
161
162
163 def is_writable_directory(dir_path):
164 """ Returns True if dir_path is writable, False otherwise
165
166 Arguments:
167 dir_path - A directory path
168 """
169 if not os.path.exists(dir_path):
170 raise ValueError("Directory does not exist: %s", dir_path)
171
172 try:
173 testfile = tempfile.TemporaryFile(dir=dir_path)
174 testfile.close()
175 except OSError:
176 return False
177
178 return True
179
180
181 def create_vnfd_from_files(vnfd_file_hdls):
182 """ Create a list of RiftVNFD instances from xml/yaml file handles
183
184 Arguments:
185 vnfd_file_hdls - Rift VNFD XML/YAML file handles
186
187 Returns:
188 A list of RiftVNFD instances
189 """
190 vnfd_dict = {}
191 for vnfd_file_hdl in vnfd_file_hdls:
192 if vnfd_file_hdl.name.endswith("yaml") or vnfd_file_hdl.name.endswith("yaml"):
193 vnfd = RiftVNFD.from_yaml_file_hdl(vnfd_file_hdl)
194 else:
195 vnfd = RiftVNFD.from_xml_file_hdl(vnfd_file_hdl)
196 vnfd_dict[vnfd.id] = vnfd
197
198 return vnfd_dict
199
200
201 def create_nsd_from_file(nsd_file_hdl):
202 """ Create a list of RiftNSD instances from yaml/xml file handles
203
204 Arguments:
205 nsd_file_hdls - Rift NSD XML/yaml file handles
206
207 Returns:
208 A list of RiftNSD instances
209 """
210 if nsd_file_hdl.name.endswith("yaml") or nsd_file_hdl.name.endswith("yaml"):
211 nsd = RiftNSD.from_yaml_file_hdl(nsd_file_hdl)
212 else:
213 nsd = RiftNSD.from_xml_file_hdl(nsd_file_hdl)
214 return nsd
215
216
217 def ddict():
218 return collections.defaultdict(dict)
219
220 def convert_vnfd_name(vnfd_name, member_idx):
221 return vnfd_name + "__" + str(member_idx)
222
223
224 def rift2openmano_nsd(rift_nsd, rift_vnfds, openmano_vnfd_ids):
225 for vnfd_id in rift_nsd.vnfd_ids:
226 if vnfd_id not in rift_vnfds:
227 raise VNFNotFoundError("VNF id %s not provided" % vnfd_id)
228
229 openmano = {}
230 openmano["name"] = rift_nsd.name
231 openmano["description"] = rift_nsd.description
232 topology = {}
233 openmano["topology"] = topology
234
235 topology["nodes"] = {}
236 for vnfd in rift_nsd.constituent_vnfds:
237 vnfd_id = vnfd.vnfd_id_ref
238 rift_vnfd = rift_vnfds[vnfd_id]
239 member_idx = vnfd.member_vnf_index
240 openmano_vnfd_id = openmano_vnfd_ids.get(vnfd_id,None)
241 if openmano_vnfd_id:
242 topology["nodes"][rift_vnfd.name + "__" + str(member_idx)] = {
243 "type": "VNF",
244 "vnf_id": openmano_vnfd_id
245 }
246 else:
247 topology["nodes"][rift_vnfd.name + "__" + str(member_idx)] = {
248 "type": "VNF",
249 "VNF model": rift_vnfd.name
250 }
251
252 for vld in rift_nsd.vlds:
253 # Openmano has both bridge_net and dataplane_net models for network types
254 # For now, since we are using openmano in developer mode lets just hardcode
255 # to bridge_net since it won't matter anyways.
256 # topology["nodes"][vld.name] = {"type": "network", "model": "bridge_net"}
257 pass
258
259 topology["connections"] = {}
260 for vld in rift_nsd.vlds:
261
262 # Create a connections entry for each external VLD
263 topology["connections"][vld.name] = {}
264 topology["connections"][vld.name]["nodes"] = []
265
266 #if vld.vim_network_name:
267 if True:
268 if vld.name not in topology["nodes"]:
269 topology["nodes"][vld.name] = {
270 "type": "external_network",
271 "model": vld.name,
272 }
273
274 # Add the external network to the list of connection points
275 topology["connections"][vld.name]["nodes"].append(
276 {vld.name: "0"}
277 )
278 elif vld.provider_network.has_field("physical_network"):
279 # Add the external datacenter network to the topology
280 # node list if it isn't already added
281 ext_net_name = vld.provider_network.physical_network
282 ext_net_name_with_seg = ext_net_name
283 if vld.provider_network.has_field("segmentation_id"):
284 ext_net_name_with_seg += ":{}".format(vld.provider_network.segmentation_id)
285
286 if ext_net_name not in topology["nodes"]:
287 topology["nodes"][ext_net_name] = {
288 "type": "external_network",
289 "model": ext_net_name_with_seg,
290 }
291
292 # Add the external network to the list of connection points
293 topology["connections"][vld.name]["nodes"].append(
294 {ext_net_name: "0"}
295 )
296
297
298 for vnfd_cp in vld.vnfd_connection_point_ref:
299
300 # Get the RIFT VNF for this external VLD connection point
301 vnfd = rift_vnfds[vnfd_cp.vnfd_id_ref]
302
303 # For each VNF in this connection, use the same interface name
304 topology["connections"][vld.name]["type"] = "link"
305 # Vnf ref is the vnf name with the member_vnf_idx appended
306 member_idx = vnfd_cp.member_vnf_index_ref
307 vnf_ref = vnfd.name + "__" + str(member_idx)
308 topology["connections"][vld.name]["nodes"].append(
309 {
310 vnf_ref: vnfd_cp.vnfd_connection_point_ref
311 }
312 )
313
314 return openmano
315
316 def cloud_init(rift_vnfd_id, vdu):
317 """ Populate cloud_init with script from
318 either the inline contents or from the file provided
319 """
320 vnfd_package_store = rift.package.store.VnfdPackageFilesystemStore(logger)
321
322 cloud_init_msg = None
323 if vdu.cloud_init is not None:
324 logger.debug("cloud_init script provided inline %s", vdu.cloud_init)
325 cloud_init_msg = vdu.cloud_init
326 elif vdu.cloud_init_file is not None:
327 # Get cloud-init script contents from the file provided in the cloud_init_file param
328 logger.debug("cloud_init script provided in file %s", vdu.cloud_init_file)
329 filename = vdu.cloud_init_file
330 vnfd_package_store.refresh()
331 stored_package = vnfd_package_store.get_package(rift_vnfd_id)
332 cloud_init_extractor = rift.package.cloud_init.PackageCloudInitExtractor(logger)
333 try:
334 cloud_init_msg = cloud_init_extractor.read_script(stored_package, filename)
335 except rift.package.cloud_init.CloudInitExtractionError as e:
336 raise ValueError(e)
337 else:
338 logger.debug("VDU translation: cloud-init script not provided")
339 return
340
341 logger.debug("Current cloud init msg is {}".format(cloud_init_msg))
342 return cloud_init_msg
343
344 def rift2openmano_vnfd(rift_vnfd, rift_nsd):
345 openmano_vnf = {"vnf":{}}
346 vnf = openmano_vnf["vnf"]
347
348 vnf["name"] = rift_vnfd.name
349 vnf["description"] = rift_vnfd.description
350
351 vnf["external-connections"] = []
352
353 def find_vdu_and_ext_if_by_cp_ref(cp_ref_name):
354 for vdu in rift_vnfd.vdus:
355 for ext_if in vdu.external_interface:
356 if ext_if.vnfd_connection_point_ref == cp_ref_name:
357 return vdu, ext_if
358
359 raise ValueError("External connection point reference %s not found" % cp_ref_name)
360
361 def find_vdu_and_int_if_by_cp_ref(cp_ref_id):
362 for vdu in rift_vnfd.vdus:
363 for int_if in vdu.internal_interface:
364 if int_if.vdu_internal_connection_point_ref == cp_ref_id:
365 return vdu, int_if
366
367 raise ValueError("Internal connection point reference %s not found" % cp_ref_id)
368
369 def rift2openmano_if_type(ext_if):
370
371 cp_ref_name = ext_if.vnfd_connection_point_ref
372 for vld in rift_nsd.vlds:
373
374 # if it is an explicit mgmt_network then check if the given
375 # cp_ref is a part of it
376 if not vld.mgmt_network:
377 continue
378
379 for vld_cp in vld.vnfd_connection_point_ref:
380 if vld_cp.vnfd_connection_point_ref == cp_ref_name:
381 return "mgmt"
382
383
384 rift_type = ext_if.virtual_interface.type_yang
385 # Retaining it for backward compatibility!
386 if rift_type == "OM_MGMT":
387 return "mgmt"
388 elif rift_type == "VIRTIO" or rift_type == "E1000":
389 return "bridge"
390 else:
391 return "data"
392
393 def rift2openmano_vif(rift_type):
394 if rift_type == "VIRTIO":
395 return "virtio"
396 elif rift_type == "E1000":
397 return "e1000"
398 else:
399 raise ValueError("VDU Virtual Interface type {} not supported".format(rift_type))
400
401 # Add all external connections
402 for cp in rift_vnfd.cps:
403 # Find the VDU and and external interface for this connection point
404 vdu, ext_if = find_vdu_and_ext_if_by_cp_ref(cp.name)
405 connection = {
406 "name": cp.name,
407 "type": rift2openmano_if_type(ext_if),
408 "VNFC": vdu.name,
409 "local_iface_name": ext_if.name,
410 "description": "%s iface on VDU %s" % (ext_if.name, vdu.name),
411 }
412
413 vnf["external-connections"].append(connection)
414
415 # Add all internal networks
416 for vld in rift_vnfd.internal_vlds:
417 connection = {
418 "name": vld.name,
419 "description": vld.description,
420 "type": "data",
421 "elements": [],
422 }
423
424 # Add the specific VDU connection points
425 for int_cp in vld.internal_connection_point:
426 vdu, int_if = find_vdu_and_int_if_by_cp_ref(int_cp.id_ref)
427 connection["elements"].append({
428 "VNFC": vdu.name,
429 "local_iface_name": int_if.name,
430 })
431 if "internal-connections" not in vnf:
432 vnf["internal-connections"] = []
433
434 vnf["internal-connections"].append(connection)
435
436 # Add VDU's
437 vnf["VNFC"] = []
438 for vdu in rift_vnfd.vdus:
439 vnfc = {
440 "name": vdu.name,
441 "description": vdu.name,
442 "bridge-ifaces": [],
443 }
444
445 if vdu.vm_flavor.has_field("storage_gb") and vdu.vm_flavor.storage_gb:
446 vnfc["disk"] = vdu.vm_flavor.storage_gb
447
448 if vdu.has_field("image"):
449 if os.path.isabs(vdu.image):
450 vnfc["VNFC image"] = vdu.image
451 else:
452 vnfc["image name"] = vdu.image
453 if vdu.has_field("image_checksum"):
454 vnfc["image checksum"] = vdu.image_checksum
455
456 dedicated_int = False
457 for intf in list(vdu.internal_interface) + list(vdu.external_interface):
458 if intf.virtual_interface.type_yang in ["SR_IOV", "PCI_PASSTHROUGH"]:
459 dedicated_int = True
460 if vdu.guest_epa.has_field("numa_node_policy") or dedicated_int:
461 vnfc["numas"] = [{
462 "memory": max(int(vdu.vm_flavor.memory_mb/1024), 1),
463 "interfaces":[],
464 }]
465 numa_node_policy = vdu.guest_epa.numa_node_policy
466 if numa_node_policy.has_field("node"):
467 numa_node = numa_node_policy.node[0]
468
469 if numa_node.has_field("paired_threads"):
470 if numa_node.paired_threads.has_field("num_paired_threads"):
471 vnfc["numas"][0]["paired-threads"] = numa_node.paired_threads.num_paired_threads
472 if len(numa_node.paired_threads.paired_thread_ids) > 0:
473 vnfc["numas"][0]["paired-threads-id"] = []
474 for pair in numa_node.paired_threads.paired_thread_ids:
475 vnfc["numas"][0]["paired-threads-id"].append(
476 [pair.thread_a, pair.thread_b]
477 )
478
479 else:
480 if vdu.vm_flavor.has_field("vcpu_count"):
481 vnfc["numas"][0]["cores"] = max(vdu.vm_flavor.vcpu_count, 1)
482
483 else:
484 if vdu.vm_flavor.has_field("vcpu_count") and vdu.vm_flavor.vcpu_count:
485 vnfc["vcpus"] = vdu.vm_flavor.vcpu_count
486
487 if vdu.vm_flavor.has_field("memory_mb") and vdu.vm_flavor.memory_mb:
488 vnfc["ram"] = vdu.vm_flavor.memory_mb
489
490
491 if vdu.has_field("hypervisor_epa"):
492 vnfc["hypervisor"] = {}
493 if vdu.hypervisor_epa.has_field("type"):
494 if vdu.hypervisor_epa.type_yang == "REQUIRE_KVM":
495 vnfc["hypervisor"]["type"] = "QEMU-kvm"
496
497 if vdu.hypervisor_epa.has_field("version"):
498 vnfc["hypervisor"]["version"] = vdu.hypervisor_epa.version
499
500 if vdu.has_field("host_epa"):
501 vnfc["processor"] = {}
502 if vdu.host_epa.has_field("om_cpu_model_string"):
503 vnfc["processor"]["model"] = vdu.host_epa.om_cpu_model_string
504 if vdu.host_epa.has_field("om_cpu_feature"):
505 vnfc["processor"]["features"] = []
506 for feature in vdu.host_epa.om_cpu_feature:
507 vnfc["processor"]["features"].append(feature.feature)
508
509 if vdu.has_field("volumes"):
510 vnfc["devices"] = []
511 # Sort volumes as device-list is implictly ordered by Openmano
512 newvollist = sorted(vdu.volumes, key=lambda k: k.name)
513 for iter_num, volume in enumerate(newvollist):
514 if iter_num == 0:
515 # Convert the first volume to vnfc.image
516 if os.path.isabs(volume.image):
517 vnfc["VNFC image"] = volume.image
518 else:
519 vnfc["image name"] = volume.image
520 if volume.has_field("image_checksum"):
521 vnfc["image checksum"] = volume.image_checksum
522 else:
523 # Add Openmano devices
524 device = {}
525 device["type"] = volume.device_type
526 device["image"] = volume.image
527 vnfc["devices"].append(device)
528
529 vnfc_boot_data_init = False
530 if vdu.has_field("cloud_init") or vdu.has_field("cloud_init_file"):
531 vnfc['boot-data'] = dict()
532 vnfc_boot_data_init = True
533 vnfc['boot-data']['user-data'] = cloud_init(rift_vnfd.id, vdu)
534
535 if vdu.has_field("supplemental_boot_data"):
536 if vdu.supplemental_boot_data.has_field('boot_data_drive'):
537 if vdu.supplemental_boot_data.boot_data_drive is True:
538 if vnfc_boot_data_init is False:
539 vnfc['boot-data'] = dict()
540 vnfc_boot_data_init = True
541 vnfc['boot-data']['boot-data-drive'] = vdu.supplemental_boot_data.boot_data_drive
542
543 vnf["VNFC"].append(vnfc)
544
545 for int_if in list(vdu.internal_interface) + list(vdu.external_interface):
546 intf = {
547 "name": int_if.name,
548 }
549 if int_if.virtual_interface.has_field("vpci"):
550 intf["vpci"] = int_if.virtual_interface.vpci
551
552 if int_if.virtual_interface.type_yang in ["VIRTIO", "E1000"]:
553 intf["model"] = rift2openmano_vif(int_if.virtual_interface.type_yang)
554 vnfc["bridge-ifaces"].append(intf)
555
556 elif int_if.virtual_interface.type_yang in ["OM_MGMT"]:
557 vnfc["bridge-ifaces"].append(intf)
558
559 elif int_if.virtual_interface.type_yang == "SR_IOV":
560 intf["bandwidth"] = "10 Gbps"
561 intf["dedicated"] = "no"
562 vnfc["numas"][0]["interfaces"].append(intf)
563
564 elif int_if.virtual_interface.type_yang == "PCI_PASSTHROUGH":
565 intf["bandwidth"] = "10 Gbps"
566 intf["dedicated"] = "yes"
567 if "interfaces" not in vnfc["numas"][0]:
568 vnfc["numas"][0]["interfaces"] = []
569 vnfc["numas"][0]["interfaces"].append(intf)
570 else:
571 raise ValueError("Interface type %s not supported" % int_if.virtual_interface)
572
573 if int_if.virtual_interface.has_field("bandwidth"):
574 if int_if.virtual_interface.bandwidth != 0:
575 bps = int_if.virtual_interface.bandwidth
576
577 # Calculate the bits per second conversion
578 for x in [('M', 1000000), ('G', 1000000000)]:
579 if bps/x[1] >= 1:
580 intf["bandwidth"] = "{} {}bps".format(math.ceil(bps/x[1]), x[0])
581
582
583 return openmano_vnf
584
585
586 def parse_args(argv=sys.argv[1:]):
587 """ Parse the command line arguments
588
589 Arguments:
590 arv - The list of arguments to parse
591
592 Returns:
593 Argparse Namespace instance
594 """
595 parser = argparse.ArgumentParser()
596 parser.add_argument(
597 '-o', '--outdir',
598 default='-',
599 help="Directory to output converted descriptors. Default is stdout",
600 )
601
602 parser.add_argument(
603 '-n', '--nsd-file-hdl',
604 metavar="nsd_file",
605 type=argparse.FileType('r'),
606 help="Rift NSD Descriptor File",
607 )
608
609 parser.add_argument(
610 '-v', '--vnfd-file-hdls',
611 metavar="vnfd_file",
612 action='append',
613 type=argparse.FileType('r'),
614 help="Rift VNFD Descriptor File",
615 )
616
617 args = parser.parse_args(argv)
618
619 if not os.path.exists(args.outdir):
620 os.makedirs(args.outdir)
621
622 if not is_writable_directory(args.outdir):
623 logging.error("Directory %s is not writable", args.outdir)
624 sys.exit(1)
625
626 return args
627
628
629 def write_yaml_to_file(name, outdir, desc_dict):
630 file_name = "%s.yaml" % name
631 yaml_str = yaml.dump(desc_dict)
632 if outdir == "-":
633 sys.stdout.write(yaml_str)
634 return
635
636 file_path = os.path.join(outdir, file_name)
637 dir_path = os.path.dirname(file_path)
638 if not os.path.exists(dir_path):
639 os.makedirs(dir_path)
640
641 with open(file_path, "w") as hdl:
642 hdl.write(yaml_str)
643
644 logger.info("Wrote descriptor to %s", file_path)
645
646
647 def main(argv=sys.argv[1:]):
648 args = parse_args(argv)
649
650 nsd = None
651 openmano_vnfr_ids = dict()
652 vnf_dict = None
653 if args.vnfd_file_hdls is not None:
654 vnf_dict = create_vnfd_from_files(args.vnfd_file_hdls)
655
656 for vnfd in vnf_dict:
657 openmano_vnfr_ids[vnfd] = vnfd
658
659 if args.nsd_file_hdl is not None:
660 nsd = create_nsd_from_file(args.nsd_file_hdl)
661
662 openmano_nsd = rift2openmano_nsd(nsd, vnf_dict, openmano_vnfr_ids)
663
664 write_yaml_to_file(openmano_nsd["name"], args.outdir, openmano_nsd)
665
666 for vnf in vnf_dict.values():
667 openmano_vnf = rift2openmano_vnfd(vnf, nsd)
668 write_yaml_to_file(openmano_vnf["vnf"]["name"], args.outdir, openmano_vnf)
669
670
671 if __name__ == "__main__":
672 logging.basicConfig(level=logging.WARNING)
673 main()