Volume ref support
[osm/SO.git] / rwcal / plugins / vala / rwcal_openstack / rift / rwcal / openstack / utils / compute.py
1 #!/usr/bin/python
2
3 #
4 # Copyright 2017 RIFT.IO Inc
5 #
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at
9 #
10 # http://www.apache.org/licenses/LICENSE-2.0
11 #
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 #
18 import uuid
19 import gi
20 gi.require_version('RwcalYang', '1.0')
21 from gi.repository import RwcalYang
22
23
24 class ImageValidateError(Exception):
25 pass
26
27 class VolumeValidateError(Exception):
28 pass
29
30 class AffinityGroupError(Exception):
31 pass
32
33
34 class ComputeUtils(object):
35 """
36 Utility class for compute operations
37 """
38 epa_types = ['vm_flavor',
39 'guest_epa',
40 'host_epa',
41 'host_aggregate',
42 'hypervisor_epa',
43 'vswitch_epa']
44 def __init__(self, driver):
45 """
46 Constructor for class
47 Arguments:
48 driver: object of OpenstackDriver()
49 """
50 self._driver = driver
51 self.log = driver.log
52
53 @property
54 def driver(self):
55 return self._driver
56
57 def search_vdu_flavor(self, vdu_params):
58 """
59 Function to search a matching flavor for VDU instantiation
60 from already existing flavors
61
62 Arguments:
63 vdu_params: Protobuf GI object RwcalYang.VDUInitParams()
64
65 Returns:
66 flavor_id(string): Flavor id for VDU instantiation
67 None if no flavor could be found
68 """
69 kwargs = { 'vcpus': vdu_params.vm_flavor.vcpu_count,
70 'ram' : vdu_params.vm_flavor.memory_mb,
71 'disk' : vdu_params.vm_flavor.storage_gb,}
72
73 flavors = self.driver.nova_flavor_find(**kwargs)
74 flavor_list = list()
75 for flv in flavors:
76 flavor_list.append(self.driver.utils.flavor.parse_flavor_info(flv))
77
78 flavor_id = self.driver.utils.flavor.match_resource_flavor(vdu_params, flavor_list)
79 return flavor_id
80
81 def select_vdu_flavor(self, vdu_params):
82 """
83 This function attempts to find a pre-existing flavor matching required
84 parameters for VDU instantiation. If no such flavor is found, a new one
85 is created.
86
87 Arguments:
88 vdu_params: Protobuf GI object RwcalYang.VDUInitParams()
89
90 Returns:
91 flavor_id(string): Flavor id for VDU instantiation
92 """
93 flavor_id = self.search_vdu_flavor(vdu_params)
94 if flavor_id is not None:
95 self.log.info("Found flavor with id: %s matching requirements for VDU: %s",
96 flavor_id, vdu_params.name)
97 return flavor_id
98
99 flavor = RwcalYang.FlavorInfoItem()
100 flavor.name = str(uuid.uuid4())
101
102 epa_dict = { k: v for k, v in vdu_params.as_dict().items()
103 if k in ComputeUtils.epa_types }
104
105 flavor.from_dict(epa_dict)
106
107 flavor_id = self.driver.nova_flavor_create(name = flavor.name,
108 ram = flavor.vm_flavor.memory_mb,
109 vcpus = flavor.vm_flavor.vcpu_count,
110 disk = flavor.vm_flavor.storage_gb,
111 epa_specs = self.driver.utils.flavor.get_extra_specs(flavor))
112 return flavor_id
113
114 def make_vdu_flavor_args(self, vdu_params):
115 """
116 Creates flavor related arguments for VDU operation
117 Arguments:
118 vdu_params: Protobuf GI object RwcalYang.VDUInitParams()
119
120 Returns:
121 A dictionary {'flavor_id': <flavor-id>}
122 """
123 return {'flavor_id': self.select_vdu_flavor(vdu_params)}
124
125
126 def make_vdu_image_args(self, vdu_params):
127 """
128 Creates image related arguments for VDU operation
129 Arguments:
130 vdu_params: Protobuf GI object RwcalYang.VDUInitParams()
131
132 Returns:
133 A dictionary {'image_id': <image-id>}
134
135 """
136 kwargs = dict()
137 if vdu_params.has_field('image_name'):
138 kwargs['image_id'] = self.resolve_image_n_validate(vdu_params.image_name,
139 vdu_params.image_checksum)
140 elif vdu_params.has_field('image_id'):
141 kwargs['image_id'] = vdu_params.image_id
142
143 return kwargs
144
145 def resolve_image_n_validate(self, image_name, checksum = None):
146 """
147 Resolve the image_name to image-object by matching image_name and checksum
148
149 Arguments:
150 image_name (string): Name of image
151 checksums (string): Checksum associated with image
152
153 Raises ImageValidateError in case of Errors
154 """
155 image_info = [ i for i in self.driver._glance_image_list if i['name'] == image_name]
156
157 if not image_info:
158 self.log.error("No image with name: %s found", image_name)
159 raise ImageValidateError("No image with name %s found" %(image_name))
160
161 for image in image_info:
162 if 'status' not in image or image['status'] != 'active':
163 self.log.error("Image %s not in active state. Current state: %s",
164 image_name, image['status'])
165 raise ImageValidateError("Image with name %s found in incorrect (%s) state"
166 %(image_name, image['status']))
167 if not checksum or checksum == image['checksum']:
168 break
169 else:
170 self.log.info("No image found with matching name: %s and checksum: %s",
171 image_name, checksum)
172 raise ImageValidateError("No image found with matching name: %s and checksum: %s"
173 %(image_name, checksum))
174 return image['id']
175
176 def resolve_volume_n_validate(self, volume_ref):
177 """
178 Resolve the volume reference
179
180 Arguments:
181 volume_ref (string): Name of volume reference
182
183 Raises VolumeValidateError in case of Errors
184 """
185
186 for vol in self.driver._cinder_volume_list:
187 voldict = vol.to_dict()
188 if voldict['display_name'] == volume_ref:
189 if 'status' in voldict and voldict['status'] == 'available':
190 return voldict['id']
191 else:
192 self.log.error("Volume %s not in available state. Current state: %s",
193 volume_ref, voldict['status'])
194 raise VolumeValidateError("Volume with name %s found in incorrect (%s) state"
195 %(volume_ref, vol['status']))
196
197 self.log.info("No volume found with matching name: %s ", volume_ref)
198 raise VolumeValidateError("No volume found with matching name: %s " %(volume_ref))
199
200 def make_vdu_volume_args(self, volume, vdu_params):
201 """
202 Arguments:
203 volume: Protobuf GI object RwcalYang.VDUInitParams_Volumes()
204 vdu_params: Protobuf GI object RwcalYang.VDUInitParams()
205
206 Returns:
207 A dictionary required to create volume for VDU
208
209 Raises VolumeValidateError in case of Errors
210 """
211 kwargs = dict()
212
213 kwargs['boot_index'] = volume.boot_priority
214 if volume.has_field("image"):
215 # Support image->volume
216 kwargs['source_type'] = "image"
217 kwargs['uuid'] = self.resolve_image_n_validate(volume.image, volume.image_checksum)
218 kwargs['delete_on_termination'] = True
219 elif "volume_ref" in volume:
220 # Support volume-ref->volume (only ref)
221 kwargs['source_type'] = "volume"
222 kwargs['uuid'] = self.resolve_volume_n_validate(volume.volume_ref)
223 kwargs['delete_on_termination'] = False
224 else:
225 # Support blank->volume
226 kwargs['source_type'] = "blank"
227 kwargs['delete_on_termination'] = True
228 kwargs['device_name'] = volume.name
229 kwargs['destination_type'] = "volume"
230 kwargs['volume_size'] = volume.size
231
232 if volume.has_field('device_type'):
233 if volume.device_type in ['cdrom', 'disk']:
234 kwargs['device_type'] = volume.device_type
235 else:
236 self.log.error("Unsupported device_type <%s> found for volume: %s",
237 volume.device_type, volume.name)
238 raise VolumeValidateError("Unsupported device_type <%s> found for volume: %s"
239 %(volume.device_type, volume.name))
240 else:
241 self.log.error("Mandatory field <device_type> not specified for volume: %s",
242 volume.name)
243 raise VolumeValidateError("Mandatory field <device_type> not specified for volume: %s"
244 %(volume.name))
245
246 if volume.has_field('device_bus'):
247 if volume.device_bus in ['ide', 'virtio', 'scsi']:
248 kwargs['disk_bus'] = volume.device_bus
249 else:
250 self.log.error("Unsupported device_bus <%s> found for volume: %s",
251 volume.device_bus, volume.name)
252 raise VolumeValidateError("Unsupported device_bus <%s> found for volume: %s"
253 %(volume.device_bus, volume.name))
254 else:
255 self.log.error("Mandatory field <device_bus> not specified for volume: %s",
256 volume.name)
257 raise VolumeValidateError("Mandatory field <device_bus> not specified for volume: %s"
258 %(volume.name))
259
260 return kwargs
261
262 def make_vdu_storage_args(self, vdu_params):
263 """
264 Creates volume related arguments for VDU operation
265
266 Arguments:
267 vdu_params: Protobuf GI object RwcalYang.VDUInitParams()
268
269 Returns:
270 A dictionary required for volumes creation for VDU instantiation
271 """
272 kwargs = dict()
273 if vdu_params.has_field('volumes'):
274 kwargs['block_device_mapping_v2'] = list()
275 # Ignore top-level image
276 kwargs['image_id'] = ""
277 for volume in vdu_params.volumes:
278 kwargs['block_device_mapping_v2'].append(self.make_vdu_volume_args(volume, vdu_params))
279 return kwargs
280
281 def make_vdu_network_args(self, vdu_params):
282 """
283 Creates VDU network related arguments for VDU operation
284 Arguments:
285 vdu_params: Protobuf GI object RwcalYang.VDUInitParams()
286
287 Returns:
288 A dictionary {'port_list' : [ports], 'network_list': [networks]}
289
290 """
291 kwargs = dict()
292 kwargs['port_list'], kwargs['network_list'] = self.driver.utils.network.setup_vdu_networking(vdu_params)
293 return kwargs
294
295
296 def make_vdu_boot_config_args(self, vdu_params):
297 """
298 Creates VDU boot config related arguments for VDU operation
299 Arguments:
300 vdu_params: Protobuf GI object RwcalYang.VDUInitParams()
301
302 Returns:
303 A dictionary {
304 'userdata' : <cloud-init> ,
305 'config_drive': True/False,
306 'files' : [ file name ],
307 'metadata' : <metadata string>
308 }
309 """
310 kwargs = dict()
311 metadata = dict()
312
313 if vdu_params.has_field('node_id'):
314 metadata['rift_node_id'] = vdu_params.node_id
315 kwargs['metadata'] = metadata
316
317 if vdu_params.has_field('vdu_init') and vdu_params.vdu_init.has_field('userdata'):
318 kwargs['userdata'] = vdu_params.vdu_init.userdata
319 else:
320 kwargs['userdata'] = ''
321
322 if not vdu_params.has_field('supplemental_boot_data'):
323 return kwargs
324
325 if vdu_params.supplemental_boot_data.has_field('config_file'):
326 files = dict()
327 for cf in vdu_params.supplemental_boot_data.config_file:
328 files[cf.dest] = cf.source
329 kwargs['files'] = files
330
331 if vdu_params.supplemental_boot_data.has_field('boot_data_drive'):
332 kwargs['config_drive'] = vdu_params.supplemental_boot_data.boot_data_drive
333 else:
334 kwargs['config_drive'] = False
335
336 try:
337 # Rift model only
338 if vdu_params.supplemental_boot_data.has_field('custom_meta_data'):
339 metadata = dict()
340 for cm in vdu_params.supplemental_boot_data.custom_meta_data:
341 metadata[cm.name] = cm.value
342 kwargs['metadata'] = metadata
343 except Exception as e:
344 pass
345
346 return kwargs
347
348 def _select_affinity_group(self, group_name):
349 """
350 Selects the affinity group based on name and return its id
351 Arguments:
352 group_name (string): Name of the Affinity/Anti-Affinity group
353 Returns:
354 Id of the matching group
355
356 Raises exception AffinityGroupError if no matching group is found
357 """
358 groups = [g['id'] for g in self.driver._nova_affinity_group if g['name'] == group_name]
359 if not groups:
360 self.log.error("No affinity/anti-affinity group with name: %s found", group_name)
361 raise AffinityGroupError("No affinity/anti-affinity group with name: %s found" %(group_name))
362 return groups[0]
363
364
365 def make_vdu_server_placement_args(self, vdu_params):
366 """
367 Function to create kwargs required for nova server placement
368
369 Arguments:
370 vdu_params: Protobuf GI object RwcalYang.VDUInitParams()
371
372 Returns:
373 A dictionary { 'availability_zone' : < Zone >, 'scheduler_hints': <group-id> }
374
375 """
376 kwargs = dict()
377
378 if vdu_params.has_field('availability_zone') \
379 and vdu_params.availability_zone.has_field('name'):
380 kwargs['availability_zone'] = vdu_params.availability_zone
381
382 if vdu_params.has_field('server_group'):
383 kwargs['scheduler_hints'] = {
384 'group': self._select_affinity_group(vdu_params.server_group)
385 }
386 return kwargs
387
388 def make_vdu_server_security_args(self, vdu_params, account):
389 """
390 Function to create kwargs required for nova security group
391
392 Arguments:
393 vdu_params: Protobuf GI object RwcalYang.VDUInitParams()
394 account: Protobuf GI object RwcalYang.CloudAccount()
395
396 Returns:
397 A dictionary {'security_groups' : < group > }
398 """
399 kwargs = dict()
400 if account.openstack.security_groups:
401 kwargs['security_groups'] = account.openstack.security_groups
402 return kwargs
403
404
405 def make_vdu_create_args(self, vdu_params, account):
406 """
407 Function to create kwargs required for nova_server_create API
408
409 Arguments:
410 vdu_params: Protobuf GI object RwcalYang.VDUInitParams()
411 account: Protobuf GI object RwcalYang.CloudAccount()
412
413 Returns:
414 A kwargs dictionary for VDU create operation
415 """
416 kwargs = dict()
417
418 kwargs['name'] = vdu_params.name
419
420 kwargs.update(self.make_vdu_flavor_args(vdu_params))
421 kwargs.update(self.make_vdu_storage_args(vdu_params))
422 kwargs.update(self.make_vdu_image_args(vdu_params))
423 kwargs.update(self.make_vdu_network_args(vdu_params))
424 kwargs.update(self.make_vdu_boot_config_args(vdu_params))
425 kwargs.update(self.make_vdu_server_placement_args(vdu_params))
426 kwargs.update(self.make_vdu_server_security_args(vdu_params, account))
427 return kwargs
428
429
430 def _parse_vdu_mgmt_address_info(self, vm_info):
431 """
432 Get management_ip and public_ip for VDU
433
434 Arguments:
435 vm_info : A dictionary object return by novaclient library listing VM attributes
436
437 Returns:
438 A tuple of mgmt_ip (string) and public_ip (string)
439 """
440 mgmt_ip = None
441 public_ip = None
442 if 'addresses' in vm_info:
443 for network_name, network_info in vm_info['addresses'].items():
444 if network_info and network_name == self.driver.mgmt_network:
445 for interface in network_info:
446 if 'OS-EXT-IPS:type' in interface:
447 if interface['OS-EXT-IPS:type'] == 'fixed':
448 mgmt_ip = interface['addr']
449 elif interface['OS-EXT-IPS:type'] == 'floating':
450 public_ip = interface['addr']
451 return (mgmt_ip, public_ip)
452
453 def get_vdu_epa_info(self, vm_info):
454 """
455 Get flavor information (including EPA) for VDU
456
457 Arguments:
458 vm_info : A dictionary returned by novaclient library listing VM attributes
459 Returns:
460 flavor_info: A dictionary object returned by novaclient library listing flavor attributes
461 """
462 if 'flavor' in vm_info and 'id' in vm_info['flavor']:
463 try:
464 flavor_info = self.driver.nova_flavor_get(vm_info['flavor']['id'])
465 return flavor_info
466 except Exception as e:
467 self.log.exception("Exception %s occured during get-flavor", str(e))
468 return dict()
469
470 def _parse_vdu_cp_info(self, vdu_id):
471 """
472 Get connection point information for VDU identified by vdu_id
473 Arguments:
474 vdu_id (string) : VDU Id (vm_info['id'])
475 Returns:
476 A List of object RwcalYang.VDUInfoParams_ConnectionPoints()
477
478 """
479 cp_list = []
480 # Fill the port information
481 port_list = self.driver.neutron_port_list(**{'device_id': vdu_id})
482 for port in port_list:
483 cp_info = self.driver.utils.network._parse_cp(port)
484 cp = RwcalYang.VDUInfoParams_ConnectionPoints()
485 cp.from_dict(cp_info.as_dict())
486 cp_list.append(cp)
487 return cp_list
488
489 def _parse_vdu_state_info(self, vm_info):
490 """
491 Get VDU state information
492
493 Arguments:
494 vm_info : A dictionary returned by novaclient library listing VM attributes
495
496 Returns:
497 state (string): State of the VDU
498 """
499 if 'status' in vm_info:
500 if vm_info['status'] == 'ACTIVE':
501 vdu_state = 'active'
502 elif vm_info['status'] == 'ERROR':
503 vdu_state = 'failed'
504 else:
505 vdu_state = 'inactive'
506 else:
507 vdu_state = 'unknown'
508 return vdu_state
509
510 def _parse_vdu_server_group_info(self, vm_info):
511 """
512 Get VDU server group information
513 Arguments:
514 vm_info : A dictionary returned by novaclient library listing VM attributes
515
516 Returns:
517 server_group_name (string): Name of the server group to which VM belongs, else empty string
518
519 """
520 server_group = [ v['name']
521 for v in self.driver.nova_server_group_list()
522 if vm_info['id'] in v['members'] ]
523 if server_group:
524 return server_group[0]
525 else:
526 return str()
527
528 def _parse_vdu_boot_config_data(self, vm_info):
529 """
530 Parses VDU supplemental boot data
531 Arguments:
532 vm_info : A dictionary returned by novaclient library listing VM attributes
533
534 Returns:
535 List of RwcalYang.VDUInfoParams_SupplementalBootData()
536 """
537 supplemental_boot_data = None
538 node_id = None
539 if 'config_drive' in vm_info:
540 supplemental_boot_data = RwcalYang.VDUInfoParams_SupplementalBootData()
541 supplemental_boot_data.boot_data_drive = vm_info['config_drive']
542 # Look for any metadata
543 if 'metadata' not in vm_info:
544 return node_id, supplemental_boot_data
545 if supplemental_boot_data is None:
546 supplemental_boot_data = RwcalYang.VDUInfoParams_SupplementalBootData()
547 for key, value in vm_info['metadata'].items():
548 if key == 'rift_node_id':
549 node_id = value
550 else:
551 try:
552 # rift only
553 cm = supplemental_boot_data.custom_meta_data.add()
554 cm.name = key
555 cm.value = str(value)
556 except Exception as e:
557 pass
558 return node_id, supplemental_boot_data
559
560 def _parse_vdu_volume_info(self, vm_info):
561 """
562 Get VDU server group information
563 Arguments:
564 vm_info : A dictionary returned by novaclient library listing VM attributes
565
566 Returns:
567 List of RwcalYang.VDUInfoParams_Volumes()
568 """
569 volumes = list()
570
571 try:
572 volume_list = self.driver.nova_volume_list(vm_info['id'])
573 except Exception as e:
574 self.log.exception("Exception %s occured during nova-volume-list", str(e))
575 return volumes
576
577 for v in volume_list:
578 volume = RwcalYang.VDUInfoParams_Volumes()
579 try:
580 volume.name = (v['device']).split('/')[2]
581 volume.volume_id = v['volumeId']
582 details = self.driver.cinder_volume_get(volume.volume_id)
583 for k, v in details.metadata.items():
584 vd = volume.custom_meta_data.add()
585 vd.name = k
586 vd.value = v
587 except Exception as e:
588 self.log.exception("Exception %s occured during volume list parsing", str(e))
589 continue
590 else:
591 volumes.append(volume)
592 return volumes
593
594 def _parse_vdu_console_url(self, vm_info):
595 """
596 Get VDU console URL
597 Arguments:
598 vm_info : A dictionary returned by novaclient library listing VM attributes
599
600 Returns:
601 console_url(string): Console URL for VM
602 """
603 console_url = None
604 if self._parse_vdu_state_info(vm_info) == 'active':
605 try:
606 serv_console_url = self.driver.nova_server_console(vm_info['id'])
607 if 'console' in serv_console_url:
608 console_url = serv_console_url['console']['url']
609 else:
610 self.log.error("Error fetching console url. This could be an Openstack issue. Console : " + str(serv_console_url))
611
612
613 except Exception as e:
614 self.log.exception("Exception %s occured during volume list parsing", str(e))
615 return console_url
616
617 def parse_cloud_vdu_info(self, vm_info):
618 """
619 Parse vm_info dictionary (return by python-client) and put values in GI object for VDU
620
621 Arguments:
622 vm_info : A dictionary object return by novaclient library listing VM attributes
623
624 Returns:
625 Protobuf GI Object of type RwcalYang.VDUInfoParams()
626 """
627 vdu = RwcalYang.VDUInfoParams()
628 vdu.name = vm_info['name']
629 vdu.vdu_id = vm_info['id']
630 vdu.cloud_type = 'openstack'
631
632 if 'config_drive' in vm_info:
633 vdu.supplemental_boot_data.boot_data_drive = vm_info['config_drive']
634
635 if 'image' in vm_info and 'id' in vm_info['image']:
636 vdu.image_id = vm_info['image']['id']
637
638 if 'availability_zone' in vm_info:
639 vdu.availability_zone = vm_info['availability_zone']
640
641 vdu.state = self._parse_vdu_state_info(vm_info)
642 management_ip,public_ip = self._parse_vdu_mgmt_address_info(vm_info)
643
644 if management_ip:
645 vdu.management_ip = management_ip
646
647 if public_ip:
648 vdu.public_ip = public_ip
649
650 if 'flavor' in vm_info and 'id' in vm_info['flavor']:
651 vdu.flavor_id = vm_info['flavor']['id']
652 flavor_info = self.get_vdu_epa_info(vm_info)
653 vm_flavor = self.driver.utils.flavor.parse_vm_flavor_epa_info(flavor_info)
654 guest_epa = self.driver.utils.flavor.parse_guest_epa_info(flavor_info)
655 host_epa = self.driver.utils.flavor.parse_host_epa_info(flavor_info)
656 host_aggregates = self.driver.utils.flavor.parse_host_aggregate_epa_info(flavor_info)
657
658 vdu.vm_flavor.from_dict(vm_flavor.as_dict())
659 vdu.guest_epa.from_dict(guest_epa.as_dict())
660 vdu.host_epa.from_dict(host_epa.as_dict())
661 for aggr in host_aggregates:
662 ha = vdu.host_aggregate.add()
663 ha.from_dict(aggr.as_dict())
664
665 vdu.node_id, vdu.supplemental_boot_data = self._parse_vdu_boot_config_data(vm_info)
666
667 cp_list = self._parse_vdu_cp_info(vdu.vdu_id)
668 for cp in cp_list:
669 vdu.connection_points.append(cp)
670
671 vdu.server_group.name = self._parse_vdu_server_group_info(vm_info)
672
673 for v in self._parse_vdu_volume_info(vm_info):
674 vdu.volumes.append(v)
675
676 vdu.console_url = self._parse_vdu_console_url(vm_info)
677 return vdu
678
679
680 def perform_vdu_network_cleanup(self, vdu_id):
681 """
682 This function cleans up networking resources related to VDU
683 Arguments:
684 vdu_id(string): VDU id
685 Returns:
686 None
687 """
688 ### Get list of floating_ips associated with this instance and delete them
689 floating_ips = [ f for f in self.driver.nova_floating_ip_list() if f.instance_id == vdu_id ]
690 for f in floating_ips:
691 self.driver.nova_floating_ip_delete(f)
692
693 ### Get list of port on VM and delete them.
694 port_list = self.driver.neutron_port_list(**{'device_id': vdu_id})
695
696 for port in port_list:
697 if ((port['device_owner'] == 'compute:None') or (port['device_owner'] == '')):
698 self.driver.neutron_port_delete(port['id'])
699