Merge pull request #228 from stevenvanrossem/master
[osm/vim-emu.git] / src / emuvim / api / openstack / compute.py
1 from mininet.link import Link
2 from resources import *
3 from docker import DockerClient
4 import logging
5 import threading
6 import uuid
7 import time
8 import ip_handler as IP
9
10
11 class HeatApiStackInvalidException(Exception):
12 """
13 Exception thrown when a submitted stack is invalid.
14 """
15
16 def __init__(self, value):
17 self.value = value
18
19 def __str__(self):
20 return repr(self.value)
21
22
23 class OpenstackCompute(object):
24 """
25 This class is a datacenter specific compute object that tracks all containers that are running in a datacenter,
26 as well as networks and configured ports.
27 It has some stack dependet logic and can check if a received stack is valid.
28
29 It also handles start and stop of containers.
30 """
31
32 def __init__(self):
33 self.dc = None
34 self.stacks = dict()
35 self.computeUnits = dict()
36 self.routers = dict()
37 self.flavors = dict()
38 self._images = dict()
39 self.nets = dict()
40 self.ports = dict()
41 self.compute_nets = dict()
42 self.dcli = DockerClient(base_url='unix://var/run/docker.sock')
43
44 @property
45 def images(self):
46 """
47 Updates the known images. Asks the docker daemon for a list of all known images and returns
48 the new dictionary.
49
50 :return: Returns the new image dictionary.
51 :rtype: ``dict``
52 """
53 for image in self.dcli.images.list():
54 if len(image.tags) > 0:
55 for t in image.tags:
56 t = t.replace(":latest", "") # only use short tag names for OSM compatibility
57 if t not in self._images:
58 self._images[t] = Image(t)
59 return self._images
60
61 def add_stack(self, stack):
62 """
63 Adds a new stack to the compute node.
64
65 :param stack: Stack dictionary.
66 :type stack: :class:`heat.resources.stack`
67 """
68 if not self.check_stack(stack):
69 self.clean_broken_stack(stack)
70 raise HeatApiStackInvalidException("Stack did not pass validity checks")
71 self.stacks[stack.id] = stack
72
73 def clean_broken_stack(self, stack):
74 for port in stack.ports.values():
75 if port.id in self.ports:
76 del self.ports[port.id]
77 for server in stack.servers.values():
78 if server.id in self.computeUnits:
79 del self.computeUnits[server.id]
80 for net in stack.nets.values():
81 if net.id in self.nets:
82 del self.nets[net.id]
83
84 def check_stack(self, stack):
85 """
86 Checks all dependencies of all servers, ports and routers and their most important parameters.
87
88 :param stack: A reference of the stack that should be checked.
89 :type stack: :class:`heat.resources.stack`
90 :return: * *True*: If the stack is completely fine.
91 * *False*: Else
92 :rtype: ``bool``
93 """
94 everything_ok = True
95 for server in stack.servers.values():
96 for port_name in server.port_names:
97 if port_name not in stack.ports:
98 logging.warning("Server %s of stack %s has a port named %s that is not known." %
99 (server.name, stack.stack_name, port_name))
100 everything_ok = False
101 if server.image is None:
102 logging.warning("Server %s holds no image." % (server.name))
103 everything_ok = False
104 if server.command is None:
105 logging.warning("Server %s holds no command." % (server.name))
106 everything_ok = False
107 for port in stack.ports.values():
108 if port.net_name not in stack.nets:
109 logging.warning("Port %s of stack %s has a network named %s that is not known." %
110 (port.name, stack.stack_name, port.net_name))
111 everything_ok = False
112 if port.intf_name is None:
113 logging.warning("Port %s has no interface name." % (port.name))
114 everything_ok = False
115 if port.ip_address is None:
116 logging.warning("Port %s has no IP address." % (port.name))
117 everything_ok = False
118 for router in stack.routers.values():
119 for subnet_name in router.subnet_names:
120 found = False
121 for net in stack.nets.values():
122 if net.subnet_name == subnet_name:
123 found = True
124 break
125 if not found:
126 logging.warning("Router %s of stack %s has a network named %s that is not known." %
127 (router.name, stack.stack_name, subnet_name))
128 everything_ok = False
129 return everything_ok
130
131 def add_flavor(self, name, cpu, memory, memory_unit, storage, storage_unit):
132 """
133 Adds a flavor to the stack.
134
135 :param name: Specifies the name of the flavor.
136 :type name: ``str``
137 :param cpu:
138 :type cpu: ``str``
139 :param memory:
140 :type memory: ``str``
141 :param memory_unit:
142 :type memory_unit: ``str``
143 :param storage:
144 :type storage: ``str``
145 :param storage_unit:
146 :type storage_unit: ``str``
147 """
148 flavor = InstanceFlavor(name, cpu, memory, memory_unit, storage, storage_unit)
149 self.flavors[flavor.name] = flavor
150 return flavor
151
152 def deploy_stack(self, stackid):
153 """
154 Deploys the stack and starts the emulation.
155
156 :param stackid: An UUID str of the stack
157 :type stackid: ``str``
158 :return: * *False*: If the Datacenter is None
159 * *True*: Else
160 :rtype: ``bool``
161 """
162 if self.dc is None:
163 return False
164
165 stack = self.stacks[stackid]
166 self.update_compute_dicts(stack)
167
168 # Create the networks first
169 for server in stack.servers.values():
170 self._start_compute(server)
171 return True
172
173 def delete_stack(self, stack_id):
174 """
175 Delete a stack and all its components.
176
177 :param stack_id: An UUID str of the stack
178 :type stack_id: ``str``
179 :return: * *False*: If the Datacenter is None
180 * *True*: Else
181 :rtype: ``bool``
182 """
183 if self.dc is None:
184 return False
185
186 # Stop all servers and their links of this stack
187 for server in self.stacks[stack_id].servers.values():
188 self.stop_compute(server)
189 self.delete_server(server)
190 for net in self.stacks[stack_id].nets.values():
191 self.delete_network(net.id)
192 for port in self.stacks[stack_id].ports.values():
193 self.delete_port(port.id)
194
195 del self.stacks[stack_id]
196 return True
197
198 def update_stack(self, old_stack_id, new_stack):
199 """
200 Determines differences within the old and the new stack and deletes, create or changes only parts that
201 differ between the two stacks.
202
203 :param old_stack_id: The ID of the old stack.
204 :type old_stack_id: ``str``
205 :param new_stack: A reference of the new stack.
206 :type new_stack: :class:`heat.resources.stack`
207 :return: * *True*: if the old stack could be updated to the new stack without any error.
208 * *False*: else
209 :rtype: ``bool``
210 """
211 if old_stack_id not in self.stacks:
212 return False
213 old_stack = self.stacks[old_stack_id]
214
215 # Update Stack IDs
216 for server in old_stack.servers.values():
217 if server.name in new_stack.servers:
218 new_stack.servers[server.name].id = server.id
219 for net in old_stack.nets.values():
220 if net.name in new_stack.nets:
221 new_stack.nets[net.name].id = net.id
222 for subnet in new_stack.nets.values():
223 if subnet.subnet_name == net.subnet_name:
224 subnet.subnet_id = net.subnet_id
225 break
226 for port in old_stack.ports.values():
227 if port.name in new_stack.ports:
228 new_stack.ports[port.name].id = port.id
229 for router in old_stack.routers.values():
230 if router.name in new_stack.routers:
231 new_stack.routers[router.name].id = router.id
232
233 # Update the compute dicts to now contain the new_stack components
234 self.update_compute_dicts(new_stack)
235
236 self.update_ip_addresses(old_stack, new_stack)
237
238 # Update all interface names - after each port has the correct UUID!!
239 for port in new_stack.ports.values():
240 port.create_intf_name()
241
242 if not self.check_stack(new_stack):
243 return False
244
245 # Remove unnecessary networks
246 for net in old_stack.nets.values():
247 if not net.name in new_stack.nets:
248 self.delete_network(net.id)
249
250 # Remove all unnecessary servers
251 for server in old_stack.servers.values():
252 if server.name in new_stack.servers:
253 if not server.compare_attributes(new_stack.servers[server.name]):
254 self.stop_compute(server)
255 else:
256 # Delete unused and changed links
257 for port_name in server.port_names:
258 if port_name in old_stack.ports and port_name in new_stack.ports:
259 if not old_stack.ports.get(port_name) == new_stack.ports.get(port_name):
260 my_links = self.dc.net.links
261 for link in my_links:
262 if str(link.intf1) == old_stack.ports[port_name].intf_name and \
263 str(link.intf1.ip) == \
264 old_stack.ports[port_name].ip_address.split('/')[0]:
265 self._remove_link(server.name, link)
266
267 # Add changed link
268 self._add_link(server.name,
269 new_stack.ports[port_name].ip_address,
270 new_stack.ports[port_name].intf_name,
271 new_stack.ports[port_name].net_name)
272 break
273 else:
274 my_links = self.dc.net.links
275 for link in my_links:
276 if str(link.intf1) == old_stack.ports[port_name].intf_name and \
277 str(link.intf1.ip) == old_stack.ports[port_name].ip_address.split('/')[0]:
278 self._remove_link(server.name, link)
279 break
280
281 # Create new links
282 for port_name in new_stack.servers[server.name].port_names:
283 if port_name not in server.port_names:
284 self._add_link(server.name,
285 new_stack.ports[port_name].ip_address,
286 new_stack.ports[port_name].intf_name,
287 new_stack.ports[port_name].net_name)
288 else:
289 self.stop_compute(server)
290
291 # Start all new servers
292 for server in new_stack.servers.values():
293 if server.name not in self.dc.containers:
294 self._start_compute(server)
295 else:
296 server.emulator_compute = self.dc.containers.get(server.name)
297
298 del self.stacks[old_stack_id]
299 self.stacks[new_stack.id] = new_stack
300 return True
301
302 def update_ip_addresses(self, old_stack, new_stack):
303 """
304 Updates the subnet and the port IP addresses - which should always be in this order!
305
306 :param old_stack: The currently running stack
307 :type old_stack: :class:`heat.resources.stack`
308 :param new_stack: The new created stack
309 :type new_stack: :class:`heat.resources.stack`
310 """
311 self.update_subnet_cidr(old_stack, new_stack)
312 self.update_port_addresses(old_stack, new_stack)
313
314 def update_port_addresses(self, old_stack, new_stack):
315 """
316 Updates the port IP addresses. First resets all issued addresses. Then get all IP addresses from the old
317 stack and sets them to the same ports in the new stack. Finally all new or changed instances will get new
318 IP addresses.
319
320 :param old_stack: The currently running stack
321 :type old_stack: :class:`heat.resources.stack`
322 :param new_stack: The new created stack
323 :type new_stack: :class:`heat.resources.stack`
324 """
325 for net in new_stack.nets.values():
326 net.reset_issued_ip_addresses()
327
328 for old_port in old_stack.ports.values():
329 for port in new_stack.ports.values():
330 if port.compare_attributes(old_port):
331 for net in new_stack.nets.values():
332 if net.name == port.net_name:
333 if net.assign_ip_address(old_port.ip_address, port.name):
334 port.ip_address = old_port.ip_address
335 port.mac_address = old_port.mac_address
336 else:
337 port.ip_address = net.get_new_ip_address(port.name)
338
339 for port in new_stack.ports.values():
340 for net in new_stack.nets.values():
341 if port.net_name == net.name and not net.is_my_ip(port.ip_address, port.name):
342 port.ip_address = net.get_new_ip_address(port.name)
343
344 def update_subnet_cidr(self, old_stack, new_stack):
345 """
346 Updates the subnet IP addresses. If the new stack contains subnets from the old stack it will take those
347 IP addresses. Otherwise it will create new IP addresses for the subnet.
348
349 :param old_stack: The currently running stack
350 :type old_stack: :class:`heat.resources.stack`
351 :param new_stack: The new created stack
352 :type new_stack: :class:`heat.resources.stack`
353 """
354 for old_subnet in old_stack.nets.values():
355 IP.free_cidr(old_subnet.get_cidr(), old_subnet.subnet_id)
356
357 for subnet in new_stack.nets.values():
358 subnet.clear_cidr()
359 for old_subnet in old_stack.nets.values():
360 if subnet.subnet_name == old_subnet.subnet_name:
361 if IP.assign_cidr(old_subnet.get_cidr(), subnet.subnet_id):
362 subnet.set_cidr(old_subnet.get_cidr())
363
364 for subnet in new_stack.nets.values():
365 if IP.is_cidr_issued(subnet.get_cidr()):
366 continue
367
368 cird = IP.get_new_cidr(subnet.subnet_id)
369 subnet.set_cidr(cird)
370 return
371
372 def update_compute_dicts(self, stack):
373 """
374 Update and add all stack components tho the compute dictionaries.
375
376 :param stack: A stack reference, to get all required components.
377 :type stack: :class:`heat.resources.stack`
378 """
379 for server in stack.servers.values():
380 self.computeUnits[server.id] = server
381 if isinstance(server.flavor, dict):
382 self.add_flavor(server.flavor['flavorName'],
383 server.flavor['vcpu'],
384 server.flavor['ram'], 'MB',
385 server.flavor['storage'], 'GB')
386 server.flavor = server.flavor['flavorName']
387 for router in stack.routers.values():
388 self.routers[router.id] = router
389 for net in stack.nets.values():
390 self.nets[net.id] = net
391 for port in stack.ports.values():
392 self.ports[port.id] = port
393
394 def _start_compute(self, server):
395 """
396 Starts a new compute object (docker container) inside the emulator.
397 Should only be called by stack modifications and not directly.
398
399 :param server: Specifies the compute resource.
400 :type server: :class:`heat.resources.server`
401 """
402 logging.debug("Starting new compute resources %s" % server.name)
403 network = list()
404
405 for port_name in server.port_names:
406 network_dict = dict()
407 port = self.find_port_by_name_or_id(port_name)
408 if port is not None:
409 network_dict['id'] = port.intf_name
410 network_dict['ip'] = port.ip_address
411 network_dict[network_dict['id']] = self.find_network_by_name_or_id(port.net_name).name
412 network.append(network_dict)
413 self.compute_nets[server.name] = network
414 c = self.dc.startCompute(server.name, image=server.image, command=server.command,
415 network=network, flavor_name=server.flavor)
416 server.emulator_compute = c
417
418 for intf in c.intfs.values():
419 for port_name in server.port_names:
420 port = self.find_port_by_name_or_id(port_name)
421 if port is not None:
422 if intf.name == port.intf_name:
423 # wait up to one second for the intf to come up
424 self.timeout_sleep(intf.isUp, 1)
425 if port.mac_address is not None:
426 intf.setMAC(port.mac_address)
427 else:
428 port.mac_address = intf.MAC()
429
430 # Start the real emulator command now as specified in the dockerfile
431 # ENV SON_EMU_CMD
432 config = c.dcinfo.get("Config", dict())
433 env = config.get("Env", list())
434 for env_var in env:
435 if "SON_EMU_CMD=" in env_var:
436 cmd = str(env_var.split("=")[1])
437 server.son_emu_command = cmd
438 # execute command in new thread to ensure that GK is not blocked by VNF
439 t = threading.Thread(target=c.cmdPrint, args=(cmd,))
440 t.daemon = True
441 t.start()
442
443 def stop_compute(self, server):
444 """
445 Determines which links should be removed before removing the server itself.
446
447 :param server: The server that should be removed
448 :type server: ``heat.resources.server``
449 """
450 logging.debug("Stopping container %s with full name %s" % (server.name, server.full_name))
451 link_names = list()
452 for port_name in server.port_names:
453 link_names.append(self.find_port_by_name_or_id(port_name).intf_name)
454 my_links = self.dc.net.links
455 for link in my_links:
456 if str(link.intf1) in link_names:
457 # Remove all self created links that connect the server to the main switch
458 self._remove_link(server.name, link)
459
460 # Stop the server and the remaining connection to the datacenter switch
461 self.dc.stopCompute(server.name)
462 # Only now delete all its ports and the server itself
463 for port_name in server.port_names:
464 self.delete_port(port_name)
465 self.delete_server(server)
466
467 def find_server_by_name_or_id(self, name_or_id):
468 """
469 Tries to find the server by ID and if this does not succeed then tries to find it via name.
470
471 :param name_or_id: UUID or name of the server.
472 :type name_or_id: ``str``
473 :return: Returns the server reference if it was found or None
474 :rtype: :class:`heat.resources.server`
475 """
476 if name_or_id in self.computeUnits:
477 return self.computeUnits[name_or_id]
478
479 for server in self.computeUnits.values():
480 if server.name == name_or_id or server.template_name == name_or_id or server.full_name == name_or_id:
481 return server
482 return None
483
484 def create_server(self, name, stack_operation=False):
485 """
486 Creates a server with the specified name. Raises an exception when a server with the given name already
487 exists!
488
489 :param name: Name of the new server.
490 :type name: ``str``
491 :param stack_operation: Allows the heat parser to create modules without adapting the current emulation.
492 :type stack_operation: ``bool``
493 :return: Returns the created server.
494 :rtype: :class:`heat.resources.server`
495 """
496 if self.find_server_by_name_or_id(name) is not None and not stack_operation:
497 raise Exception("Server with name %s already exists." % name)
498 server = Server(name)
499 server.id = str(uuid.uuid4())
500 if not stack_operation:
501 self.computeUnits[server.id] = server
502 return server
503
504 def delete_server(self, server):
505 """
506 Deletes the given server from the stack dictionary and the computeUnits dictionary.
507
508 :param server: Reference of the server that should be deleted.
509 :type server: :class:`heat.resources.server`
510 :return: * *False*: If the server name is not in the correct format ('datacentername_stackname_servername') \
511 or when no stack with the correct stackname was found.
512 * *True*: Else
513 :rtype: ``bool``
514 """
515 if server is None:
516 return False
517 name_parts = server.name.split('_')
518 if len(name_parts) < 3:
519 return False
520
521 for stack in self.stacks.values():
522 if stack.stack_name == name_parts[1]:
523 stack.servers.pop(server.id, None)
524 if self.computeUnits.pop(server.id, None) is None:
525 return False
526 return True
527
528 def find_network_by_name_or_id(self, name_or_id):
529 """
530 Tries to find the network by ID and if this does not succeed then tries to find it via name.
531
532 :param name_or_id: UUID or name of the network.
533 :type name_or_id: ``str``
534 :return: Returns the network reference if it was found or None
535 :rtype: :class:`heat.resources.net`
536 """
537 if name_or_id in self.nets:
538 return self.nets[name_or_id]
539 for net in self.nets.values():
540 if net.name == name_or_id:
541 return net
542
543 return None
544
545 def create_network(self, name, stack_operation=False):
546 """
547 Creates a new network with the given name. Raises an exception when a network with the given name already
548 exists!
549
550 :param name: Name of the new network.
551 :type name: ``str``
552 :param stack_operation: Allows the heat parser to create modules without adapting the current emulation.
553 :type stack_operation: ``bool``
554 :return: :class:`heat.resources.net`
555 """
556 logging.debug("Creating network with name %s" % name)
557 if self.find_network_by_name_or_id(name) is not None and not stack_operation:
558 logging.warning("Creating network with name %s failed, as it already exists" % name)
559 raise Exception("Network with name %s already exists." % name)
560 network = Net(name)
561 network.id = str(uuid.uuid4())
562 if not stack_operation:
563 self.nets[network.id] = network
564 return network
565
566 def delete_network(self, name_or_id):
567 """
568 Deletes the given network.
569
570 :param name_or_id: Name or UUID of the network.
571 :type name_or_id: ``str``
572 """
573 net = self.find_network_by_name_or_id(name_or_id)
574 if net is None:
575 raise Exception("Network with name or id %s does not exists." % name_or_id)
576
577 for stack in self.stacks.values():
578 stack.nets.pop(net.name, None)
579
580 self.nets.pop(net.id, None)
581
582 def create_port(self, name, stack_operation=False):
583 """
584 Creates a new port with the given name. Raises an exception when a port with the given name already
585 exists!
586
587 :param name: Name of the new port.
588 :type name: ``str``
589 :param stack_operation: Allows the heat parser to create modules without adapting the current emulation.
590 :type stack_operation: ``bool``
591 :return: Returns the created port.
592 :rtype: :class:`heat.resources.port`
593 """
594 port = self.find_port_by_name_or_id(name)
595 if port is not None and not stack_operation:
596 logging.warning("Creating port with name %s failed, as it already exists" % name)
597 raise Exception("Port with name %s already exists." % name)
598 logging.debug("Creating port with name %s" % name)
599 port = Port(name)
600 if not stack_operation:
601 self.ports[port.id] = port
602 port.create_intf_name()
603 return port
604
605 def find_port_by_name_or_id(self, name_or_id):
606 """
607 Tries to find the port by ID and if this does not succeed then tries to find it via name.
608
609 :param name_or_id: UUID or name of the network.
610 :type name_or_id: ``str``
611 :return: Returns the port reference if it was found or None
612 :rtype: :class:`heat.resources.port`
613 """
614 if name_or_id in self.ports:
615 return self.ports[name_or_id]
616 for port in self.ports.values():
617 if port.name == name_or_id or port.template_name == name_or_id:
618 return port
619
620 return None
621
622 def delete_port(self, name_or_id):
623 """
624 Deletes the given port. Raises an exception when the port was not found!
625
626 :param name_or_id: UUID or name of the port.
627 :type name_or_id: ``str``
628 """
629 port = self.find_port_by_name_or_id(name_or_id)
630 if port is None:
631 raise Exception("Port with name or id %s does not exists." % name_or_id)
632
633 my_links = self.dc.net.links
634 for link in my_links:
635 if str(link.intf1) == port.intf_name and \
636 str(link.intf1.ip) == port.ip_address.split('/')[0]:
637 self._remove_link(link.intf1.node.name, link)
638 break
639
640 self.ports.pop(port.id, None)
641 for stack in self.stacks.values():
642 stack.ports.pop(port.name, None)
643
644 def _add_link(self, node_name, ip_address, link_name, net_name):
645 """
646 Adds a new link between datacenter switch and the node with the given name.
647
648 :param node_name: Name of the required node.
649 :type node_name: ``str``
650 :param ip_address: IP-Address of the node.
651 :type ip_address: ``str``
652 :param link_name: Link name.
653 :type link_name: ``str``
654 :param net_name: Network name.
655 :type net_name: ``str``
656 """
657 node = self.dc.net.get(node_name)
658 params = {'params1': {'ip': ip_address,
659 'id': link_name,
660 link_name: net_name},
661 'intfName1': link_name,
662 'cls': Link}
663 link = self.dc.net.addLink(node, self.dc.switch, **params)
664 OpenstackCompute.timeout_sleep(link.intf1.isUp, 1)
665
666 def _remove_link(self, server_name, link):
667 """
668 Removes a link between server and datacenter switch.
669
670 :param server_name: Specifies the server where the link starts.
671 :type server_name: ``str``
672 :param link: A reference of the link which should be removed.
673 :type link: :class:`mininet.link`
674 """
675 self.dc.switch.detach(link.intf2)
676 del self.dc.switch.intfs[self.dc.switch.ports[link.intf2]]
677 del self.dc.switch.ports[link.intf2]
678 del self.dc.switch.nameToIntf[link.intf2.name]
679 self.dc.net.removeLink(link=link)
680 for intf_key in self.dc.net[server_name].intfs.keys():
681 if self.dc.net[server_name].intfs[intf_key].link == link:
682 self.dc.net[server_name].intfs[intf_key].delete()
683 del self.dc.net[server_name].intfs[intf_key]
684
685 @staticmethod
686 def timeout_sleep(function, max_sleep):
687 """
688 This function will execute a function all 0.1 seconds until it successfully returns.
689 Will return after `max_sleep` seconds if not successful.
690
691 :param function: The function to execute. Should return true if done.
692 :type function: ``function``
693 :param max_sleep: Max seconds to sleep. 1 equals 1 second.
694 :type max_sleep: ``float``
695 """
696 current_time = time.time()
697 stop_time = current_time + max_sleep
698 while not function() and current_time < stop_time:
699 current_time = time.time()
700 time.sleep(0.1)