Merge "Relation support for multi-charm VNFs"
[osm/N2VC.git] / tests / base.py
1 #!/usr/bin/env python3
2 import asyncio
3 import datetime
4 import logging
5 import n2vc.vnf
6 import pylxd
7 import pytest
8 import os
9 import shlex
10 import subprocess
11 import time
12 import uuid
13 import yaml
14
15 from juju.controller import Controller
16
17 # Disable InsecureRequestWarning w/LXD
18 import urllib3
19 urllib3.disable_warnings()
20 logging.getLogger("urllib3").setLevel(logging.WARNING)
21
22 here = os.path.dirname(os.path.realpath(__file__))
23
24
25 def is_bootstrapped():
26 result = subprocess.run(['juju', 'switch'], stdout=subprocess.PIPE)
27 return (
28 result.returncode == 0 and
29 len(result.stdout.decode().strip()) > 0)
30
31
32 bootstrapped = pytest.mark.skipif(
33 not is_bootstrapped(),
34 reason='bootstrapped Juju environment required')
35
36
37 class CleanController():
38 """
39 Context manager that automatically connects and disconnects from
40 the currently active controller.
41
42 Note: Unlike CleanModel, this will not create a new controller for you,
43 and an active controller must already be available.
44 """
45 def __init__(self):
46 self._controller = None
47
48 async def __aenter__(self):
49 self._controller = Controller()
50 await self._controller.connect()
51 return self._controller
52
53 async def __aexit__(self, exc_type, exc, tb):
54 await self._controller.disconnect()
55
56
57 def debug(msg):
58 """Format debug messages in a consistent way."""
59 now = datetime.datetime.now()
60
61 # TODO: Decide on the best way to log. Output from `logging.debug` shows up
62 # when a test fails, but print() will always show up when running tox with
63 # `-s`, which is really useful for debugging single tests without having to
64 # insert a False assert to see the log.
65 logging.debug(
66 "[{}] {}".format(now.strftime('%Y-%m-%dT%H:%M:%S'), msg)
67 )
68 # print(
69 # "[{}] {}".format(now.strftime('%Y-%m-%dT%H:%M:%S'), msg)
70 # )
71
72
73 def get_charm_path():
74 return "{}/charms".format(here)
75
76
77 def get_layer_path():
78 return "{}/charms/layers".format(here)
79
80
81 def collect_metrics(application):
82 """Invoke Juju's metrics collector.
83
84 Caveat: this shells out to the `juju collect-metrics` command, rather than
85 making an API call. At the time of writing, that API is not exposed through
86 the client library.
87 """
88
89 try:
90 subprocess.check_call(['juju', 'collect-metrics', application])
91 except subprocess.CalledProcessError as e:
92 raise Exception("Unable to collect metrics: {}".format(e))
93
94
95 def has_metrics(charm):
96 """Check if a charm has metrics defined."""
97 metricsyaml = "{}/{}/metrics.yaml".format(
98 get_layer_path(),
99 charm,
100 )
101 if os.path.exists(metricsyaml):
102 return True
103 return False
104
105
106 def get_descriptor(descriptor):
107 desc = None
108 try:
109 tmp = yaml.load(descriptor)
110
111 # Remove the envelope
112 root = list(tmp.keys())[0]
113 if root == "nsd:nsd-catalog":
114 desc = tmp['nsd:nsd-catalog']['nsd'][0]
115 elif root == "vnfd:vnfd-catalog":
116 desc = tmp['vnfd:vnfd-catalog']['vnfd'][0]
117 except ValueError:
118 assert False
119 return desc
120
121
122 def get_n2vc(loop=None):
123 """Return an instance of N2VC.VNF."""
124 log = logging.getLogger()
125 log.level = logging.DEBUG
126
127 # Extract parameters from the environment in order to run our test
128 vca_host = os.getenv('VCA_HOST', '127.0.0.1')
129 vca_port = os.getenv('VCA_PORT', 17070)
130 vca_user = os.getenv('VCA_USER', 'admin')
131 vca_charms = os.getenv('VCA_CHARMS', None)
132 vca_secret = os.getenv('VCA_SECRET', None)
133
134 client = n2vc.vnf.N2VC(
135 log=log,
136 server=vca_host,
137 port=vca_port,
138 user=vca_user,
139 secret=vca_secret,
140 artifacts=vca_charms,
141 loop=loop
142 )
143 return client
144
145
146 def create_lxd_container(public_key=None, name="test_name"):
147 """
148 Returns a container object
149
150 If public_key isn't set, we'll use the Juju ssh key
151
152 :param public_key: The public key to inject into the container
153 :param name: The name of the test being run
154 """
155 container = None
156
157 # Format name so it's valid
158 name = name.replace("_", "-").replace(".", "")
159
160 client = get_lxd_client()
161 test_machine = "test-{}-{}".format(
162 uuid.uuid4().hex[-4:],
163 name,
164 )
165
166 private_key_path, public_key_path = find_n2vc_ssh_keys()
167
168 try:
169 # create profile w/cloud-init and juju ssh key
170 if not public_key:
171 public_key = ""
172 with open(public_key_path, "r") as f:
173 public_key = f.readline()
174
175 client.profiles.create(
176 test_machine,
177 config={
178 'user.user-data': '#cloud-config\nssh_authorized_keys:\n- {}'.format(public_key)},
179 devices={
180 'root': {'path': '/', 'pool': 'default', 'type': 'disk'},
181 'eth0': {
182 'nictype': 'bridged',
183 'parent': 'lxdbr0',
184 'type': 'nic'
185 }
186 }
187 )
188 except Exception as ex:
189 debug("Error creating lxd profile {}: {}".format(test_machine, ex))
190 raise ex
191
192 try:
193 # create lxc machine
194 config = {
195 'name': test_machine,
196 'source': {
197 'type': 'image',
198 'alias': 'xenial',
199 'mode': 'pull',
200 'protocol': 'simplestreams',
201 'server': 'https://cloud-images.ubuntu.com/releases',
202 },
203 'profiles': [test_machine],
204 }
205 container = client.containers.create(config, wait=True)
206 container.start(wait=True)
207 except Exception as ex:
208 debug("Error creating lxd container {}: {}".format(test_machine, ex))
209 # This is a test-ending failure.
210 raise ex
211
212 def wait_for_network(container, timeout=30):
213 """Wait for eth0 to have an ipv4 address."""
214 starttime = time.time()
215 while(time.time() < starttime + timeout):
216 time.sleep(1)
217 if 'eth0' in container.state().network:
218 addresses = container.state().network['eth0']['addresses']
219 if len(addresses) > 0:
220 if addresses[0]['family'] == 'inet':
221 return addresses[0]
222 return None
223
224 try:
225 wait_for_network(container)
226 except Exception as ex:
227 debug(
228 "Error waiting for container {} network: {}".format(
229 test_machine,
230 ex,
231 )
232 )
233
234 # HACK: We need to give sshd a chance to bind to the interface,
235 # and pylxd's container.execute seems to be broken and fails and/or
236 # hangs trying to properly check if the service is up.
237 (exit_code, stdout, stderr) = container.execute([
238 'ping',
239 '-c', '5', # Wait for 5 ECHO_REPLY
240 '8.8.8.8', # Ping Google's public DNS
241 '-W', '15', # Set a 15 second deadline
242 ])
243 if exit_code > 0:
244 # The network failed
245 raise Exception("Unable to verify container network")
246
247 return container
248
249
250 def destroy_lxd_container(container):
251 """Stop and delete a LXD container.
252
253 Sometimes we see errors talking to LXD -- ephemerial issues like
254 load or a bug that's killed the API. We'll do our best to clean
255 up here, and we should run a cleanup after all tests are finished
256 to remove any extra containers and profiles belonging to us.
257 """
258
259 if type(container) is bool:
260 return
261
262 name = container.name
263 debug("Destroying container {}".format(name))
264
265 client = get_lxd_client()
266
267 def wait_for_stop(timeout=30):
268 """Wait for eth0 to have an ipv4 address."""
269 starttime = time.time()
270 while(time.time() < starttime + timeout):
271 time.sleep(1)
272 if container.state == "Stopped":
273 return
274
275 def wait_for_delete(timeout=30):
276 starttime = time.time()
277 while(time.time() < starttime + timeout):
278 time.sleep(1)
279 if client.containers.exists(name) is False:
280 return
281
282 try:
283 container.stop(wait=False)
284 wait_for_stop()
285 except Exception as ex:
286 debug(
287 "Error stopping container {}: {}".format(
288 name,
289 ex,
290 )
291 )
292
293 try:
294 container.delete(wait=False)
295 wait_for_delete()
296 except Exception as ex:
297 debug(
298 "Error deleting container {}: {}".format(
299 name,
300 ex,
301 )
302 )
303
304 try:
305 # Delete the profile created for this container
306 profile = client.profiles.get(name)
307 if profile:
308 profile.delete()
309 except Exception as ex:
310 debug(
311 "Error deleting profile {}: {}".format(
312 name,
313 ex,
314 )
315 )
316
317
318 def find_lxd_config():
319 """Find the LXD configuration directory."""
320 paths = []
321 paths.append(os.path.expanduser("~/.config/lxc"))
322 paths.append(os.path.expanduser("~/snap/lxd/current/.config/lxc"))
323
324 for path in paths:
325 if os.path.exists(path):
326 crt = os.path.expanduser("{}/client.crt".format(path))
327 key = os.path.expanduser("{}/client.key".format(path))
328 if os.path.exists(crt) and os.path.exists(key):
329 return (crt, key)
330 return (None, None)
331
332
333 def find_n2vc_ssh_keys():
334 """Find the N2VC ssh keys."""
335
336 paths = []
337 paths.append(os.path.expanduser("~/.ssh/"))
338
339 for path in paths:
340 if os.path.exists(path):
341 private = os.path.expanduser("{}/id_n2vc_rsa".format(path))
342 public = os.path.expanduser("{}/id_n2vc_rsa.pub".format(path))
343 if os.path.exists(private) and os.path.exists(public):
344 return (private, public)
345 return (None, None)
346
347
348 def find_juju_ssh_keys():
349 """Find the Juju ssh keys."""
350
351 paths = []
352 paths.append(os.path.expanduser("~/.local/share/juju/ssh/"))
353
354 for path in paths:
355 if os.path.exists(path):
356 private = os.path.expanduser("{}/juju_id_rsa".format(path))
357 public = os.path.expanduser("{}/juju_id_rsa.pub".format(path))
358 if os.path.exists(private) and os.path.exists(public):
359 return (private, public)
360 return (None, None)
361
362
363 def get_juju_private_key():
364 keys = find_juju_ssh_keys()
365 return keys[0]
366
367
368 def get_lxd_client(host="127.0.0.1", port="8443", verify=False):
369 """ Get the LXD client."""
370 client = None
371 (crt, key) = find_lxd_config()
372
373 if crt and key:
374 client = pylxd.Client(
375 endpoint="https://{}:{}".format(host, port),
376 cert=(crt, key),
377 verify=verify,
378 )
379
380 return client
381
382
383 # TODO: This is marked serial but can be run in parallel with work, including:
384 # - Fixing an event loop issue; seems that all tests stop when one test stops?
385
386
387 @pytest.mark.serial
388 class TestN2VC(object):
389 """TODO:
390 1. Validator Validation
391
392 Automatically validate the descriptors we're using here, unless the test author explicitly wants to skip them. Useful to make sure tests aren't being run against invalid descriptors, validating functionality that may fail against a properly written descriptor.
393
394 We need to have a flag (instance variable) that controls this behavior. It may be necessary to skip validation and run against a descriptor implementing features that have not yet been released in the Information Model.
395 """
396
397 """
398 The six phases of integration testing, for the test itself and each charm?:
399
400 setup/teardown_class:
401 1. Prepare - Verify the environment and create a new model
402 2. Deploy - Mark the test as ready to execute
403 3. Configure - Configuration to reach Active state
404 4. Test - Execute primitive(s) to verify success
405 5. Collect - Collect any useful artifacts for debugging (charm, logs)
406 6. Destroy - Destroy the model
407
408
409 1. Prepare - Building of charm
410 2. Deploy - Deploying charm
411 3. Configure - Configuration to reach Active state
412 4. Test - Execute primitive(s) to verify success
413 5. Collect - Collect any useful artifacts for debugging (charm, logs)
414 6. Destroy - Destroy the charm
415
416 """
417 @classmethod
418 def setup_class(self):
419 """ setup any state specific to the execution of the given class (which
420 usually contains tests).
421 """
422 # Initialize instance variable(s)
423 self.n2vc = None
424
425 # Track internal state for each test run
426 self.state = {}
427
428 # Parse the test's descriptors
429 self.nsd = get_descriptor(self.NSD_YAML)
430 self.vnfd = get_descriptor(self.VNFD_YAML)
431
432 self.ns_name = self.nsd['name']
433 self.vnf_name = self.vnfd['name']
434
435 # Hard-coded to default for now, but this may change in the future.
436 self.model = "default"
437
438 self.charms = {}
439 self.parse_vnf_descriptor()
440 assert self.charms is not {}
441
442 # Track artifacts, like compiled charms, that will need to be removed
443 self.artifacts = {}
444
445 # Build the charm(s) needed for this test
446 for charm in self.get_charm_names():
447 self.get_charm(charm)
448
449 # A bit of a hack, in order to allow the N2VC callback to run parallel
450 # to pytest. Test(s) should wait for this flag to change to False
451 # before returning.
452 self._running = True
453 self._stopping = False
454
455 @classmethod
456 def teardown_class(self):
457 """ teardown any state that was previously setup with a call to
458 setup_class.
459 """
460 debug("Running teardown_class...")
461 try:
462
463 debug("Destroying LXD containers...")
464 for application in self.state:
465 if self.state[application]['container']:
466 destroy_lxd_container(self.state[application]['container'])
467 debug("Destroying LXD containers...done.")
468
469 # Logout of N2VC
470 if self.n2vc:
471 debug("teardown_class(): Logging out of N2VC...")
472 yield from self.n2vc.logout()
473 debug("teardown_class(): Logging out of N2VC...done.")
474
475 debug("Running teardown_class...done.")
476 except Exception as ex:
477 debug("Exception in teardown_class: {}".format(ex))
478
479 @classmethod
480 def all_charms_active(self):
481 """Determine if the all deployed charms are active."""
482 active = 0
483
484 for application in self.state:
485 if 'status' in self.state[application]:
486 debug("status of {} is '{}'".format(
487 application,
488 self.state[application]['status'],
489 ))
490 if self.state[application]['status'] == 'active':
491 active += 1
492
493 debug("Active charms: {}/{}".format(
494 active,
495 len(self.charms),
496 ))
497
498 if active == len(self.charms):
499 return True
500
501 return False
502
503 @classmethod
504 def are_tests_finished(self):
505 appcount = len(self.state)
506
507 # If we don't have state yet, keep running.
508 if appcount == 0:
509 debug("No applications")
510 return False
511
512 if self._stopping:
513 debug("_stopping is True")
514 return True
515
516 appdone = 0
517 for application in self.state:
518 if self.state[application]['done']:
519 appdone += 1
520
521 debug("{}/{} charms tested".format(appdone, appcount))
522
523 if appcount == appdone:
524 return True
525
526 return False
527
528 @classmethod
529 async def running(self, timeout=600):
530 """Returns if the test is still running.
531
532 @param timeout The time, in seconds, to wait for the test to complete.
533 """
534 if self.are_tests_finished():
535 await self.stop()
536 return False
537
538 await asyncio.sleep(30)
539
540 return self._running
541
542 @classmethod
543 def get_charm(self, charm):
544 """Build and return the path to the test charm.
545
546 Builds one of the charms in tests/charms/layers and returns the path
547 to the compiled charm. The charm will automatically be removed when
548 when the test is complete.
549
550 Returns: The path to the built charm or None if `charm build` failed.
551 """
552
553 # Make sure the charm snap is installed
554 try:
555 subprocess.check_call(['which', 'charm'])
556 except subprocess.CalledProcessError as e:
557 raise Exception("charm snap not installed.")
558
559 if charm not in self.artifacts:
560 try:
561 # Note: This builds the charm under N2VC/tests/charms/builds/
562 # Currently, the snap-installed command only has write access
563 # to the $HOME (changing in an upcoming release) so writing to
564 # /tmp isn't possible at the moment.
565 builds = get_charm_path()
566
567 if not os.path.exists("{}/builds/{}".format(builds, charm)):
568 cmd = "charm build {}/{} -o {}/".format(
569 get_layer_path(),
570 charm,
571 builds,
572 )
573 subprocess.check_call(shlex.split(cmd))
574
575 self.artifacts[charm] = {
576 'tmpdir': builds,
577 'charm': "{}/builds/{}".format(builds, charm),
578 }
579 except subprocess.CalledProcessError as e:
580 raise Exception("charm build failed: {}.".format(e))
581
582 return self.artifacts[charm]['charm']
583
584 @classmethod
585 async def deploy(self, vnf_index, charm, params, loop):
586 """An inner function to do the deployment of a charm from
587 either a vdu or vnf.
588 """
589
590 if not self.n2vc:
591 self.n2vc = get_n2vc(loop=loop)
592
593 application = self.n2vc.FormatApplicationName(
594 self.ns_name,
595 self.vnf_name,
596 str(vnf_index),
597 )
598
599 # Initialize the state of the application
600 self.state[application] = {
601 'status': None, # Juju status
602 'container': None, # lxd container, for proxy charms
603 'actions': {}, # Actions we've executed
604 'done': False, # Are we done testing this charm?
605 'phase': "deploy", # What phase is this application in?
606 }
607
608 debug("Deploying charm at {}".format(self.artifacts[charm]))
609
610 # If this is a native charm, we need to provision the underlying
611 # machine ala an LXC container.
612 machine_spec = {}
613
614 if not self.isproxy(application):
615 debug("Creating container for native charm")
616 # args = ("default", application, None, None)
617 self.state[application]['container'] = create_lxd_container(
618 name=os.path.basename(__file__)
619 )
620
621 hostname = self.get_container_ip(
622 self.state[application]['container'],
623 )
624
625 machine_spec = {
626 'host': hostname,
627 'user': 'ubuntu',
628 }
629
630 await self.n2vc.DeployCharms(
631 self.ns_name,
632 application,
633 self.vnfd,
634 self.get_charm(charm),
635 params,
636 machine_spec,
637 self.n2vc_callback,
638 )
639
640 @classmethod
641 def parse_vnf_descriptor(self):
642 """Parse the VNF descriptor to make running tests easier.
643
644 Parse the charm information in the descriptor to make it easy to write
645 tests to run again it.
646
647 Each charm becomes a dictionary in a list:
648 [
649 'is-proxy': True,
650 'vnf-member-index': 1,
651 'vnf-name': '',
652 'charm-name': '',
653 'initial-config-primitive': {},
654 'config-primitive': {}
655 ]
656 - charm name
657 - is this a proxy charm?
658 - what are the initial-config-primitives (day 1)?
659 - what are the config primitives (day 2)?
660
661 """
662 charms = {}
663
664 # You'd think this would be explicit, but it's just an incremental
665 # value that should be consistent.
666 vnf_member_index = 0
667
668 """Get all vdu and/or vdu config in a descriptor."""
669 config = self.get_config()
670 for cfg in config:
671 if 'juju' in cfg:
672
673 # Get the name to be used for the deployed application
674 application_name = n2vc.vnf.N2VC().FormatApplicationName(
675 self.ns_name,
676 self.vnf_name,
677 str(vnf_member_index),
678 )
679
680 charm = {
681 'application-name': application_name,
682 'proxy': True,
683 'vnf-member-index': vnf_member_index,
684 'vnf-name': self.vnf_name,
685 'name': None,
686 'initial-config-primitive': {},
687 'config-primitive': {},
688 }
689
690 juju = cfg['juju']
691 charm['name'] = juju['charm']
692
693 if 'proxy' in juju:
694 charm['proxy'] = juju['proxy']
695
696 if 'initial-config-primitive' in cfg:
697 charm['initial-config-primitive'] = \
698 cfg['initial-config-primitive']
699
700 if 'config-primitive' in cfg:
701 charm['config-primitive'] = cfg['config-primitive']
702
703 charms[application_name] = charm
704
705 # Increment the vnf-member-index
706 vnf_member_index += 1
707
708 self.charms = charms
709
710 @classmethod
711 def isproxy(self, application_name):
712
713 assert application_name in self.charms
714 assert 'proxy' in self.charms[application_name]
715 assert type(self.charms[application_name]['proxy']) is bool
716
717 # debug(self.charms[application_name])
718 return self.charms[application_name]['proxy']
719
720 @classmethod
721 def get_config(self):
722 """Return an iterable list of config items (vdu and vnf).
723
724 As far as N2VC is concerned, the config section for vdu and vnf are
725 identical. This joins them together so tests only need to iterate
726 through one list.
727 """
728 configs = []
729
730 """Get all vdu and/or vdu config in a descriptor."""
731 vnf_config = self.vnfd.get("vnf-configuration")
732 if vnf_config:
733 juju = vnf_config['juju']
734 if juju:
735 configs.append(vnf_config)
736
737 for vdu in self.vnfd['vdu']:
738 vdu_config = vdu.get('vdu-configuration')
739 if vdu_config:
740 juju = vdu_config['juju']
741 if juju:
742 configs.append(vdu_config)
743
744 return configs
745
746 @classmethod
747 def get_charm_names(self):
748 """Return a list of charms used by the test descriptor."""
749
750 charms = {}
751
752 # Check if the VDUs in this VNF have a charm
753 for config in self.get_config():
754 juju = config['juju']
755
756 name = juju['charm']
757 if name not in charms:
758 charms[name] = 1
759
760 return charms.keys()
761
762 @classmethod
763 def get_phase(self, application):
764 return self.state[application]['phase']
765
766 @classmethod
767 def set_phase(self, application, phase):
768 self.state[application]['phase'] = phase
769
770 @classmethod
771 async def configure_proxy_charm(self, *args):
772 """Configure a container for use via ssh."""
773 (model, application, _, _) = args
774
775 try:
776 if self.get_phase(application) == "deploy":
777 self.set_phase(application, "configure")
778
779 debug("Start CreateContainer for {}".format(application))
780 self.state[application]['container'] = \
781 await self.CreateContainer(*args)
782 debug("Done CreateContainer for {}".format(application))
783
784 if self.state[application]['container']:
785 debug("Configure {} for container".format(application))
786 if await self.configure_ssh_proxy(application):
787 await asyncio.sleep(0.1)
788 return True
789 else:
790 debug("Failed to configure container for {}".format(application))
791 else:
792 debug("skipping CreateContainer for {}: {}".format(
793 application,
794 self.get_phase(application),
795 ))
796
797 except Exception as ex:
798 debug("configure_proxy_charm exception: {}".format(ex))
799 finally:
800 await asyncio.sleep(0.1)
801
802 return False
803
804 @classmethod
805 async def execute_charm_tests(self, *args):
806 (model, application, _, _) = args
807
808 debug("Executing charm test(s) for {}".format(application))
809
810 if self.state[application]['done']:
811 debug("Trying to execute tests against finished charm...aborting")
812 return False
813
814 try:
815 phase = self.get_phase(application)
816 # We enter the test phase when after deploy (for native charms) or
817 # configure, for proxy charms.
818 if phase in ["deploy", "configure"]:
819 self.set_phase(application, "test")
820 if self.are_tests_finished():
821 raise Exception("Trying to execute init-config on finished test")
822
823 if await self.execute_initial_config_primitives(application):
824 # check for metrics
825 await self.check_metrics(application)
826
827 debug("Done testing {}".format(application))
828 self.state[application]['done'] = True
829
830 except Exception as ex:
831 debug("Exception in execute_charm_tests: {}".format(ex))
832 finally:
833 await asyncio.sleep(0.1)
834
835 return True
836
837 @classmethod
838 async def CreateContainer(self, *args):
839 """Create a LXD container for use with a proxy charm.abs
840
841 1. Get the public key from the charm via `get-ssh-public-key` action
842 2. Create container with said key injected for the ubuntu user
843
844 Returns a Container object
845 """
846 # Create and configure a LXD container for use with a proxy charm.
847 (model, application, _, _) = args
848
849 debug("[CreateContainer] {}".format(args))
850 container = None
851
852 try:
853 # Execute 'get-ssh-public-key' primitive and get returned value
854 uuid = await self.n2vc.ExecutePrimitive(
855 model,
856 application,
857 "get-ssh-public-key",
858 None,
859 )
860
861 result = await self.n2vc.GetPrimitiveOutput(model, uuid)
862 pubkey = result['pubkey']
863
864 container = create_lxd_container(
865 public_key=pubkey,
866 name=os.path.basename(__file__)
867 )
868
869 return container
870 except Exception as ex:
871 debug("Error creating container: {}".format(ex))
872 pass
873
874 return None
875
876 @classmethod
877 async def stop(self):
878 """Stop the test.
879
880 - Remove charms
881 - Stop and delete containers
882 - Logout of N2VC
883
884 TODO: Clean up duplicate code between teardown_class() and stop()
885 """
886 debug("stop() called")
887
888 if self.n2vc and self._running and not self._stopping:
889 self._running = False
890 self._stopping = True
891
892 for application in self.charms:
893 try:
894 await self.n2vc.RemoveCharms(self.model, application)
895
896 while True:
897 # Wait for the application to be removed
898 await asyncio.sleep(10)
899 if not await self.n2vc.HasApplication(
900 self.model,
901 application,
902 ):
903 break
904
905 # Need to wait for the charm to finish, because native charms
906 if self.state[application]['container']:
907 debug("Deleting LXD container...")
908 destroy_lxd_container(
909 self.state[application]['container']
910 )
911 self.state[application]['container'] = None
912 debug("Deleting LXD container...done.")
913 else:
914 debug("No container found for {}".format(application))
915 except Exception as e:
916 debug("Error while deleting container: {}".format(e))
917
918 # Logout of N2VC
919 try:
920 debug("stop(): Logging out of N2VC...")
921 await self.n2vc.logout()
922 self.n2vc = None
923 debug("stop(): Logging out of N2VC...Done.")
924 except Exception as ex:
925 debug(ex)
926
927 # Let the test know we're finished.
928 debug("Marking test as finished.")
929 # self._running = False
930 else:
931 debug("Skipping stop()")
932
933 @classmethod
934 def get_container_ip(self, container):
935 """Return the IPv4 address of container's eth0 interface."""
936 ipaddr = None
937 if container:
938 addresses = container.state().network['eth0']['addresses']
939 # The interface may have more than one address, but we only need
940 # the first one for testing purposes.
941 ipaddr = addresses[0]['address']
942
943 return ipaddr
944
945 @classmethod
946 async def configure_ssh_proxy(self, application, task=None):
947 """Configure the proxy charm to use the lxd container.
948
949 Configure the charm to use a LXD container as it's VNF.
950 """
951 debug("Configuring ssh proxy for {}".format(application))
952
953 mgmtaddr = self.get_container_ip(
954 self.state[application]['container'],
955 )
956
957 debug(
958 "Setting ssh-hostname for {} to {}".format(
959 application,
960 mgmtaddr,
961 )
962 )
963
964 await self.n2vc.ExecutePrimitive(
965 self.model,
966 application,
967 "config",
968 None,
969 params={
970 'ssh-hostname': mgmtaddr,
971 'ssh-username': 'ubuntu',
972 }
973 )
974
975 return True
976
977 @classmethod
978 async def execute_initial_config_primitives(self, application, task=None):
979 debug("Executing initial_config_primitives for {}".format(application))
980 try:
981 init_config = self.charms[application]
982
983 """
984 The initial-config-primitive is run during deploy but may fail
985 on some steps because proxy charm access isn't configured.
986
987 Re-run those actions so we can inspect the status.
988 """
989 uuids = await self.n2vc.ExecuteInitialPrimitives(
990 self.model,
991 application,
992 init_config,
993 )
994
995 """
996 ExecutePrimitives will return a list of uuids. We need to check the
997 status of each. The test continues if all Actions succeed, and
998 fails if any of them fail.
999 """
1000 await self.wait_for_uuids(application, uuids)
1001 debug("Primitives for {} finished.".format(application))
1002
1003 return True
1004 except Exception as ex:
1005 debug("execute_initial_config_primitives exception: {}".format(ex))
1006
1007 return False
1008
1009 @classmethod
1010 async def check_metrics(self, application, task=None):
1011 """Check and run metrics, if present.
1012
1013 Checks to see if metrics are specified by the charm. If so, collects
1014 the metrics.
1015
1016 If no metrics, then mark the test as finished.
1017 """
1018 if has_metrics(self.charms[application]['name']):
1019 debug("Collecting metrics for {}".format(application))
1020
1021 metrics = await self.n2vc.GetMetrics(
1022 self.model,
1023 application,
1024 )
1025
1026 return await self.verify_metrics(application, metrics)
1027
1028 @classmethod
1029 async def verify_metrics(self, application, metrics):
1030 """Verify the charm's metrics.
1031
1032 Verify that the charm has sent metrics successfully.
1033
1034 Stops the test when finished.
1035 """
1036 debug("Verifying metrics for {}: {}".format(application, metrics))
1037
1038 if len(metrics):
1039 return True
1040
1041 else:
1042 # TODO: Ran into a case where it took 9 attempts before metrics
1043 # were available; the controller is slow sometimes.
1044 await asyncio.sleep(30)
1045 return await self.check_metrics(application)
1046
1047 @classmethod
1048 async def wait_for_uuids(self, application, uuids):
1049 """Wait for primitives to execute.
1050
1051 The task will provide a list of uuids representing primitives that are
1052 queued to run.
1053 """
1054 debug("Waiting for uuids for {}: {}".format(application, uuids))
1055 waitfor = len(uuids)
1056 finished = 0
1057
1058 while waitfor > finished:
1059 for uid in uuids:
1060 await asyncio.sleep(10)
1061
1062 if uuid not in self.state[application]['actions']:
1063 self.state[application]['actions'][uid] = "pending"
1064
1065 status = self.state[application]['actions'][uid]
1066
1067 # Have we already marked this as done?
1068 if status in ["pending", "running"]:
1069
1070 debug("Getting status of {} ({})...".format(uid, status))
1071 status = await self.n2vc.GetPrimitiveStatus(
1072 self.model,
1073 uid,
1074 )
1075 debug("...state of {} is {}".format(uid, status))
1076 self.state[application]['actions'][uid] = status
1077
1078 if status in ['completed', 'failed']:
1079 finished += 1
1080
1081 debug("{}/{} actions complete".format(finished, waitfor))
1082
1083 # Wait for the primitive to finish and try again
1084 if waitfor > finished:
1085 debug("Waiting 10s for action to finish...")
1086 await asyncio.sleep(10)
1087
1088 @classmethod
1089 def n2vc_callback(self, *args, **kwargs):
1090 (model, application, status, message) = args
1091 # debug("callback: {}".format(args))
1092
1093 if application not in self.state:
1094 # Initialize the state of the application
1095 self.state[application] = {
1096 'status': None, # Juju status
1097 'container': None, # lxd container, for proxy charms
1098 'actions': {}, # Actions we've executed
1099 'done': False, # Are we done testing this charm?
1100 'phase': "deploy", # What phase is this application in?
1101 }
1102
1103 self.state[application]['status'] = status
1104
1105 if status in ['waiting', 'maintenance', 'unknown']:
1106 # Nothing to do for these
1107 return
1108
1109 debug("callback: {}".format(args))
1110
1111 if self.state[application]['done']:
1112 debug("{} is done".format(application))
1113 return
1114
1115 if status in ["blocked"] and self.isproxy(application):
1116 if self.state[application]['phase'] == "deploy":
1117 debug("Configuring proxy charm for {}".format(application))
1118 asyncio.ensure_future(self.configure_proxy_charm(*args))
1119
1120 elif status in ["active"]:
1121 """When a charm is active, we can assume that it has been properly
1122 configured (not blocked), regardless of if it's a proxy or not.
1123
1124 All primitives should be complete by init_config_primitive
1125 """
1126 asyncio.ensure_future(self.execute_charm_tests(*args))