41fa191425a4e3476469bacfad74a1b8356a7400
[osm/N2VC.git] / tests / base.py
1 #!/usr/bin/env python3
2 import asyncio
3 import datetime
4 import logging
5 import n2vc.vnf
6 import pylxd
7 import pytest
8 import os
9 import shlex
10 import subprocess
11 import time
12 import uuid
13 import yaml
14
15 from juju.controller import Controller
16
17 # Disable InsecureRequestWarning w/LXD
18 import urllib3
19 urllib3.disable_warnings()
20 logging.getLogger("urllib3").setLevel(logging.WARNING)
21
22 here = os.path.dirname(os.path.realpath(__file__))
23
24
25 def is_bootstrapped():
26 result = subprocess.run(['juju', 'switch'], stdout=subprocess.PIPE)
27 return (
28 result.returncode == 0 and len(result.stdout.decode().strip()) > 0)
29
30
31 bootstrapped = pytest.mark.skipif(
32 not is_bootstrapped(),
33 reason='bootstrapped Juju environment required')
34
35
36 class CleanController():
37 """
38 Context manager that automatically connects and disconnects from
39 the currently active controller.
40
41 Note: Unlike CleanModel, this will not create a new controller for you,
42 and an active controller must already be available.
43 """
44 def __init__(self):
45 self._controller = None
46
47 async def __aenter__(self):
48 self._controller = Controller()
49 await self._controller.connect()
50 return self._controller
51
52 async def __aexit__(self, exc_type, exc, tb):
53 await self._controller.disconnect()
54
55
56 def debug(msg):
57 """Format debug messages in a consistent way."""
58 now = datetime.datetime.now()
59
60 # TODO: Decide on the best way to log. Output from `logging.debug` shows up
61 # when a test fails, but print() will always show up when running tox with
62 # `-s`, which is really useful for debugging single tests without having to
63 # insert a False assert to see the log.
64 logging.debug(
65 "[{}] {}".format(now.strftime('%Y-%m-%dT%H:%M:%S'), msg)
66 )
67 print(
68 "[{}] {}".format(now.strftime('%Y-%m-%dT%H:%M:%S'), msg)
69 )
70
71
72 def get_charm_path():
73 return "{}/charms".format(here)
74
75
76 def get_layer_path():
77 return "{}/charms/layers".format(here)
78
79
80 def collect_metrics(application):
81 """Invoke Juju's metrics collector.
82
83 Caveat: this shells out to the `juju collect-metrics` command, rather than
84 making an API call. At the time of writing, that API is not exposed through
85 the client library.
86 """
87
88 try:
89 subprocess.check_call(['juju', 'collect-metrics', application])
90 except subprocess.CalledProcessError as e:
91 raise Exception("Unable to collect metrics: {}".format(e))
92
93
94 def has_metrics(charm):
95 """Check if a charm has metrics defined."""
96 metricsyaml = "{}/{}/metrics.yaml".format(
97 get_layer_path(),
98 charm,
99 )
100 if os.path.exists(metricsyaml):
101 return True
102 return False
103
104
105 def get_descriptor(descriptor):
106 desc = None
107 try:
108 tmp = yaml.load(descriptor)
109
110 # Remove the envelope
111 root = list(tmp.keys())[0]
112 if root == "nsd:nsd-catalog":
113 desc = tmp['nsd:nsd-catalog']['nsd'][0]
114 elif root == "vnfd:vnfd-catalog":
115 desc = tmp['vnfd:vnfd-catalog']['vnfd'][0]
116 except ValueError:
117 assert False
118 return desc
119
120
121 def get_n2vc(loop=None):
122 """Return an instance of N2VC.VNF."""
123 log = logging.getLogger()
124 log.level = logging.DEBUG
125
126 # Extract parameters from the environment in order to run our test
127 vca_host = os.getenv('VCA_HOST', '127.0.0.1')
128 vca_port = os.getenv('VCA_PORT', 17070)
129 vca_user = os.getenv('VCA_USER', 'admin')
130 vca_charms = os.getenv('VCA_CHARMS', None)
131 vca_secret = os.getenv('VCA_SECRET', None)
132
133 client = n2vc.vnf.N2VC(
134 log=log,
135 server=vca_host,
136 port=vca_port,
137 user=vca_user,
138 secret=vca_secret,
139 artifacts=vca_charms,
140 loop=loop
141 )
142 return client
143
144
145 def create_lxd_container(public_key=None, name="test_name"):
146 """
147 Returns a container object
148
149 If public_key isn't set, we'll use the Juju ssh key
150
151 :param public_key: The public key to inject into the container
152 :param name: The name of the test being run
153 """
154 container = None
155
156 # Format name so it's valid
157 name = name.replace("_", "-").replace(".", "")
158
159 client = get_lxd_client()
160 test_machine = "test-{}-{}".format(
161 uuid.uuid4().hex[-4:],
162 name,
163 )
164
165 private_key_path, public_key_path = find_n2vc_ssh_keys()
166
167 try:
168 # create profile w/cloud-init and juju ssh key
169 if not public_key:
170 public_key = ""
171 with open(public_key_path, "r") as f:
172 public_key = f.readline()
173
174 client.profiles.create(
175 test_machine,
176 config={
177 'user.user-data': '#cloud-config\nssh_authorized_keys:\n- {}'.format(public_key)},
178 devices={
179 'root': {'path': '/', 'pool': 'default', 'type': 'disk'},
180 'eth0': {
181 'nictype': 'bridged',
182 'parent': 'lxdbr0',
183 'type': 'nic'
184 }
185 }
186 )
187 except Exception as ex:
188 debug("Error creating lxd profile {}: {}".format(test_machine, ex))
189 raise ex
190
191 try:
192 # create lxc machine
193 config = {
194 'name': test_machine,
195 'source': {
196 'type': 'image',
197 'alias': 'xenial',
198 'mode': 'pull',
199 'protocol': 'simplestreams',
200 'server': 'https://cloud-images.ubuntu.com/releases',
201 },
202 'profiles': [test_machine],
203 }
204 container = client.containers.create(config, wait=True)
205 container.start(wait=True)
206 except Exception as ex:
207 debug("Error creating lxd container {}: {}".format(test_machine, ex))
208 # This is a test-ending failure.
209 raise ex
210
211 def wait_for_network(container, timeout=30):
212 """Wait for eth0 to have an ipv4 address."""
213 starttime = time.time()
214 while(time.time() < starttime + timeout):
215 time.sleep(1)
216 if 'eth0' in container.state().network:
217 addresses = container.state().network['eth0']['addresses']
218 if len(addresses) > 0:
219 if addresses[0]['family'] == 'inet':
220 return addresses[0]
221 return None
222
223 try:
224 wait_for_network(container)
225 except Exception as ex:
226 debug(
227 "Error waiting for container {} network: {}".format(
228 test_machine,
229 ex,
230 )
231 )
232
233 # HACK: We need to give sshd a chance to bind to the interface,
234 # and pylxd's container.execute seems to be broken and fails and/or
235 # hangs trying to properly check if the service is up.
236 (exit_code, stdout, stderr) = container.execute([
237 'ping',
238 '-c', '5', # Wait for 5 ECHO_REPLY
239 '8.8.8.8', # Ping Google's public DNS
240 '-W', '15', # Set a 15 second deadline
241 ])
242 if exit_code > 0:
243 # The network failed
244 raise Exception("Unable to verify container network")
245
246 return container
247
248
249 def destroy_lxd_container(container):
250 """Stop and delete a LXD container.
251
252 Sometimes we see errors talking to LXD -- ephemerial issues like
253 load or a bug that's killed the API. We'll do our best to clean
254 up here, and we should run a cleanup after all tests are finished
255 to remove any extra containers and profiles belonging to us.
256 """
257
258 if type(container) is bool:
259 return
260
261 name = container.name
262 debug("Destroying container {}".format(name))
263
264 client = get_lxd_client()
265
266 def wait_for_stop(timeout=30):
267 """Wait for eth0 to have an ipv4 address."""
268 starttime = time.time()
269 while(time.time() < starttime + timeout):
270 time.sleep(1)
271 if container.state == "Stopped":
272 return
273
274 def wait_for_delete(timeout=30):
275 starttime = time.time()
276 while(time.time() < starttime + timeout):
277 time.sleep(1)
278 if client.containers.exists(name) is False:
279 return
280
281 try:
282 container.stop(wait=False)
283 wait_for_stop()
284 except Exception as ex:
285 debug(
286 "Error stopping container {}: {}".format(
287 name,
288 ex,
289 )
290 )
291
292 try:
293 container.delete(wait=False)
294 wait_for_delete()
295 except Exception as ex:
296 debug(
297 "Error deleting container {}: {}".format(
298 name,
299 ex,
300 )
301 )
302
303 try:
304 # Delete the profile created for this container
305 profile = client.profiles.get(name)
306 if profile:
307 profile.delete()
308 except Exception as ex:
309 debug(
310 "Error deleting profile {}: {}".format(
311 name,
312 ex,
313 )
314 )
315
316
317 def find_lxd_config():
318 """Find the LXD configuration directory."""
319 paths = []
320 paths.append(os.path.expanduser("~/.config/lxc"))
321 paths.append(os.path.expanduser("~/snap/lxd/current/.config/lxc"))
322
323 for path in paths:
324 if os.path.exists(path):
325 crt = os.path.expanduser("{}/client.crt".format(path))
326 key = os.path.expanduser("{}/client.key".format(path))
327 if os.path.exists(crt) and os.path.exists(key):
328 return (crt, key)
329 return (None, None)
330
331
332 def find_n2vc_ssh_keys():
333 """Find the N2VC ssh keys."""
334
335 paths = []
336 paths.append(os.path.expanduser("~/.ssh/"))
337
338 for path in paths:
339 if os.path.exists(path):
340 private = os.path.expanduser("{}/id_n2vc_rsa".format(path))
341 public = os.path.expanduser("{}/id_n2vc_rsa.pub".format(path))
342 if os.path.exists(private) and os.path.exists(public):
343 return (private, public)
344 return (None, None)
345
346
347 def find_juju_ssh_keys():
348 """Find the Juju ssh keys."""
349
350 paths = []
351 paths.append(os.path.expanduser("~/.local/share/juju/ssh/"))
352
353 for path in paths:
354 if os.path.exists(path):
355 private = os.path.expanduser("{}/juju_id_rsa".format(path))
356 public = os.path.expanduser("{}/juju_id_rsa.pub".format(path))
357 if os.path.exists(private) and os.path.exists(public):
358 return (private, public)
359 return (None, None)
360
361
362 def get_juju_private_key():
363 keys = find_juju_ssh_keys()
364 return keys[0]
365
366
367 def get_lxd_client(host="127.0.0.1", port="8443", verify=False):
368 """ Get the LXD client."""
369 client = None
370 (crt, key) = find_lxd_config()
371
372 if crt and key:
373 client = pylxd.Client(
374 endpoint="https://{}:{}".format(host, port),
375 cert=(crt, key),
376 verify=verify,
377 )
378
379 return client
380
381
382 # TODO: This is marked serial but can be run in parallel with work, including:
383 # - Fixing an event loop issue; seems that all tests stop when one test stops?
384
385
386 @pytest.mark.serial
387 class TestN2VC(object):
388 """TODO:
389 1. Validator Validation
390
391 Automatically validate the descriptors we're using here, unless the test author explicitly wants to skip them. Useful to make sure tests aren't being run against invalid descriptors, validating functionality that may fail against a properly written descriptor.
392
393 We need to have a flag (instance variable) that controls this behavior. It may be necessary to skip validation and run against a descriptor implementing features that have not yet been released in the Information Model.
394 """
395
396 """
397 The six phases of integration testing, for the test itself and each charm?:
398
399 setup/teardown_class:
400 1. Prepare - Verify the environment and create a new model
401 2. Deploy - Mark the test as ready to execute
402 3. Configure - Configuration to reach Active state
403 4. Test - Execute primitive(s) to verify success
404 5. Collect - Collect any useful artifacts for debugging (charm, logs)
405 6. Destroy - Destroy the model
406
407
408 1. Prepare - Building of charm
409 2. Deploy - Deploying charm
410 3. Configure - Configuration to reach Active state
411 4. Test - Execute primitive(s) to verify success
412 5. Collect - Collect any useful artifacts for debugging (charm, logs)
413 6. Destroy - Destroy the charm
414
415 """
416 @classmethod
417 def setup_class(self):
418 """ setup any state specific to the execution of the given class (which
419 usually contains tests).
420 """
421 # Initialize instance variable(s)
422 self.n2vc = None
423
424 # Track internal state for each test run
425 self.state = {}
426
427 # Parse the test's descriptors
428 self.nsd = get_descriptor(self.NSD_YAML)
429 self.vnfd = get_descriptor(self.VNFD_YAML)
430
431 self.ns_name = self.nsd['name']
432 self.vnf_name = self.vnfd['name']
433
434 self.charms = {}
435 self.parse_vnf_descriptor()
436 assert self.charms is not {}
437
438 # Track artifacts, like compiled charms, that will need to be removed
439 self.artifacts = {}
440
441 # Build the charm(s) needed for this test
442 for charm in self.get_charm_names():
443 self.get_charm(charm)
444
445 # A bit of a hack, in order to allow the N2VC callback to run parallel
446 # to pytest. Test(s) should wait for this flag to change to False
447 # before returning.
448 self._running = True
449 self._stopping = False
450
451 @classmethod
452 def teardown_class(self):
453 """ teardown any state that was previously setup with a call to
454 setup_class.
455 """
456 debug("Running teardown_class...")
457 try:
458
459 debug("Destroying LXD containers...")
460 for application in self.state:
461 if self.state[application]['container']:
462 destroy_lxd_container(self.state[application]['container'])
463 debug("Destroying LXD containers...done.")
464
465 # Logout of N2VC
466 if self.n2vc:
467 debug("teardown_class(): Logging out of N2VC...")
468 yield from self.n2vc.logout()
469 debug("teardown_class(): Logging out of N2VC...done.")
470
471 debug("Running teardown_class...done.")
472 except Exception as ex:
473 debug("Exception in teardown_class: {}".format(ex))
474
475 @classmethod
476 def all_charms_active(self):
477 """Determine if the all deployed charms are active."""
478 active = 0
479
480 for application in self.state:
481 if 'status' in self.state[application]:
482 debug("status of {} is '{}'".format(
483 application,
484 self.state[application]['status'],
485 ))
486 if self.state[application]['status'] == 'active':
487 active += 1
488
489 debug("Active charms: {}/{}".format(
490 active,
491 len(self.charms),
492 ))
493
494 if active == len(self.charms):
495 return True
496
497 return False
498
499 @classmethod
500 def are_tests_finished(self):
501 appcount = len(self.state)
502
503 # If we don't have state yet, keep running.
504 if appcount == 0:
505 debug("No applications")
506 return False
507
508 if self._stopping:
509 debug("_stopping is True")
510 return True
511
512 appdone = 0
513 for application in self.state:
514 if self.state[application]['done']:
515 appdone += 1
516
517 debug("{}/{} charms tested".format(appdone, appcount))
518
519 if appcount == appdone:
520 return True
521
522 return False
523
524 @classmethod
525 async def running(self, timeout=600):
526 """Returns if the test is still running.
527
528 @param timeout The time, in seconds, to wait for the test to complete.
529 """
530 if self.are_tests_finished():
531 await self.stop()
532 return False
533
534 await asyncio.sleep(30)
535
536 return self._running
537
538 @classmethod
539 def get_charm(self, charm):
540 """Build and return the path to the test charm.
541
542 Builds one of the charms in tests/charms/layers and returns the path
543 to the compiled charm. The charm will automatically be removed when
544 when the test is complete.
545
546 Returns: The path to the built charm or None if `charm build` failed.
547 """
548
549 # Make sure the charm snap is installed
550 try:
551 subprocess.check_call(['which', 'charm'])
552 except subprocess.CalledProcessError:
553 raise Exception("charm snap not installed.")
554
555 if charm not in self.artifacts:
556 try:
557 # Note: This builds the charm under N2VC/tests/charms/builds/
558 # Currently, the snap-installed command only has write access
559 # to the $HOME (changing in an upcoming release) so writing to
560 # /tmp isn't possible at the moment.
561 builds = get_charm_path()
562
563 if not os.path.exists("{}/builds/{}".format(builds, charm)):
564 cmd = "charm build --no-local-layers {}/{} -o {}/".format(
565 get_layer_path(),
566 charm,
567 builds,
568 )
569 subprocess.check_call(shlex.split(cmd))
570
571 self.artifacts[charm] = {
572 'tmpdir': builds,
573 'charm': "{}/builds/{}".format(builds, charm),
574 }
575 except subprocess.CalledProcessError as e:
576 raise Exception("charm build failed: {}.".format(e))
577
578 return self.artifacts[charm]['charm']
579
580 @classmethod
581 async def deploy(self, vnf_index, charm, params, loop):
582 """An inner function to do the deployment of a charm from
583 either a vdu or vnf.
584 """
585
586 if not self.n2vc:
587 self.n2vc = get_n2vc(loop=loop)
588
589 debug("Creating model for Network Service {}".format(self.ns_name))
590 await self.n2vc.CreateNetworkService(self.ns_name)
591
592 application = self.n2vc.FormatApplicationName(
593 self.ns_name,
594 self.vnf_name,
595 str(vnf_index),
596 )
597
598 # Initialize the state of the application
599 self.state[application] = {
600 'status': None, # Juju status
601 'container': None, # lxd container, for proxy charms
602 'actions': {}, # Actions we've executed
603 'done': False, # Are we done testing this charm?
604 'phase': "deploy", # What phase is this application in?
605 }
606
607 debug("Deploying charm at {}".format(self.artifacts[charm]))
608
609 # If this is a native charm, we need to provision the underlying
610 # machine ala an LXC container.
611 machine_spec = {}
612
613 if not self.isproxy(application):
614 debug("Creating container for native charm")
615 # args = ("default", application, None, None)
616 self.state[application]['container'] = create_lxd_container(
617 name=os.path.basename(__file__)
618 )
619
620 hostname = self.get_container_ip(
621 self.state[application]['container'],
622 )
623
624 machine_spec = {
625 'host': hostname,
626 'user': 'ubuntu',
627 }
628
629 await self.n2vc.DeployCharms(
630 self.ns_name,
631 application,
632 self.vnfd,
633 self.get_charm(charm),
634 params,
635 machine_spec,
636 self.n2vc_callback,
637 )
638
639 @classmethod
640 def parse_vnf_descriptor(self):
641 """Parse the VNF descriptor to make running tests easier.
642
643 Parse the charm information in the descriptor to make it easy to write
644 tests to run again it.
645
646 Each charm becomes a dictionary in a list:
647 [
648 'is-proxy': True,
649 'vnf-member-index': 1,
650 'vnf-name': '',
651 'charm-name': '',
652 'initial-config-primitive': {},
653 'config-primitive': {}
654 ]
655 - charm name
656 - is this a proxy charm?
657 - what are the initial-config-primitives (day 1)?
658 - what are the config primitives (day 2)?
659
660 """
661 charms = {}
662
663 # You'd think this would be explicit, but it's just an incremental
664 # value that should be consistent.
665 vnf_member_index = 0
666
667 """Get all vdu and/or vdu config in a descriptor."""
668 config = self.get_config()
669 for cfg in config:
670 if 'juju' in cfg:
671
672 # Get the name to be used for the deployed application
673 application_name = n2vc.vnf.N2VC().FormatApplicationName(
674 self.ns_name,
675 self.vnf_name,
676 str(vnf_member_index),
677 )
678
679 charm = {
680 'application-name': application_name,
681 'proxy': True,
682 'vnf-member-index': vnf_member_index,
683 'vnf-name': self.vnf_name,
684 'name': None,
685 'initial-config-primitive': {},
686 'config-primitive': {},
687 }
688
689 juju = cfg['juju']
690 charm['name'] = juju['charm']
691
692 if 'proxy' in juju:
693 charm['proxy'] = juju['proxy']
694
695 if 'initial-config-primitive' in cfg:
696 charm['initial-config-primitive'] = \
697 cfg['initial-config-primitive']
698
699 if 'config-primitive' in cfg:
700 charm['config-primitive'] = cfg['config-primitive']
701
702 charms[application_name] = charm
703
704 # Increment the vnf-member-index
705 vnf_member_index += 1
706
707 self.charms = charms
708
709 @classmethod
710 def isproxy(self, application_name):
711
712 assert application_name in self.charms
713 assert 'proxy' in self.charms[application_name]
714 assert type(self.charms[application_name]['proxy']) is bool
715
716 # debug(self.charms[application_name])
717 return self.charms[application_name]['proxy']
718
719 @classmethod
720 def get_config(self):
721 """Return an iterable list of config items (vdu and vnf).
722
723 As far as N2VC is concerned, the config section for vdu and vnf are
724 identical. This joins them together so tests only need to iterate
725 through one list.
726 """
727 configs = []
728
729 """Get all vdu and/or vdu config in a descriptor."""
730 vnf_config = self.vnfd.get("vnf-configuration")
731 if vnf_config:
732 juju = vnf_config['juju']
733 if juju:
734 configs.append(vnf_config)
735
736 for vdu in self.vnfd['vdu']:
737 vdu_config = vdu.get('vdu-configuration')
738 if vdu_config:
739 juju = vdu_config['juju']
740 if juju:
741 configs.append(vdu_config)
742
743 return configs
744
745 @classmethod
746 def get_charm_names(self):
747 """Return a list of charms used by the test descriptor."""
748
749 charms = {}
750
751 # Check if the VDUs in this VNF have a charm
752 for config in self.get_config():
753 juju = config['juju']
754
755 name = juju['charm']
756 if name not in charms:
757 charms[name] = 1
758
759 return charms.keys()
760
761 @classmethod
762 def get_phase(self, application):
763 return self.state[application]['phase']
764
765 @classmethod
766 def set_phase(self, application, phase):
767 self.state[application]['phase'] = phase
768
769 @classmethod
770 async def configure_proxy_charm(self, *args):
771 """Configure a container for use via ssh."""
772 (model, application, _, _) = args
773
774 try:
775 if self.get_phase(application) == "deploy":
776 self.set_phase(application, "configure")
777
778 debug("Start CreateContainer for {}".format(application))
779 self.state[application]['container'] = \
780 await self.CreateContainer(*args)
781 debug("Done CreateContainer for {}".format(application))
782
783 if self.state[application]['container']:
784 debug("Configure {} for container".format(application))
785 if await self.configure_ssh_proxy(application):
786 await asyncio.sleep(0.1)
787 return True
788 else:
789 debug("Failed to configure container for {}".format(application))
790 else:
791 debug("skipping CreateContainer for {}: {}".format(
792 application,
793 self.get_phase(application),
794 ))
795
796 except Exception as ex:
797 debug("configure_proxy_charm exception: {}".format(ex))
798 finally:
799 await asyncio.sleep(0.1)
800
801 return False
802
803 @classmethod
804 async def execute_charm_tests(self, *args):
805 (model, application, _, _) = args
806
807 debug("Executing charm test(s) for {}".format(application))
808
809 if self.state[application]['done']:
810 debug("Trying to execute tests against finished charm...aborting")
811 return False
812
813 try:
814 phase = self.get_phase(application)
815 # We enter the test phase when after deploy (for native charms) or
816 # configure, for proxy charms.
817 if phase in ["deploy", "configure"]:
818 self.set_phase(application, "test")
819 if self.are_tests_finished():
820 raise Exception("Trying to execute init-config on finished test")
821
822 if await self.execute_initial_config_primitives(application):
823 # check for metrics
824 await self.check_metrics(application)
825
826 debug("Done testing {}".format(application))
827 self.state[application]['done'] = True
828
829 except Exception as ex:
830 debug("Exception in execute_charm_tests: {}".format(ex))
831 finally:
832 await asyncio.sleep(0.1)
833
834 return True
835
836 @classmethod
837 async def CreateContainer(self, *args):
838 """Create a LXD container for use with a proxy charm.abs
839
840 1. Get the public key from the charm via `get-ssh-public-key` action
841 2. Create container with said key injected for the ubuntu user
842
843 Returns a Container object
844 """
845 # Create and configure a LXD container for use with a proxy charm.
846 (model, application, _, _) = args
847
848 debug("[CreateContainer] {}".format(args))
849 container = None
850
851 try:
852 # Execute 'get-ssh-public-key' primitive and get returned value
853 uuid = await self.n2vc.ExecutePrimitive(
854 model,
855 application,
856 "get-ssh-public-key",
857 None,
858 )
859
860 result = await self.n2vc.GetPrimitiveOutput(model, uuid)
861 pubkey = result['pubkey']
862
863 container = create_lxd_container(
864 public_key=pubkey,
865 name=os.path.basename(__file__)
866 )
867
868 return container
869 except Exception as ex:
870 debug("Error creating container: {}".format(ex))
871 pass
872
873 return None
874
875 @classmethod
876 async def stop(self):
877 """Stop the test.
878
879 - Remove charms
880 - Stop and delete containers
881 - Logout of N2VC
882
883 TODO: Clean up duplicate code between teardown_class() and stop()
884 """
885 debug("stop() called")
886
887 if self.n2vc and self._running and not self._stopping:
888 self._running = False
889 self._stopping = True
890
891 for application in self.charms:
892 try:
893 await self.n2vc.RemoveCharms(self.ns_name, application)
894
895 await self.n2vc.DestroyNetworkService(self.ns_name)
896
897 while True:
898 # Wait for the application to be removed
899 await asyncio.sleep(10)
900 if not await self.n2vc.HasApplication(
901 self.ns_name,
902 application,
903 ):
904 break
905
906 # Need to wait for the charm to finish, because native charms
907 if self.state[application]['container']:
908 debug("Deleting LXD container...")
909 destroy_lxd_container(
910 self.state[application]['container']
911 )
912 self.state[application]['container'] = None
913 debug("Deleting LXD container...done.")
914 else:
915 debug("No container found for {}".format(application))
916 except Exception as e:
917 debug("Error while deleting container: {}".format(e))
918
919 # Logout of N2VC
920 try:
921 debug("stop(): Logging out of N2VC...")
922 await self.n2vc.logout()
923 self.n2vc = None
924 debug("stop(): Logging out of N2VC...Done.")
925 except Exception as ex:
926 debug(ex)
927
928 # Let the test know we're finished.
929 debug("Marking test as finished.")
930 # self._running = False
931 else:
932 debug("Skipping stop()")
933
934 @classmethod
935 def get_container_ip(self, container):
936 """Return the IPv4 address of container's eth0 interface."""
937 ipaddr = None
938 if container:
939 addresses = container.state().network['eth0']['addresses']
940 # The interface may have more than one address, but we only need
941 # the first one for testing purposes.
942 ipaddr = addresses[0]['address']
943
944 return ipaddr
945
946 @classmethod
947 async def configure_ssh_proxy(self, application, task=None):
948 """Configure the proxy charm to use the lxd container.
949
950 Configure the charm to use a LXD container as it's VNF.
951 """
952 debug("Configuring ssh proxy for {}".format(application))
953
954 mgmtaddr = self.get_container_ip(
955 self.state[application]['container'],
956 )
957
958 debug(
959 "Setting ssh-hostname for {} to {}".format(
960 application,
961 mgmtaddr,
962 )
963 )
964
965 await self.n2vc.ExecutePrimitive(
966 self.ns_name,
967 application,
968 "config",
969 None,
970 params={
971 'ssh-hostname': mgmtaddr,
972 'ssh-username': 'ubuntu',
973 }
974 )
975
976 return True
977
978 @classmethod
979 async def execute_initial_config_primitives(self, application, task=None):
980 debug("Executing initial_config_primitives for {}".format(application))
981 try:
982 init_config = self.charms[application]
983
984 """
985 The initial-config-primitive is run during deploy but may fail
986 on some steps because proxy charm access isn't configured.
987
988 Re-run those actions so we can inspect the status.
989 """
990 uuids = await self.n2vc.ExecuteInitialPrimitives(
991 self.ns_name,
992 application,
993 init_config,
994 )
995
996 """
997 ExecutePrimitives will return a list of uuids. We need to check the
998 status of each. The test continues if all Actions succeed, and
999 fails if any of them fail.
1000 """
1001 await self.wait_for_uuids(application, uuids)
1002 debug("Primitives for {} finished.".format(application))
1003
1004 return True
1005 except Exception as ex:
1006 debug("execute_initial_config_primitives exception: {}".format(ex))
1007
1008 return False
1009
1010 @classmethod
1011 async def check_metrics(self, application, task=None):
1012 """Check and run metrics, if present.
1013
1014 Checks to see if metrics are specified by the charm. If so, collects
1015 the metrics.
1016
1017 If no metrics, then mark the test as finished.
1018 """
1019 if has_metrics(self.charms[application]['name']):
1020 debug("Collecting metrics for {}".format(application))
1021
1022 metrics = await self.n2vc.GetMetrics(
1023 self.ns_name,
1024 application,
1025 )
1026
1027 return await self.verify_metrics(application, metrics)
1028
1029 @classmethod
1030 async def verify_metrics(self, application, metrics):
1031 """Verify the charm's metrics.
1032
1033 Verify that the charm has sent metrics successfully.
1034
1035 Stops the test when finished.
1036 """
1037 debug("Verifying metrics for {}: {}".format(application, metrics))
1038
1039 if len(metrics):
1040 return True
1041
1042 else:
1043 # TODO: Ran into a case where it took 9 attempts before metrics
1044 # were available; the controller is slow sometimes.
1045 await asyncio.sleep(30)
1046 return await self.check_metrics(application)
1047
1048 @classmethod
1049 async def wait_for_uuids(self, application, uuids):
1050 """Wait for primitives to execute.
1051
1052 The task will provide a list of uuids representing primitives that are
1053 queued to run.
1054 """
1055 debug("Waiting for uuids for {}: {}".format(application, uuids))
1056 waitfor = len(uuids)
1057 finished = 0
1058
1059 while waitfor > finished:
1060 for uid in uuids:
1061 await asyncio.sleep(10)
1062
1063 if uuid not in self.state[application]['actions']:
1064 self.state[application]['actions'][uid] = "pending"
1065
1066 status = self.state[application]['actions'][uid]
1067
1068 # Have we already marked this as done?
1069 if status in ["pending", "running"]:
1070
1071 debug("Getting status of {} ({})...".format(uid, status))
1072 status = await self.n2vc.GetPrimitiveStatus(
1073 self.ns_name,
1074 uid,
1075 )
1076 debug("...state of {} is {}".format(uid, status))
1077 self.state[application]['actions'][uid] = status
1078
1079 if status in ['completed', 'failed']:
1080 finished += 1
1081
1082 debug("{}/{} actions complete".format(finished, waitfor))
1083
1084 # Wait for the primitive to finish and try again
1085 if waitfor > finished:
1086 debug("Waiting 10s for action to finish...")
1087 await asyncio.sleep(10)
1088
1089 @classmethod
1090 def n2vc_callback(self, *args, **kwargs):
1091 (model, application, status, message) = args
1092 # debug("callback: {}".format(args))
1093
1094 if application not in self.state:
1095 # Initialize the state of the application
1096 self.state[application] = {
1097 'status': None, # Juju status
1098 'container': None, # lxd container, for proxy charms
1099 'actions': {}, # Actions we've executed
1100 'done': False, # Are we done testing this charm?
1101 'phase': "deploy", # What phase is this application in?
1102 }
1103
1104 self.state[application]['status'] = status
1105
1106 if status in ['waiting', 'maintenance', 'unknown']:
1107 # Nothing to do for these
1108 return
1109
1110 debug("callback: {}".format(args))
1111
1112 if self.state[application]['done']:
1113 debug("{} is done".format(application))
1114 return
1115
1116 if status in ["blocked"] and self.isproxy(application):
1117 if self.state[application]['phase'] == "deploy":
1118 debug("Configuring proxy charm for {}".format(application))
1119 asyncio.ensure_future(self.configure_proxy_charm(*args))
1120
1121 elif status in ["active"]:
1122 """When a charm is active, we can assume that it has been properly
1123 configured (not blocked), regardless of if it's a proxy or not.
1124
1125 All primitives should be complete by init_config_primitive
1126 """
1127 asyncio.ensure_future(self.execute_charm_tests(*args))