tests/base.py

   1 #!/usr/bin/env python3
   2 import asyncio
   3 import datetime
   4 import logging
   5 import n2vc.vnf
   6 import pylxd
   7 import pytest
   8 import os
   9 import shlex
  10 import subprocess
  11 import time
  12 import uuid
  13 import yaml
  14
  15 from juju.controller import Controller
  16
  17 # Disable InsecureRequestWarning w/LXD
  18 import urllib3
  19 urllib3.disable_warnings()
  20 logging.getLogger("urllib3").setLevel(logging.WARNING)
  21
  22 here = os.path.dirname(os.path.realpath(__file__))
  23
  24
  25 def is_bootstrapped():
  26     result = subprocess.run(['juju', 'switch'], stdout=subprocess.PIPE)
  27     return (
  28         result.returncode == 0 and
  29         len(result.stdout.decode().strip()) > 0)
  30
  31
  32 bootstrapped = pytest.mark.skipif(
  33     not is_bootstrapped(),
  34     reason='bootstrapped Juju environment required')
  35
  36
  37 class CleanController():
  38     """
  39     Context manager that automatically connects and disconnects from
  40     the currently active controller.
  41
  42     Note: Unlike CleanModel, this will not create a new controller for you,
  43     and an active controller must already be available.
  44     """
  45     def __init__(self):
  46         self._controller = None
  47
  48     async def __aenter__(self):
  49         self._controller = Controller()
  50         await self._controller.connect()
  51         return self._controller
  52
  53     async def __aexit__(self, exc_type, exc, tb):
  54         await self._controller.disconnect()
  55
  56
  57 def debug(msg):
  58     """Format debug messages in a consistent way."""
  59     now = datetime.datetime.now()
  60
  61     # TODO: Decide on the best way to log. Output from `logging.debug` shows up
  62     # when a test fails, but print() will always show up when running tox with
  63     # `-s`, which is really useful for debugging single tests without having to
  64     # insert a False assert to see the log.
  65     logging.debug(
  66         "[{}] {}".format(now.strftime('%Y-%m-%dT%H:%M:%S'), msg)
  67     )
  68     # print(
  69     #     "[{}] {}".format(now.strftime('%Y-%m-%dT%H:%M:%S'), msg)
  70     # )
  71
  72
  73 def get_charm_path():
  74     return "{}/charms".format(here)
  75
  76
  77 def get_layer_path():
  78     return "{}/charms/layers".format(here)
  79
  80
  81 def collect_metrics(application):
  82     """Invoke Juju's metrics collector.
  83
  84     Caveat: this shells out to the `juju collect-metrics` command, rather than
  85     making an API call. At the time of writing, that API is not exposed through
  86     the client library.
  87     """
  88
  89     try:
  90         subprocess.check_call(['juju', 'collect-metrics', application])
  91     except subprocess.CalledProcessError as e:
  92         raise Exception("Unable to collect metrics: {}".format(e))
  93
  94
  95 def has_metrics(charm):
  96     """Check if a charm has metrics defined."""
  97     metricsyaml = "{}/{}/metrics.yaml".format(
  98         get_layer_path(),
  99         charm,
 100     )
 101     if os.path.exists(metricsyaml):
 102         return True
 103     return False
 104
 105
 106 def get_descriptor(descriptor):
 107     desc = None
 108     try:
 109         tmp = yaml.load(descriptor)
 110
 111         # Remove the envelope
 112         root = list(tmp.keys())[0]
 113         if root == "nsd:nsd-catalog":
 114             desc = tmp['nsd:nsd-catalog']['nsd'][0]
 115         elif root == "vnfd:vnfd-catalog":
 116             desc = tmp['vnfd:vnfd-catalog']['vnfd'][0]
 117     except ValueError:
 118         assert False
 119     return desc
 120
 121
 122 def get_n2vc(loop=None):
 123     """Return an instance of N2VC.VNF."""
 124     log = logging.getLogger()
 125     log.level = logging.DEBUG
 126
 127     # Extract parameters from the environment in order to run our test
 128     vca_host = os.getenv('VCA_HOST', '127.0.0.1')
 129     vca_port = os.getenv('VCA_PORT', 17070)
 130     vca_user = os.getenv('VCA_USER', 'admin')
 131     vca_charms = os.getenv('VCA_CHARMS', None)
 132     vca_secret = os.getenv('VCA_SECRET', None)
 133
 134     client = n2vc.vnf.N2VC(
 135         log=log,
 136         server=vca_host,
 137         port=vca_port,
 138         user=vca_user,
 139         secret=vca_secret,
 140         artifacts=vca_charms,
 141         loop=loop
 142     )
 143     return client
 144
 145
 146 def create_lxd_container(public_key=None, name="test_name"):
 147     """
 148     Returns a container object
 149
 150     If public_key isn't set, we'll use the Juju ssh key
 151
 152     :param public_key: The public key to inject into the container
 153     :param name: The name of the test being run
 154     """
 155     container = None
 156
 157     # Format name so it's valid
 158     name = name.replace("_", "-").replace(".", "")
 159
 160     client = get_lxd_client()
 161     test_machine = "test-{}-{}".format(
 162         uuid.uuid4().hex[-4:],
 163         name,
 164     )
 165
 166     private_key_path, public_key_path = find_juju_ssh_keys()
 167
 168     try:
 169         # create profile w/cloud-init and juju ssh key
 170         if not public_key:
 171             public_key = ""
 172             with open(public_key_path, "r") as f:
 173                 public_key = f.readline()
 174
 175         client.profiles.create(
 176             test_machine,
 177             config={
 178                 'user.user-data': '#cloud-config\nssh_authorized_keys:\n- {}'.format(public_key)},
 179             devices={
 180                 'root': {'path': '/', 'pool': 'default', 'type': 'disk'},
 181                 'eth0': {
 182                     'nictype': 'bridged',
 183                     'parent': 'lxdbr0',
 184                     'type': 'nic'
 185                 }
 186             }
 187         )
 188     except Exception as ex:
 189         debug("Error creating lxd profile {}: {}".format(test_machine, ex))
 190         raise ex
 191
 192     try:
 193         # create lxc machine
 194         config = {
 195             'name': test_machine,
 196             'source': {
 197                 'type': 'image',
 198                 'alias': 'xenial',
 199                 'mode': 'pull',
 200                 'protocol': 'simplestreams',
 201                 'server': 'https://cloud-images.ubuntu.com/releases',
 202             },
 203             'profiles': [test_machine],
 204         }
 205         container = client.containers.create(config, wait=True)
 206         container.start(wait=True)
 207     except Exception as ex:
 208         debug("Error creating lxd container {}: {}".format(test_machine, ex))
 209         # This is a test-ending failure.
 210         raise ex
 211
 212     def wait_for_network(container, timeout=30):
 213         """Wait for eth0 to have an ipv4 address."""
 214         starttime = time.time()
 215         while(time.time() < starttime + timeout):
 216             time.sleep(1)
 217             if 'eth0' in container.state().network:
 218                 addresses = container.state().network['eth0']['addresses']
 219                 if len(addresses) > 0:
 220                     if addresses[0]['family'] == 'inet':
 221                         return addresses[0]
 222         return None
 223
 224     try:
 225         wait_for_network(container)
 226     except Exception as ex:
 227         debug(
 228             "Error waiting for container {} network: {}".format(
 229                 test_machine,
 230                 ex,
 231             )
 232         )
 233
 234     # HACK: We need to give sshd a chance to bind to the interface,
 235     # and pylxd's container.execute seems to be broken and fails and/or
 236     # hangs trying to properly check if the service is up.
 237     (exit_code, stdout, stderr) = container.execute([
 238         'ping',
 239         '-c', '5',   # Wait for 5 ECHO_REPLY
 240         '8.8.8.8',   # Ping Google's public DNS
 241         '-W', '15',  # Set a 15 second deadline
 242     ])
 243     if exit_code > 0:
 244         # The network failed
 245         raise Exception("Unable to verify container network")
 246
 247     return container
 248
 249
 250 def destroy_lxd_container(container):
 251     """Stop and delete a LXD container.
 252
 253     Sometimes we see errors talking to LXD -- ephemerial issues like
 254     load or a bug that's killed the API. We'll do our best to clean
 255     up here, and we should run a cleanup after all tests are finished
 256     to remove any extra containers and profiles belonging to us.
 257     """
 258
 259     if type(container) is bool:
 260         return
 261
 262     name = container.name
 263     debug("Destroying container {}".format(name))
 264
 265     client = get_lxd_client()
 266
 267     def wait_for_stop(timeout=30):
 268         """Wait for eth0 to have an ipv4 address."""
 269         starttime = time.time()
 270         while(time.time() < starttime + timeout):
 271             time.sleep(1)
 272             if container.state == "Stopped":
 273                 return
 274
 275     def wait_for_delete(timeout=30):
 276         starttime = time.time()
 277         while(time.time() < starttime + timeout):
 278             time.sleep(1)
 279             if client.containers.exists(name) is False:
 280                 return
 281
 282     try:
 283         container.stop(wait=False)
 284         wait_for_stop()
 285     except Exception as ex:
 286         debug(
 287             "Error stopping container {}: {}".format(
 288                 name,
 289                 ex,
 290             )
 291         )
 292
 293     try:
 294         container.delete(wait=False)
 295         wait_for_delete()
 296     except Exception as ex:
 297         debug(
 298             "Error deleting container {}: {}".format(
 299                 name,
 300                 ex,
 301             )
 302         )
 303
 304     try:
 305         # Delete the profile created for this container
 306         profile = client.profiles.get(name)
 307         if profile:
 308             profile.delete()
 309     except Exception as ex:
 310         debug(
 311             "Error deleting profile {}: {}".format(
 312                 name,
 313                 ex,
 314             )
 315         )
 316
 317
 318 def find_lxd_config():
 319     """Find the LXD configuration directory."""
 320     paths = []
 321     paths.append(os.path.expanduser("~/.config/lxc"))
 322     paths.append(os.path.expanduser("~/snap/lxd/current/.config/lxc"))
 323
 324     for path in paths:
 325         if os.path.exists(path):
 326             crt = os.path.expanduser("{}/client.crt".format(path))
 327             key = os.path.expanduser("{}/client.key".format(path))
 328             if os.path.exists(crt) and os.path.exists(key):
 329                 return (crt, key)
 330     return (None, None)
 331
 332
 333 def find_juju_ssh_keys():
 334     """Find the Juju ssh keys."""
 335
 336     paths = []
 337     paths.append(os.path.expanduser("~/.local/share/juju/ssh/"))
 338
 339     for path in paths:
 340         if os.path.exists(path):
 341             private = os.path.expanduser("{}/juju_id_rsa".format(path))
 342             public = os.path.expanduser("{}/juju_id_rsa.pub".format(path))
 343             if os.path.exists(private) and os.path.exists(public):
 344                 return (private, public)
 345     return (None, None)
 346
 347
 348 def get_juju_private_key():
 349     keys = find_juju_ssh_keys()
 350     return keys[0]
 351
 352
 353 def get_lxd_client(host="127.0.0.1", port="8443", verify=False):
 354     """ Get the LXD client."""
 355     client = None
 356     (crt, key) = find_lxd_config()
 357
 358     if crt and key:
 359         client = pylxd.Client(
 360             endpoint="https://{}:{}".format(host, port),
 361             cert=(crt, key),
 362             verify=verify,
 363         )
 364
 365     return client
 366
 367
 368 # TODO: This is marked serial but can be run in parallel with work, including:
 369 # - Fixing an event loop issue; seems that all tests stop when one test stops?
 370
 371
 372 @pytest.mark.serial
 373 class TestN2VC(object):
 374     """TODO:
 375     1. Validator Validation
 376
 377     Automatically validate the descriptors we're using here, unless the test author explicitly wants to skip them. Useful to make sure tests aren't being run against invalid descriptors, validating functionality that may fail against a properly written descriptor.
 378
 379     We need to have a flag (instance variable) that controls this behavior. It may be necessary to skip validation and run against a descriptor implementing features that have not yet been released in the Information Model.
 380     """
 381
 382     """
 383     The six phases of integration testing, for the test itself and each charm?:
 384
 385     setup/teardown_class:
 386     1. Prepare      - Verify the environment and create a new model
 387     2. Deploy       - Mark the test as ready to execute
 388     3. Configure    - Configuration to reach Active state
 389     4. Test         - Execute primitive(s) to verify success
 390     5. Collect      - Collect any useful artifacts for debugging (charm, logs)
 391     6. Destroy      - Destroy the model
 392
 393
 394     1. Prepare      - Building of charm
 395     2. Deploy       - Deploying charm
 396     3. Configure    - Configuration to reach Active state
 397     4. Test         - Execute primitive(s) to verify success
 398     5. Collect      - Collect any useful artifacts for debugging (charm, logs)
 399     6. Destroy      - Destroy the charm
 400
 401     """
 402     @classmethod
 403     def setup_class(self):
 404         """ setup any state specific to the execution of the given class (which
 405         usually contains tests).
 406         """
 407         # Initialize instance variable(s)
 408         self.n2vc = None
 409
 410         # Track internal state for each test run
 411         self.state = {}
 412
 413         # Parse the test's descriptors
 414         self.nsd = get_descriptor(self.NSD_YAML)
 415         self.vnfd = get_descriptor(self.VNFD_YAML)
 416
 417         self.ns_name = self.nsd['name']
 418         self.vnf_name = self.vnfd['name']
 419
 420         # Hard-coded to default for now, but this may change in the future.
 421         self.model = "default"
 422
 423         self.charms = {}
 424         self.parse_vnf_descriptor()
 425         assert self.charms is not {}
 426
 427         # Track artifacts, like compiled charms, that will need to be removed
 428         self.artifacts = {}
 429
 430         # Build the charm(s) needed for this test
 431         for charm in self.get_charm_names():
 432             self.get_charm(charm)
 433
 434         # A bit of a hack, in order to allow the N2VC callback to run parallel
 435         # to pytest. Test(s) should wait for this flag to change to False
 436         # before returning.
 437         self._running = True
 438         self._stopping = False
 439
 440     @classmethod
 441     def teardown_class(self):
 442         """ teardown any state that was previously setup with a call to
 443         setup_class.
 444         """
 445         debug("Running teardown_class...")
 446         try:
 447             debug("Destroying LXD containers...")
 448             for application in self.state:
 449                 if self.state[application]['container']:
 450                     destroy_lxd_container(self.state[application]['container'])
 451             debug("Destroying LXD containers...done.")
 452
 453             # Logout of N2VC
 454             if self.n2vc:
 455                 debug("Logging out of N2VC...")
 456                 yield from self.n2vc.logout()
 457                 debug("Logging out of N2VC...done.")
 458             debug("Running teardown_class...done.")
 459         except Exception as ex:
 460             debug("Exception in teardown_class: {}".format(ex))
 461
 462     @classmethod
 463     def all_charms_active(self):
 464         """Determine if the all deployed charms are active."""
 465         active = 0
 466
 467         for application in self.state:
 468             if 'status' in self.state[application]:
 469                 debug("status of {} is '{}'".format(
 470                     application,
 471                     self.state[application]['status'],
 472                 ))
 473                 if self.state[application]['status'] == 'active':
 474                     active += 1
 475
 476         debug("Active charms: {}/{}".format(
 477             active,
 478             len(self.charms),
 479         ))
 480
 481         if active == len(self.charms):
 482             return True
 483
 484         return False
 485
 486     @classmethod
 487     def are_tests_finished(self):
 488         appcount = len(self.state)
 489
 490         # If we don't have state yet, keep running.
 491         if appcount == 0:
 492             debug("No applications")
 493             return False
 494
 495         if self._stopping:
 496             debug("_stopping is True")
 497             return True
 498
 499         appdone = 0
 500         for application in self.state:
 501             if self.state[application]['done']:
 502                 appdone += 1
 503
 504         debug("{}/{} charms tested".format(appdone, appcount))
 505
 506         if appcount == appdone:
 507             return True
 508
 509         return False
 510
 511     @classmethod
 512     async def running(self, timeout=600):
 513         """Returns if the test is still running.
 514
 515         @param timeout The time, in seconds, to wait for the test to complete.
 516         """
 517         if self.are_tests_finished():
 518             await self.stop()
 519             return False
 520
 521         await asyncio.sleep(30)
 522
 523         return self._running
 524
 525     @classmethod
 526     def get_charm(self, charm):
 527         """Build and return the path to the test charm.
 528
 529         Builds one of the charms in tests/charms/layers and returns the path
 530         to the compiled charm. The charm will automatically be removed when
 531         when the test is complete.
 532
 533         Returns: The path to the built charm or None if `charm build` failed.
 534         """
 535
 536         # Make sure the charm snap is installed
 537         try:
 538             subprocess.check_call(['which', 'charm'])
 539         except subprocess.CalledProcessError as e:
 540             raise Exception("charm snap not installed.")
 541
 542         if charm not in self.artifacts:
 543             try:
 544                 # Note: This builds the charm under N2VC/tests/charms/builds/
 545                 # Currently, the snap-installed command only has write access
 546                 # to the $HOME (changing in an upcoming release) so writing to
 547                 # /tmp isn't possible at the moment.
 548                 builds = get_charm_path()
 549
 550                 if not os.path.exists("{}/builds/{}".format(builds, charm)):
 551                     cmd = "charm build {}/{} -o {}/".format(
 552                         get_layer_path(),
 553                         charm,
 554                         builds,
 555                     )
 556                     subprocess.check_call(shlex.split(cmd))
 557
 558                 self.artifacts[charm] = {
 559                     'tmpdir': builds,
 560                     'charm': "{}/builds/{}".format(builds, charm),
 561                 }
 562             except subprocess.CalledProcessError as e:
 563                 raise Exception("charm build failed: {}.".format(e))
 564
 565         return self.artifacts[charm]['charm']
 566
 567     @classmethod
 568     async def deploy(self, vnf_index, charm, params, loop):
 569         """An inner function to do the deployment of a charm from
 570         either a vdu or vnf.
 571         """
 572
 573         if not self.n2vc:
 574             self.n2vc = get_n2vc(loop=loop)
 575
 576         vnf_name = self.n2vc.FormatApplicationName(
 577             self.ns_name,
 578             self.vnf_name,
 579             str(vnf_index),
 580         )
 581         debug("Deploying charm at {}".format(self.artifacts[charm]))
 582
 583         await self.n2vc.DeployCharms(
 584             self.ns_name,
 585             vnf_name,
 586             self.vnfd,
 587             self.get_charm(charm),
 588             params,
 589             {},
 590             self.n2vc_callback,
 591         )
 592
 593     @classmethod
 594     def parse_vnf_descriptor(self):
 595         """Parse the VNF descriptor to make running tests easier.
 596
 597         Parse the charm information in the descriptor to make it easy to write
 598         tests to run again it.
 599
 600         Each charm becomes a dictionary in a list:
 601         [
 602             'is-proxy': True,
 603             'vnf-member-index': 1,
 604             'vnf-name': '',
 605             'charm-name': '',
 606             'initial-config-primitive': {},
 607             'config-primitive': {}
 608         ]
 609         - charm name
 610         - is this a proxy charm?
 611         - what are the initial-config-primitives (day 1)?
 612         - what are the config primitives (day 2)?
 613
 614         """
 615         charms = {}
 616
 617         # You'd think this would be explicit, but it's just an incremental
 618         # value that should be consistent.
 619         vnf_member_index = 0
 620
 621         """Get all vdu and/or vdu config in a descriptor."""
 622         config = self.get_config()
 623         for cfg in config:
 624             if 'juju' in cfg:
 625
 626                 # Get the name to be used for the deployed application
 627                 application_name = n2vc.vnf.N2VC().FormatApplicationName(
 628                     self.ns_name,
 629                     self.vnf_name,
 630                     str(vnf_member_index),
 631                 )
 632
 633                 charm = {
 634                     'application-name': application_name,
 635                     'proxy': True,
 636                     'vnf-member-index': vnf_member_index,
 637                     'vnf-name': self.vnf_name,
 638                     'name': None,
 639                     'initial-config-primitive': {},
 640                     'config-primitive': {},
 641                 }
 642
 643                 juju = cfg['juju']
 644                 charm['name'] = juju['charm']
 645
 646                 if 'proxy' in juju:
 647                     charm['proxy'] = juju['proxy']
 648
 649                 if 'initial-config-primitive' in cfg:
 650                     charm['initial-config-primitive'] = \
 651                         cfg['initial-config-primitive']
 652
 653                 if 'config-primitive' in cfg:
 654                     charm['config-primitive'] = cfg['config-primitive']
 655
 656                 charms[application_name] = charm
 657
 658             # Increment the vnf-member-index
 659             vnf_member_index += 1
 660
 661         self.charms = charms
 662
 663     @classmethod
 664     def isproxy(self, application_name):
 665
 666         assert application_name in self.charms
 667         assert 'proxy' in self.charms[application_name]
 668         assert type(self.charms[application_name]['proxy']) is bool
 669
 670         # debug(self.charms[application_name])
 671         return self.charms[application_name]['proxy']
 672
 673     @classmethod
 674     def get_config(self):
 675         """Return an iterable list of config items (vdu and vnf).
 676
 677         As far as N2VC is concerned, the config section for vdu and vnf are
 678         identical. This joins them together so tests only need to iterate
 679         through one list.
 680         """
 681         configs = []
 682
 683         """Get all vdu and/or vdu config in a descriptor."""
 684         vnf_config = self.vnfd.get("vnf-configuration")
 685         if vnf_config:
 686             juju = vnf_config['juju']
 687             if juju:
 688                 configs.append(vnf_config)
 689
 690         for vdu in self.vnfd['vdu']:
 691             vdu_config = vdu.get('vdu-configuration')
 692             if vdu_config:
 693                 juju = vdu_config['juju']
 694                 if juju:
 695                     configs.append(vdu_config)
 696
 697         return configs
 698
 699     @classmethod
 700     def get_charm_names(self):
 701         """Return a list of charms used by the test descriptor."""
 702
 703         charms = {}
 704
 705         # Check if the VDUs in this VNF have a charm
 706         for config in self.get_config():
 707             juju = config['juju']
 708
 709             name = juju['charm']
 710             if name not in charms:
 711                 charms[name] = 1
 712
 713         return charms.keys()
 714
 715     @classmethod
 716     def get_phase(self, application):
 717         return self.state[application]['phase']
 718
 719     @classmethod
 720     def set_phase(self, application, phase):
 721         self.state[application]['phase'] = phase
 722
 723     @classmethod
 724     async def configure_proxy_charm(self, *args):
 725         (model, application, _, _) = args
 726
 727         try:
 728             if self.get_phase(application) == "deploy":
 729                 self.set_phase(application, "configure")
 730
 731                 debug("Start CreateContainer for {}".format(application))
 732                 self.state[application]['container'] = \
 733                     await self.CreateContainer(*args)
 734                 debug("Done CreateContainer for {}".format(application))
 735
 736                 if self.state[application]['container']:
 737                     debug("Configure {} for container".format(application))
 738                     if await self.configure_ssh_proxy(application):
 739                         await asyncio.sleep(0.1)
 740                         return True
 741                 else:
 742                     debug("Failed to configure container for {}".format(application))
 743             else:
 744                 debug("skipping CreateContainer for {}: {}".format(
 745                     application,
 746                     self.get_phase(application),
 747                 ))
 748
 749         except Exception as ex:
 750             debug("configure_proxy_charm exception: {}".format(ex))
 751         finally:
 752             await asyncio.sleep(0.1)
 753
 754         return False
 755
 756     @classmethod
 757     async def execute_charm_tests(self, *args):
 758         (model, application, _, _) = args
 759
 760         debug("Executing charm test(s) for {}".format(application))
 761
 762         if self.state[application]['done']:
 763             debug("Trying to execute tests against finished charm...aborting")
 764             return False
 765
 766         try:
 767             phase = self.get_phase(application)
 768             # We enter the test phase when after deploy (for native charms) or
 769             # configure, for proxy charms.
 770             if phase in ["deploy", "configure"]:
 771                 self.set_phase(application, "test")
 772                 if self.are_tests_finished():
 773                     raise Exception("Trying to execute init-config on finished test")
 774
 775                 if await self.execute_initial_config_primitives(application):
 776                     # check for metrics
 777                     await self.check_metrics(application)
 778
 779                     debug("Done testing {}".format(application))
 780                     self.state[application]['done'] = True
 781
 782         except Exception as ex:
 783             debug("Exception in execute_charm_tests: {}".format(ex))
 784         finally:
 785             await asyncio.sleep(0.1)
 786
 787         return True
 788
 789     @classmethod
 790     async def CreateContainer(self, *args):
 791         """Create a LXD container for use with a proxy charm.abs
 792
 793         1. Get the public key from the charm via `get-ssh-public-key` action
 794         2. Create container with said key injected for the ubuntu user
 795
 796         Returns a Container object
 797         """
 798         # Create and configure a LXD container for use with a proxy charm.
 799         (model, application, _, _) = args
 800
 801         debug("[CreateContainer] {}".format(args))
 802         container = None
 803
 804         try:
 805             # Execute 'get-ssh-public-key' primitive and get returned value
 806             uuid = await self.n2vc.ExecutePrimitive(
 807                 model,
 808                 application,
 809                 "get-ssh-public-key",
 810                 None,
 811             )
 812
 813             result = await self.n2vc.GetPrimitiveOutput(model, uuid)
 814             pubkey = result['pubkey']
 815
 816             container = create_lxd_container(
 817                 public_key=pubkey,
 818                 name=os.path.basename(__file__)
 819             )
 820
 821             return container
 822         except Exception as ex:
 823             debug("Error creating container: {}".format(ex))
 824             pass
 825
 826         return None
 827
 828     @classmethod
 829     async def stop(self):
 830         """Stop the test.
 831
 832         - Remove charms
 833         - Stop and delete containers
 834         - Logout of N2VC
 835
 836         TODO: Clean up duplicate code between teardown_class() and stop()
 837         """
 838         debug("stop() called")
 839
 840         if self.n2vc and self._running and not self._stopping:
 841             self._running = False
 842             self._stopping = True
 843
 844             for application in self.charms:
 845                 try:
 846                     await self.n2vc.RemoveCharms(self.model, application)
 847                     if self.state[application]['container']:
 848                         debug("Deleting LXD container...")
 849                         destroy_lxd_container(
 850                             self.state[application]['container']
 851                         )
 852                         self.state[application]['container'] = None
 853                         debug("Deleting LXD container...done.")
 854                     else:
 855                         debug("No container found for {}".format(application))
 856                 except Exception as e:
 857                     debug("Error while deleting container: {}".format(e))
 858
 859             # Logout of N2VC
 860             try:
 861                 debug("Logging out of N2VC...")
 862                 await self.n2vc.logout()
 863                 self.n2vc = None
 864                 debug("Logging out of N2VC...Done.")
 865             except Exception as ex:
 866                 debug(ex)
 867
 868             # Let the test know we're finished.
 869             debug("Marking test as finished.")
 870             # self._running = False
 871         else:
 872             debug("Skipping stop()")
 873
 874     @classmethod
 875     def get_container_ip(self, container):
 876         """Return the IPv4 address of container's eth0 interface."""
 877         ipaddr = None
 878         if container:
 879             addresses = container.state().network['eth0']['addresses']
 880             # The interface may have more than one address, but we only need
 881             # the first one for testing purposes.
 882             ipaddr = addresses[0]['address']
 883
 884         return ipaddr
 885
 886     @classmethod
 887     async def configure_ssh_proxy(self, application, task=None):
 888         """Configure the proxy charm to use the lxd container.
 889
 890         Configure the charm to use a LXD container as it's VNF.
 891         """
 892         debug("Configuring ssh proxy for {}".format(application))
 893
 894         mgmtaddr = self.get_container_ip(
 895             self.state[application]['container'],
 896         )
 897
 898         debug(
 899             "Setting ssh-hostname for {} to {}".format(
 900                 application,
 901                 mgmtaddr,
 902             )
 903         )
 904
 905         await self.n2vc.ExecutePrimitive(
 906             self.model,
 907             application,
 908             "config",
 909             None,
 910             params={
 911                 'ssh-hostname': mgmtaddr,
 912                 'ssh-username': 'ubuntu',
 913             }
 914         )
 915
 916         return True
 917
 918     @classmethod
 919     async def execute_initial_config_primitives(self, application, task=None):
 920         debug("Executing initial_config_primitives for {}".format(application))
 921         try:
 922             init_config = self.charms[application]
 923
 924             """
 925             The initial-config-primitive is run during deploy but may fail
 926              on some steps because proxy charm access isn't configured.
 927
 928             Re-run those actions so we can inspect the status.
 929             """
 930             uuids = await self.n2vc.ExecuteInitialPrimitives(
 931                 self.model,
 932                 application,
 933                 init_config,
 934             )
 935
 936             """
 937             ExecutePrimitives will return a list of uuids. We need to check the
 938              status of each. The test continues if all Actions succeed, and
 939              fails if any of them fail.
 940             """
 941             await self.wait_for_uuids(application, uuids)
 942             debug("Primitives for {} finished.".format(application))
 943
 944             return True
 945         except Exception as ex:
 946             debug("execute_initial_config_primitives exception: {}".format(ex))
 947
 948         return False
 949
 950     @classmethod
 951     async def check_metrics(self, application, task=None):
 952         """Check and run metrics, if present.
 953
 954         Checks to see if metrics are specified by the charm. If so, collects
 955         the metrics.
 956
 957         If no metrics, then mark the test as finished.
 958         """
 959         if has_metrics(self.charms[application]['name']):
 960             debug("Collecting metrics for {}".format(application))
 961
 962             metrics = await self.n2vc.GetMetrics(
 963                 self.model,
 964                 application,
 965             )
 966
 967             return await self.verify_metrics(application, metrics)
 968
 969     @classmethod
 970     async def verify_metrics(self, application, metrics):
 971         """Verify the charm's metrics.
 972
 973         Verify that the charm has sent metrics successfully.
 974
 975         Stops the test when finished.
 976         """
 977         debug("Verifying metrics for {}: {}".format(application, metrics))
 978
 979         if len(metrics):
 980             return True
 981
 982         else:
 983             # TODO: Ran into a case where it took 9 attempts before metrics
 984             # were available; the controller is slow sometimes.
 985             await asyncio.sleep(30)
 986             return await self.check_metrics(application)
 987
 988     @classmethod
 989     async def wait_for_uuids(self, application, uuids):
 990         """Wait for primitives to execute.
 991
 992         The task will provide a list of uuids representing primitives that are
 993         queued to run.
 994         """
 995         debug("Waiting for uuids for {}: {}".format(application, uuids))
 996         waitfor = len(uuids)
 997         finished = 0
 998
 999         while waitfor > finished:
1000             for uid in uuids:
1001                 await asyncio.sleep(10)
1002
1003                 if uuid not in self.state[application]['actions']:
1004                     self.state[application]['actions'][uid] = "pending"
1005
1006                 status = self.state[application]['actions'][uid]
1007
1008                 # Have we already marked this as done?
1009                 if status in ["pending", "running"]:
1010
1011                     debug("Getting status of {} ({})...".format(uid, status))
1012                     status = await self.n2vc.GetPrimitiveStatus(
1013                         self.model,
1014                         uid,
1015                     )
1016                     debug("...state of {} is {}".format(uid, status))
1017                     self.state[application]['actions'][uid] = status
1018
1019                     if status in ['completed', 'failed']:
1020                         finished += 1
1021
1022             debug("{}/{} actions complete".format(finished, waitfor))
1023
1024             # Wait for the primitive to finish and try again
1025             if waitfor > finished:
1026                 debug("Waiting 10s for action to finish...")
1027                 await asyncio.sleep(10)
1028
1029     @classmethod
1030     def n2vc_callback(self, *args, **kwargs):
1031         (model, application, status, message) = args
1032         # debug("callback: {}".format(args))
1033
1034         if application not in self.state:
1035             # Initialize the state of the application
1036             self.state[application] = {
1037                 'status': None,     # Juju status
1038                 'container': None,  # lxd container, for proxy charms
1039                 'actions': {},      # Actions we've executed
1040                 'done': False,      # Are we done testing this charm?
1041                 'phase': "deploy",  # What phase is this application in?
1042             }
1043
1044         self.state[application]['status'] = status
1045
1046         if status in ['waiting', 'maintenance', 'unknown']:
1047             # Nothing to do for these
1048             return
1049
1050         debug("callback: {}".format(args))
1051
1052         if self.state[application]['done']:
1053             debug("{} is done".format(application))
1054             return
1055
1056         if status in ["blocked"] and self.isproxy(application):
1057             if self.state[application]['phase'] == "deploy":
1058                 debug("Configuring proxy charm for {}".format(application))
1059                 asyncio.ensure_future(self.configure_proxy_charm(*args))
1060
1061         elif status in ["active"]:
1062             """When a charm is active, we can assume that it has been properly
1063             configured (not blocked), regardless of if it's a proxy or not.
1064
1065             All primitives should be complete by init_config_primitive
1066             """
1067             asyncio.ensure_future(self.execute_charm_tests(*args))