tests/base.py

   1 #!/usr/bin/env python3
   2 import asyncio
   3 import datetime
   4 import logging
   5 import n2vc.vnf
   6 import pylxd
   7 import pytest
   8 import os
   9 import shlex
  10 import subprocess
  11 import time
  12 import uuid
  13 import yaml
  14
  15 from juju.controller import Controller
  16
  17 # Disable InsecureRequestWarning w/LXD
  18 import urllib3
  19 urllib3.disable_warnings()
  20 logging.getLogger("urllib3").setLevel(logging.WARNING)
  21
  22 here = os.path.dirname(os.path.realpath(__file__))
  23
  24
  25 def is_bootstrapped():
  26     result = subprocess.run(['juju', 'switch'], stdout=subprocess.PIPE)
  27     return (
  28         result.returncode == 0 and len(result.stdout.decode().strip()) > 0)
  29
  30
  31 bootstrapped = pytest.mark.skipif(
  32     not is_bootstrapped(),
  33     reason='bootstrapped Juju environment required')
  34
  35
  36 class CleanController():
  37     """
  38     Context manager that automatically connects and disconnects from
  39     the currently active controller.
  40
  41     Note: Unlike CleanModel, this will not create a new controller for you,
  42     and an active controller must already be available.
  43     """
  44     def __init__(self):
  45         self._controller = None
  46
  47     async def __aenter__(self):
  48         self._controller = Controller()
  49         await self._controller.connect()
  50         return self._controller
  51
  52     async def __aexit__(self, exc_type, exc, tb):
  53         await self._controller.disconnect()
  54
  55
  56 def debug(msg):
  57     """Format debug messages in a consistent way."""
  58     now = datetime.datetime.now()
  59
  60     # TODO: Decide on the best way to log. Output from `logging.debug` shows up
  61     # when a test fails, but print() will always show up when running tox with
  62     # `-s`, which is really useful for debugging single tests without having to
  63     # insert a False assert to see the log.
  64     logging.debug(
  65         "[{}] {}".format(now.strftime('%Y-%m-%dT%H:%M:%S'), msg)
  66     )
  67     print(
  68         "[{}] {}".format(now.strftime('%Y-%m-%dT%H:%M:%S'), msg)
  69     )
  70
  71
  72 def get_charm_path():
  73     return "{}/charms".format(here)
  74
  75
  76 def get_layer_path():
  77     return "{}/charms/layers".format(here)
  78
  79
  80 def collect_metrics(application):
  81     """Invoke Juju's metrics collector.
  82
  83     Caveat: this shells out to the `juju collect-metrics` command, rather than
  84     making an API call. At the time of writing, that API is not exposed through
  85     the client library.
  86     """
  87
  88     try:
  89         subprocess.check_call(['juju', 'collect-metrics', application])
  90     except subprocess.CalledProcessError as e:
  91         raise Exception("Unable to collect metrics: {}".format(e))
  92
  93
  94 def has_metrics(charm):
  95     """Check if a charm has metrics defined."""
  96     metricsyaml = "{}/{}/metrics.yaml".format(
  97         get_layer_path(),
  98         charm,
  99     )
 100     if os.path.exists(metricsyaml):
 101         return True
 102     return False
 103
 104
 105 def get_descriptor(descriptor):
 106     desc = None
 107     try:
 108         tmp = yaml.load(descriptor)
 109
 110         # Remove the envelope
 111         root = list(tmp.keys())[0]
 112         if root == "nsd:nsd-catalog":
 113             desc = tmp['nsd:nsd-catalog']['nsd'][0]
 114         elif root == "vnfd:vnfd-catalog":
 115             desc = tmp['vnfd:vnfd-catalog']['vnfd'][0]
 116     except ValueError:
 117         assert False
 118     return desc
 119
 120
 121 def get_n2vc(loop=None):
 122     """Return an instance of N2VC.VNF."""
 123     log = logging.getLogger()
 124     log.level = logging.DEBUG
 125
 126     # Extract parameters from the environment in order to run our test
 127     vca_host = os.getenv('VCA_HOST', '127.0.0.1')
 128     vca_port = os.getenv('VCA_PORT', 17070)
 129     vca_user = os.getenv('VCA_USER', 'admin')
 130     vca_charms = os.getenv('VCA_CHARMS', None)
 131     vca_secret = os.getenv('VCA_SECRET', None)
 132
 133     client = n2vc.vnf.N2VC(
 134         log=log,
 135         server=vca_host,
 136         port=vca_port,
 137         user=vca_user,
 138         secret=vca_secret,
 139         artifacts=vca_charms,
 140         loop=loop
 141     )
 142     return client
 143
 144
 145 def create_lxd_container(public_key=None, name="test_name"):
 146     """
 147     Returns a container object
 148
 149     If public_key isn't set, we'll use the Juju ssh key
 150
 151     :param public_key: The public key to inject into the container
 152     :param name: The name of the test being run
 153     """
 154     container = None
 155
 156     # Format name so it's valid
 157     name = name.replace("_", "-").replace(".", "")
 158
 159     client = get_lxd_client()
 160     test_machine = "test-{}-{}".format(
 161         uuid.uuid4().hex[-4:],
 162         name,
 163     )
 164
 165     private_key_path, public_key_path = find_n2vc_ssh_keys()
 166
 167     try:
 168         # create profile w/cloud-init and juju ssh key
 169         if not public_key:
 170             public_key = ""
 171             with open(public_key_path, "r") as f:
 172                 public_key = f.readline()
 173
 174         client.profiles.create(
 175             test_machine,
 176             config={
 177                 'user.user-data': '#cloud-config\nssh_authorized_keys:\n- {}'.format(public_key)},
 178             devices={
 179                 'root': {'path': '/', 'pool': 'default', 'type': 'disk'},
 180                 'eth0': {
 181                     'nictype': 'bridged',
 182                     'parent': 'lxdbr0',
 183                     'type': 'nic'
 184                 }
 185             }
 186         )
 187     except Exception as ex:
 188         debug("Error creating lxd profile {}: {}".format(test_machine, ex))
 189         raise ex
 190
 191     try:
 192         # create lxc machine
 193         config = {
 194             'name': test_machine,
 195             'source': {
 196                 'type': 'image',
 197                 'alias': 'xenial',
 198                 'mode': 'pull',
 199                 'protocol': 'simplestreams',
 200                 'server': 'https://cloud-images.ubuntu.com/releases',
 201             },
 202             'profiles': [test_machine],
 203         }
 204         container = client.containers.create(config, wait=True)
 205         container.start(wait=True)
 206     except Exception as ex:
 207         debug("Error creating lxd container {}: {}".format(test_machine, ex))
 208         # This is a test-ending failure.
 209         raise ex
 210
 211     def wait_for_network(container, timeout=30):
 212         """Wait for eth0 to have an ipv4 address."""
 213         starttime = time.time()
 214         while(time.time() < starttime + timeout):
 215             time.sleep(1)
 216             if 'eth0' in container.state().network:
 217                 addresses = container.state().network['eth0']['addresses']
 218                 if len(addresses) > 0:
 219                     if addresses[0]['family'] == 'inet':
 220                         return addresses[0]
 221         return None
 222
 223     try:
 224         wait_for_network(container)
 225     except Exception as ex:
 226         debug(
 227             "Error waiting for container {} network: {}".format(
 228                 test_machine,
 229                 ex,
 230             )
 231         )
 232
 233     # HACK: We need to give sshd a chance to bind to the interface,
 234     # and pylxd's container.execute seems to be broken and fails and/or
 235     # hangs trying to properly check if the service is up.
 236     (exit_code, stdout, stderr) = container.execute([
 237         'ping',
 238         '-c', '5',   # Wait for 5 ECHO_REPLY
 239         '8.8.8.8',   # Ping Google's public DNS
 240         '-W', '15',  # Set a 15 second deadline
 241     ])
 242     if exit_code > 0:
 243         # The network failed
 244         raise Exception("Unable to verify container network")
 245
 246     return container
 247
 248
 249 def destroy_lxd_container(container):
 250     """Stop and delete a LXD container.
 251
 252     Sometimes we see errors talking to LXD -- ephemerial issues like
 253     load or a bug that's killed the API. We'll do our best to clean
 254     up here, and we should run a cleanup after all tests are finished
 255     to remove any extra containers and profiles belonging to us.
 256     """
 257
 258     if type(container) is bool:
 259         return
 260
 261     name = container.name
 262     debug("Destroying container {}".format(name))
 263
 264     client = get_lxd_client()
 265
 266     def wait_for_stop(timeout=30):
 267         """Wait for eth0 to have an ipv4 address."""
 268         starttime = time.time()
 269         while(time.time() < starttime + timeout):
 270             time.sleep(1)
 271             if container.state == "Stopped":
 272                 return
 273
 274     def wait_for_delete(timeout=30):
 275         starttime = time.time()
 276         while(time.time() < starttime + timeout):
 277             time.sleep(1)
 278             if client.containers.exists(name) is False:
 279                 return
 280
 281     try:
 282         container.stop(wait=False)
 283         wait_for_stop()
 284     except Exception as ex:
 285         debug(
 286             "Error stopping container {}: {}".format(
 287                 name,
 288                 ex,
 289             )
 290         )
 291
 292     try:
 293         container.delete(wait=False)
 294         wait_for_delete()
 295     except Exception as ex:
 296         debug(
 297             "Error deleting container {}: {}".format(
 298                 name,
 299                 ex,
 300             )
 301         )
 302
 303     try:
 304         # Delete the profile created for this container
 305         profile = client.profiles.get(name)
 306         if profile:
 307             profile.delete()
 308     except Exception as ex:
 309         debug(
 310             "Error deleting profile {}: {}".format(
 311                 name,
 312                 ex,
 313             )
 314         )
 315
 316
 317 def find_lxd_config():
 318     """Find the LXD configuration directory."""
 319     paths = []
 320     paths.append(os.path.expanduser("~/.config/lxc"))
 321     paths.append(os.path.expanduser("~/snap/lxd/current/.config/lxc"))
 322
 323     for path in paths:
 324         if os.path.exists(path):
 325             crt = os.path.expanduser("{}/client.crt".format(path))
 326             key = os.path.expanduser("{}/client.key".format(path))
 327             if os.path.exists(crt) and os.path.exists(key):
 328                 return (crt, key)
 329     return (None, None)
 330
 331
 332 def find_n2vc_ssh_keys():
 333     """Find the N2VC ssh keys."""
 334
 335     paths = []
 336     paths.append(os.path.expanduser("~/.ssh/"))
 337
 338     for path in paths:
 339         if os.path.exists(path):
 340             private = os.path.expanduser("{}/id_n2vc_rsa".format(path))
 341             public = os.path.expanduser("{}/id_n2vc_rsa.pub".format(path))
 342             if os.path.exists(private) and os.path.exists(public):
 343                 return (private, public)
 344     return (None, None)
 345
 346
 347 def find_juju_ssh_keys():
 348     """Find the Juju ssh keys."""
 349
 350     paths = []
 351     paths.append(os.path.expanduser("~/.local/share/juju/ssh/"))
 352
 353     for path in paths:
 354         if os.path.exists(path):
 355             private = os.path.expanduser("{}/juju_id_rsa".format(path))
 356             public = os.path.expanduser("{}/juju_id_rsa.pub".format(path))
 357             if os.path.exists(private) and os.path.exists(public):
 358                 return (private, public)
 359     return (None, None)
 360
 361
 362 def get_juju_private_key():
 363     keys = find_juju_ssh_keys()
 364     return keys[0]
 365
 366
 367 def get_lxd_client(host="127.0.0.1", port="8443", verify=False):
 368     """ Get the LXD client."""
 369     client = None
 370     (crt, key) = find_lxd_config()
 371
 372     if crt and key:
 373         client = pylxd.Client(
 374             endpoint="https://{}:{}".format(host, port),
 375             cert=(crt, key),
 376             verify=verify,
 377         )
 378
 379     return client
 380
 381
 382 # TODO: This is marked serial but can be run in parallel with work, including:
 383 # - Fixing an event loop issue; seems that all tests stop when one test stops?
 384
 385
 386 @pytest.mark.serial
 387 class TestN2VC(object):
 388     """TODO:
 389     1. Validator Validation
 390
 391     Automatically validate the descriptors we're using here, unless the test author explicitly wants to skip them. Useful to make sure tests aren't being run against invalid descriptors, validating functionality that may fail against a properly written descriptor.
 392
 393     We need to have a flag (instance variable) that controls this behavior. It may be necessary to skip validation and run against a descriptor implementing features that have not yet been released in the Information Model.
 394     """
 395
 396     """
 397     The six phases of integration testing, for the test itself and each charm?:
 398
 399     setup/teardown_class:
 400     1. Prepare      - Verify the environment and create a new model
 401     2. Deploy       - Mark the test as ready to execute
 402     3. Configure    - Configuration to reach Active state
 403     4. Test         - Execute primitive(s) to verify success
 404     5. Collect      - Collect any useful artifacts for debugging (charm, logs)
 405     6. Destroy      - Destroy the model
 406
 407
 408     1. Prepare      - Building of charm
 409     2. Deploy       - Deploying charm
 410     3. Configure    - Configuration to reach Active state
 411     4. Test         - Execute primitive(s) to verify success
 412     5. Collect      - Collect any useful artifacts for debugging (charm, logs)
 413     6. Destroy      - Destroy the charm
 414
 415     """
 416     @classmethod
 417     def setup_class(self):
 418         """ setup any state specific to the execution of the given class (which
 419         usually contains tests).
 420         """
 421         # Initialize instance variable(s)
 422         self.n2vc = None
 423
 424         # Track internal state for each test run
 425         self.state = {}
 426
 427         # Parse the test's descriptors
 428         self.nsd = get_descriptor(self.NSD_YAML)
 429         self.vnfd = get_descriptor(self.VNFD_YAML)
 430
 431         self.ns_name = self.nsd['name']
 432         self.vnf_name = self.vnfd['name']
 433
 434         self.charms = {}
 435         self.parse_vnf_descriptor()
 436         assert self.charms is not {}
 437
 438         # Track artifacts, like compiled charms, that will need to be removed
 439         self.artifacts = {}
 440
 441         # Build the charm(s) needed for this test
 442         for charm in self.get_charm_names():
 443             self.get_charm(charm)
 444
 445         # A bit of a hack, in order to allow the N2VC callback to run parallel
 446         # to pytest. Test(s) should wait for this flag to change to False
 447         # before returning.
 448         self._running = True
 449         self._stopping = False
 450
 451     @classmethod
 452     def teardown_class(self):
 453         """ teardown any state that was previously setup with a call to
 454         setup_class.
 455         """
 456         debug("Running teardown_class...")
 457         try:
 458
 459             debug("Destroying LXD containers...")
 460             for application in self.state:
 461                 if self.state[application]['container']:
 462                     destroy_lxd_container(self.state[application]['container'])
 463             debug("Destroying LXD containers...done.")
 464
 465             # Logout of N2VC
 466             if self.n2vc:
 467                 debug("teardown_class(): Logging out of N2VC...")
 468                 yield from self.n2vc.logout()
 469                 debug("teardown_class(): Logging out of N2VC...done.")
 470
 471             debug("Running teardown_class...done.")
 472         except Exception as ex:
 473             debug("Exception in teardown_class: {}".format(ex))
 474
 475     @classmethod
 476     def all_charms_active(self):
 477         """Determine if the all deployed charms are active."""
 478         active = 0
 479
 480         for application in self.state:
 481             if 'status' in self.state[application]:
 482                 debug("status of {} is '{}'".format(
 483                     application,
 484                     self.state[application]['status'],
 485                 ))
 486                 if self.state[application]['status'] == 'active':
 487                     active += 1
 488
 489         debug("Active charms: {}/{}".format(
 490             active,
 491             len(self.charms),
 492         ))
 493
 494         if active == len(self.charms):
 495             return True
 496
 497         return False
 498
 499     @classmethod
 500     def are_tests_finished(self):
 501         appcount = len(self.state)
 502
 503         # If we don't have state yet, keep running.
 504         if appcount == 0:
 505             debug("No applications")
 506             return False
 507
 508         if self._stopping:
 509             debug("_stopping is True")
 510             return True
 511
 512         appdone = 0
 513         for application in self.state:
 514             if self.state[application]['done']:
 515                 appdone += 1
 516
 517         debug("{}/{} charms tested".format(appdone, appcount))
 518
 519         if appcount == appdone:
 520             return True
 521
 522         return False
 523
 524     @classmethod
 525     async def running(self, timeout=600):
 526         """Returns if the test is still running.
 527
 528         @param timeout The time, in seconds, to wait for the test to complete.
 529         """
 530         if self.are_tests_finished():
 531             await self.stop()
 532             return False
 533
 534         await asyncio.sleep(30)
 535
 536         return self._running
 537
 538     @classmethod
 539     def get_charm(self, charm):
 540         """Build and return the path to the test charm.
 541
 542         Builds one of the charms in tests/charms/layers and returns the path
 543         to the compiled charm. The charm will automatically be removed when
 544         when the test is complete.
 545
 546         Returns: The path to the built charm or None if `charm build` failed.
 547         """
 548
 549         # Make sure the charm snap is installed
 550         try:
 551             subprocess.check_call(['which', 'charm'])
 552         except subprocess.CalledProcessError:
 553             raise Exception("charm snap not installed.")
 554
 555         if charm not in self.artifacts:
 556             try:
 557                 # Note: This builds the charm under N2VC/tests/charms/builds/
 558                 # Currently, the snap-installed command only has write access
 559                 # to the $HOME (changing in an upcoming release) so writing to
 560                 # /tmp isn't possible at the moment.
 561                 builds = get_charm_path()
 562
 563                 if not os.path.exists("{}/builds/{}".format(builds, charm)):
 564                     cmd = "charm build --no-local-layers {}/{} -o {}/".format(
 565                         get_layer_path(),
 566                         charm,
 567                         builds,
 568                     )
 569                     subprocess.check_call(shlex.split(cmd))
 570
 571             except subprocess.CalledProcessError as e:
 572                 # charm build will return error code 100 if the charm fails
 573                 # the auto-run of charm proof, which we can safely ignore for
 574                 # our CI charms.
 575                 if e.returncode != 100:
 576                     raise Exception("charm build failed: {}.".format(e))
 577
 578             self.artifacts[charm] = {
 579                 'tmpdir': builds,
 580                 'charm': "{}/builds/{}".format(builds, charm),
 581             }
 582
 583         return self.artifacts[charm]['charm']
 584
 585     @classmethod
 586     async def deploy(self, vnf_index, charm, params, loop):
 587         """An inner function to do the deployment of a charm from
 588         either a vdu or vnf.
 589         """
 590
 591         if not self.n2vc:
 592             self.n2vc = get_n2vc(loop=loop)
 593
 594         application = self.n2vc.FormatApplicationName(
 595             self.ns_name,
 596             self.vnf_name,
 597             str(vnf_index),
 598         )
 599
 600         # Initialize the state of the application
 601         self.state[application] = {
 602             'status': None,     # Juju status
 603             'container': None,  # lxd container, for proxy charms
 604             'actions': {},      # Actions we've executed
 605             'done': False,      # Are we done testing this charm?
 606             'phase': "deploy",  # What phase is this application in?
 607         }
 608
 609         debug("Deploying charm at {}".format(self.artifacts[charm]))
 610
 611         # If this is a native charm, we need to provision the underlying
 612         # machine ala an LXC container.
 613         machine_spec = {}
 614
 615         if not self.isproxy(application):
 616             debug("Creating container for native charm")
 617             # args = ("default", application, None, None)
 618             self.state[application]['container'] = create_lxd_container(
 619                 name=os.path.basename(__file__)
 620             )
 621
 622             hostname = self.get_container_ip(
 623                 self.state[application]['container'],
 624             )
 625
 626             machine_spec = {
 627                 'host': hostname,
 628                 'user': 'ubuntu',
 629             }
 630
 631         await self.n2vc.DeployCharms(
 632             self.ns_name,
 633             application,
 634             self.vnfd,
 635             self.get_charm(charm),
 636             params,
 637             machine_spec,
 638             self.n2vc_callback,
 639         )
 640
 641     @classmethod
 642     def parse_vnf_descriptor(self):
 643         """Parse the VNF descriptor to make running tests easier.
 644
 645         Parse the charm information in the descriptor to make it easy to write
 646         tests to run again it.
 647
 648         Each charm becomes a dictionary in a list:
 649         [
 650             'is-proxy': True,
 651             'vnf-member-index': 1,
 652             'vnf-name': '',
 653             'charm-name': '',
 654             'initial-config-primitive': {},
 655             'config-primitive': {}
 656         ]
 657         - charm name
 658         - is this a proxy charm?
 659         - what are the initial-config-primitives (day 1)?
 660         - what are the config primitives (day 2)?
 661
 662         """
 663         charms = {}
 664
 665         # You'd think this would be explicit, but it's just an incremental
 666         # value that should be consistent.
 667         vnf_member_index = 0
 668
 669         """Get all vdu and/or vdu config in a descriptor."""
 670         config = self.get_config()
 671         for cfg in config:
 672             if 'juju' in cfg:
 673
 674                 # Get the name to be used for the deployed application
 675                 application_name = n2vc.vnf.N2VC().FormatApplicationName(
 676                     self.ns_name,
 677                     self.vnf_name,
 678                     str(vnf_member_index),
 679                 )
 680
 681                 charm = {
 682                     'application-name': application_name,
 683                     'proxy': True,
 684                     'vnf-member-index': vnf_member_index,
 685                     'vnf-name': self.vnf_name,
 686                     'name': None,
 687                     'initial-config-primitive': {},
 688                     'config-primitive': {},
 689                 }
 690
 691                 juju = cfg['juju']
 692                 charm['name'] = juju['charm']
 693
 694                 if 'proxy' in juju:
 695                     charm['proxy'] = juju['proxy']
 696
 697                 if 'initial-config-primitive' in cfg:
 698                     charm['initial-config-primitive'] = \
 699                         cfg['initial-config-primitive']
 700
 701                 if 'config-primitive' in cfg:
 702                     charm['config-primitive'] = cfg['config-primitive']
 703
 704                 charms[application_name] = charm
 705
 706             # Increment the vnf-member-index
 707             vnf_member_index += 1
 708
 709         self.charms = charms
 710
 711     @classmethod
 712     def isproxy(self, application_name):
 713
 714         assert application_name in self.charms
 715         assert 'proxy' in self.charms[application_name]
 716         assert type(self.charms[application_name]['proxy']) is bool
 717
 718         # debug(self.charms[application_name])
 719         return self.charms[application_name]['proxy']
 720
 721     @classmethod
 722     def get_config(self):
 723         """Return an iterable list of config items (vdu and vnf).
 724
 725         As far as N2VC is concerned, the config section for vdu and vnf are
 726         identical. This joins them together so tests only need to iterate
 727         through one list.
 728         """
 729         configs = []
 730
 731         """Get all vdu and/or vdu config in a descriptor."""
 732         vnf_config = self.vnfd.get("vnf-configuration")
 733         if vnf_config:
 734             juju = vnf_config['juju']
 735             if juju:
 736                 configs.append(vnf_config)
 737
 738         for vdu in self.vnfd['vdu']:
 739             vdu_config = vdu.get('vdu-configuration')
 740             if vdu_config:
 741                 juju = vdu_config['juju']
 742                 if juju:
 743                     configs.append(vdu_config)
 744
 745         return configs
 746
 747     @classmethod
 748     def get_charm_names(self):
 749         """Return a list of charms used by the test descriptor."""
 750
 751         charms = {}
 752
 753         # Check if the VDUs in this VNF have a charm
 754         for config in self.get_config():
 755             juju = config['juju']
 756
 757             name = juju['charm']
 758             if name not in charms:
 759                 charms[name] = 1
 760
 761         return charms.keys()
 762
 763     @classmethod
 764     def get_phase(self, application):
 765         return self.state[application]['phase']
 766
 767     @classmethod
 768     def set_phase(self, application, phase):
 769         self.state[application]['phase'] = phase
 770
 771     @classmethod
 772     async def configure_proxy_charm(self, *args):
 773         """Configure a container for use via ssh."""
 774         (model, application, _, _) = args
 775
 776         try:
 777             if self.get_phase(application) == "deploy":
 778                 self.set_phase(application, "configure")
 779
 780                 debug("Start CreateContainer for {}".format(application))
 781                 self.state[application]['container'] = \
 782                     await self.CreateContainer(*args)
 783                 debug("Done CreateContainer for {}".format(application))
 784
 785                 if self.state[application]['container']:
 786                     debug("Configure {} for container".format(application))
 787                     if await self.configure_ssh_proxy(application):
 788                         await asyncio.sleep(0.1)
 789                         return True
 790                 else:
 791                     debug("Failed to configure container for {}".format(application))
 792             else:
 793                 debug("skipping CreateContainer for {}: {}".format(
 794                     application,
 795                     self.get_phase(application),
 796                 ))
 797
 798         except Exception as ex:
 799             debug("configure_proxy_charm exception: {}".format(ex))
 800         finally:
 801             await asyncio.sleep(0.1)
 802
 803         return False
 804
 805     @classmethod
 806     async def execute_charm_tests(self, *args):
 807         (model, application, _, _) = args
 808
 809         debug("Executing charm test(s) for {}".format(application))
 810
 811         if self.state[application]['done']:
 812             debug("Trying to execute tests against finished charm...aborting")
 813             return False
 814
 815         try:
 816             phase = self.get_phase(application)
 817             # We enter the test phase when after deploy (for native charms) or
 818             # configure, for proxy charms.
 819             if phase in ["deploy", "configure"]:
 820                 self.set_phase(application, "test")
 821                 if self.are_tests_finished():
 822                     raise Exception("Trying to execute init-config on finished test")
 823
 824                 if await self.execute_initial_config_primitives(application):
 825                     # check for metrics
 826                     await self.check_metrics(application)
 827
 828                     debug("Done testing {}".format(application))
 829                     self.state[application]['done'] = True
 830
 831         except Exception as ex:
 832             debug("Exception in execute_charm_tests: {}".format(ex))
 833         finally:
 834             await asyncio.sleep(0.1)
 835
 836         return True
 837
 838     @classmethod
 839     async def CreateContainer(self, *args):
 840         """Create a LXD container for use with a proxy charm.abs
 841
 842         1. Get the public key from the charm via `get-ssh-public-key` action
 843         2. Create container with said key injected for the ubuntu user
 844
 845         Returns a Container object
 846         """
 847         # Create and configure a LXD container for use with a proxy charm.
 848         (model, application, _, _) = args
 849
 850         debug("[CreateContainer] {}".format(args))
 851         container = None
 852
 853         try:
 854             # Execute 'get-ssh-public-key' primitive and get returned value
 855             uuid = await self.n2vc.ExecutePrimitive(
 856                 model,
 857                 application,
 858                 "get-ssh-public-key",
 859                 None,
 860             )
 861
 862             result = await self.n2vc.GetPrimitiveOutput(model, uuid)
 863             pubkey = result['pubkey']
 864
 865             container = create_lxd_container(
 866                 public_key=pubkey,
 867                 name=os.path.basename(__file__)
 868             )
 869
 870             return container
 871         except Exception as ex:
 872             debug("Error creating container: {}".format(ex))
 873             pass
 874
 875         return None
 876
 877     @classmethod
 878     async def stop(self):
 879         """Stop the test.
 880
 881         - Remove charms
 882         - Stop and delete containers
 883         - Logout of N2VC
 884
 885         TODO: Clean up duplicate code between teardown_class() and stop()
 886         """
 887         debug("stop() called")
 888
 889         if self.n2vc and self._running and not self._stopping:
 890             self._running = False
 891             self._stopping = True
 892
 893             for application in self.charms:
 894                 try:
 895                     await self.n2vc.RemoveCharms(self.ns_name, application)
 896
 897                     while True:
 898                         # Wait for the application to be removed
 899                         await asyncio.sleep(10)
 900                         if not await self.n2vc.HasApplication(
 901                             self.ns_name,
 902                             application,
 903                         ):
 904                             break
 905                     await self.n2vc.DestroyNetworkService(self.ns_name)
 906
 907                     # Need to wait for the charm to finish, because native charms
 908                     if self.state[application]['container']:
 909                         debug("Deleting LXD container...")
 910                         destroy_lxd_container(
 911                             self.state[application]['container']
 912                         )
 913                         self.state[application]['container'] = None
 914                         debug("Deleting LXD container...done.")
 915                     else:
 916                         debug("No container found for {}".format(application))
 917                 except Exception as e:
 918                     debug("Error while deleting container: {}".format(e))
 919
 920             # Logout of N2VC
 921             try:
 922                 debug("stop(): Logging out of N2VC...")
 923                 await self.n2vc.logout()
 924                 self.n2vc = None
 925                 debug("stop(): Logging out of N2VC...Done.")
 926             except Exception as ex:
 927                 debug(ex)
 928
 929             # Let the test know we're finished.
 930             debug("Marking test as finished.")
 931             # self._running = False
 932         else:
 933             debug("Skipping stop()")
 934
 935     @classmethod
 936     def get_container_ip(self, container):
 937         """Return the IPv4 address of container's eth0 interface."""
 938         ipaddr = None
 939         if container:
 940             addresses = container.state().network['eth0']['addresses']
 941             # The interface may have more than one address, but we only need
 942             # the first one for testing purposes.
 943             ipaddr = addresses[0]['address']
 944
 945         return ipaddr
 946
 947     @classmethod
 948     async def configure_ssh_proxy(self, application, task=None):
 949         """Configure the proxy charm to use the lxd container.
 950
 951         Configure the charm to use a LXD container as it's VNF.
 952         """
 953         debug("Configuring ssh proxy for {}".format(application))
 954
 955         mgmtaddr = self.get_container_ip(
 956             self.state[application]['container'],
 957         )
 958
 959         debug(
 960             "Setting ssh-hostname for {} to {}".format(
 961                 application,
 962                 mgmtaddr,
 963             )
 964         )
 965
 966         await self.n2vc.ExecutePrimitive(
 967             self.ns_name,
 968             application,
 969             "config",
 970             None,
 971             params={
 972                 'ssh-hostname': mgmtaddr,
 973                 'ssh-username': 'ubuntu',
 974             }
 975         )
 976
 977         return True
 978
 979     @classmethod
 980     async def execute_initial_config_primitives(self, application, task=None):
 981         debug("Executing initial_config_primitives for {}".format(application))
 982         try:
 983             init_config = self.charms[application]
 984
 985             """
 986             The initial-config-primitive is run during deploy but may fail
 987              on some steps because proxy charm access isn't configured.
 988
 989             Re-run those actions so we can inspect the status.
 990             """
 991             uuids = await self.n2vc.ExecuteInitialPrimitives(
 992                 self.ns_name,
 993                 application,
 994                 init_config,
 995             )
 996
 997             """
 998             ExecutePrimitives will return a list of uuids. We need to check the
 999              status of each. The test continues if all Actions succeed, and
1000              fails if any of them fail.
1001             """
1002             await self.wait_for_uuids(application, uuids)
1003             debug("Primitives for {} finished.".format(application))
1004
1005             return True
1006         except Exception as ex:
1007             debug("execute_initial_config_primitives exception: {}".format(ex))
1008
1009         return False
1010
1011     @classmethod
1012     async def check_metrics(self, application, task=None):
1013         """Check and run metrics, if present.
1014
1015         Checks to see if metrics are specified by the charm. If so, collects
1016         the metrics.
1017
1018         If no metrics, then mark the test as finished.
1019         """
1020         if has_metrics(self.charms[application]['name']):
1021             debug("Collecting metrics for {}".format(application))
1022
1023             metrics = await self.n2vc.GetMetrics(
1024                 self.ns_name,
1025                 application,
1026             )
1027
1028             return await self.verify_metrics(application, metrics)
1029
1030     @classmethod
1031     async def verify_metrics(self, application, metrics):
1032         """Verify the charm's metrics.
1033
1034         Verify that the charm has sent metrics successfully.
1035
1036         Stops the test when finished.
1037         """
1038         debug("Verifying metrics for {}: {}".format(application, metrics))
1039
1040         if len(metrics):
1041             return True
1042
1043         else:
1044             # TODO: Ran into a case where it took 9 attempts before metrics
1045             # were available; the controller is slow sometimes.
1046             await asyncio.sleep(30)
1047             return await self.check_metrics(application)
1048
1049     @classmethod
1050     async def wait_for_uuids(self, application, uuids):
1051         """Wait for primitives to execute.
1052
1053         The task will provide a list of uuids representing primitives that are
1054         queued to run.
1055         """
1056         debug("Waiting for uuids for {}: {}".format(application, uuids))
1057         waitfor = len(uuids)
1058         finished = 0
1059
1060         while waitfor > finished:
1061             for uid in uuids:
1062                 await asyncio.sleep(10)
1063
1064                 if uuid not in self.state[application]['actions']:
1065                     self.state[application]['actions'][uid] = "pending"
1066
1067                 status = self.state[application]['actions'][uid]
1068
1069                 # Have we already marked this as done?
1070                 if status in ["pending", "running"]:
1071
1072                     debug("Getting status of {} ({})...".format(uid, status))
1073                     status = await self.n2vc.GetPrimitiveStatus(
1074                         self.ns_name,
1075                         uid,
1076                     )
1077                     debug("...state of {} is {}".format(uid, status))
1078                     self.state[application]['actions'][uid] = status
1079
1080                     if status in ['completed', 'failed']:
1081                         finished += 1
1082
1083             debug("{}/{} actions complete".format(finished, waitfor))
1084
1085             # Wait for the primitive to finish and try again
1086             if waitfor > finished:
1087                 debug("Waiting 10s for action to finish...")
1088                 await asyncio.sleep(10)
1089
1090     @classmethod
1091     def n2vc_callback(self, *args, **kwargs):
1092         (model, application, status, message) = args
1093         # debug("callback: {}".format(args))
1094
1095         if application not in self.state:
1096             # Initialize the state of the application
1097             self.state[application] = {
1098                 'status': None,     # Juju status
1099                 'container': None,  # lxd container, for proxy charms
1100                 'actions': {},      # Actions we've executed
1101                 'done': False,      # Are we done testing this charm?
1102                 'phase': "deploy",  # What phase is this application in?
1103             }
1104
1105         self.state[application]['status'] = status
1106
1107         if status in ['waiting', 'maintenance', 'unknown']:
1108             # Nothing to do for these
1109             return
1110
1111         debug("callback: {}".format(args))
1112
1113         if self.state[application]['done']:
1114             debug("{} is done".format(application))
1115             return
1116
1117         if status in ["blocked"] and self.isproxy(application):
1118             if self.state[application]['phase'] == "deploy":
1119                 debug("Configuring proxy charm for {}".format(application))
1120                 asyncio.ensure_future(self.configure_proxy_charm(*args))
1121
1122         elif status in ["active"]:
1123             """When a charm is active, we can assume that it has been properly
1124             configured (not blocked), regardless of if it's a proxy or not.
1125
1126             All primitives should be complete by init_config_primitive
1127             """
1128             asyncio.ensure_future(self.execute_charm_tests(*args))