tests/base.py

   1 #!/usr/bin/env python3
   2 import asyncio
   3 import datetime
   4 import logging
   5 import n2vc.vnf
   6 import pylxd
   7 import pytest
   8 import os
   9 import shlex
  10 import subprocess
  11 import time
  12 import uuid
  13 import yaml
  14
  15 from juju.controller import Controller
  16
  17 # Disable InsecureRequestWarning w/LXD
  18 import urllib3
  19 urllib3.disable_warnings()
  20 logging.getLogger("urllib3").setLevel(logging.WARNING)
  21
  22 here = os.path.dirname(os.path.realpath(__file__))
  23
  24
  25 def is_bootstrapped():
  26     result = subprocess.run(['juju', 'switch'], stdout=subprocess.PIPE)
  27     return (
  28         result.returncode == 0 and len(result.stdout.decode().strip()) > 0)
  29
  30
  31 bootstrapped = pytest.mark.skipif(
  32     not is_bootstrapped(),
  33     reason='bootstrapped Juju environment required')
  34
  35
  36 class CleanController():
  37     """
  38     Context manager that automatically connects and disconnects from
  39     the currently active controller.
  40
  41     Note: Unlike CleanModel, this will not create a new controller for you,
  42     and an active controller must already be available.
  43     """
  44     def __init__(self):
  45         self._controller = None
  46
  47     async def __aenter__(self):
  48         self._controller = Controller()
  49         await self._controller.connect()
  50         return self._controller
  51
  52     async def __aexit__(self, exc_type, exc, tb):
  53         await self._controller.disconnect()
  54
  55
  56 def debug(msg):
  57     """Format debug messages in a consistent way."""
  58     now = datetime.datetime.now()
  59
  60     # TODO: Decide on the best way to log. Output from `logging.debug` shows up
  61     # when a test fails, but print() will always show up when running tox with
  62     # `-s`, which is really useful for debugging single tests without having to
  63     # insert a False assert to see the log.
  64     logging.debug(
  65         "[{}] {}".format(now.strftime('%Y-%m-%dT%H:%M:%S'), msg)
  66     )
  67     print(
  68         "[{}] {}".format(now.strftime('%Y-%m-%dT%H:%M:%S'), msg)
  69     )
  70
  71
  72 def get_charm_path():
  73     return "{}/charms".format(here)
  74
  75
  76 def get_layer_path():
  77     return "{}/charms/layers".format(here)
  78
  79
  80 def collect_metrics(application):
  81     """Invoke Juju's metrics collector.
  82
  83     Caveat: this shells out to the `juju collect-metrics` command, rather than
  84     making an API call. At the time of writing, that API is not exposed through
  85     the client library.
  86     """
  87
  88     try:
  89         subprocess.check_call(['juju', 'collect-metrics', application])
  90     except subprocess.CalledProcessError as e:
  91         raise Exception("Unable to collect metrics: {}".format(e))
  92
  93
  94 def has_metrics(charm):
  95     """Check if a charm has metrics defined."""
  96     metricsyaml = "{}/{}/metrics.yaml".format(
  97         get_layer_path(),
  98         charm,
  99     )
 100     if os.path.exists(metricsyaml):
 101         return True
 102     return False
 103
 104
 105 def get_descriptor(descriptor):
 106     desc = None
 107     try:
 108         tmp = yaml.load(descriptor)
 109
 110         # Remove the envelope
 111         root = list(tmp.keys())[0]
 112         if root == "nsd:nsd-catalog":
 113             desc = tmp['nsd:nsd-catalog']['nsd'][0]
 114         elif root == "vnfd:vnfd-catalog":
 115             desc = tmp['vnfd:vnfd-catalog']['vnfd'][0]
 116     except ValueError:
 117         assert False
 118     return desc
 119
 120
 121 def get_n2vc(loop=None):
 122     """Return an instance of N2VC.VNF."""
 123     log = logging.getLogger()
 124     log.level = logging.DEBUG
 125
 126     # Extract parameters from the environment in order to run our test
 127     vca_host = os.getenv('VCA_HOST', '127.0.0.1')
 128     vca_port = os.getenv('VCA_PORT', 17070)
 129     vca_user = os.getenv('VCA_USER', 'admin')
 130     vca_charms = os.getenv('VCA_CHARMS', None)
 131     vca_secret = os.getenv('VCA_SECRET', None)
 132
 133     client = n2vc.vnf.N2VC(
 134         log=log,
 135         server=vca_host,
 136         port=vca_port,
 137         user=vca_user,
 138         secret=vca_secret,
 139         artifacts=vca_charms,
 140         loop=loop
 141     )
 142     return client
 143
 144
 145 def create_lxd_container(public_key=None, name="test_name"):
 146     """
 147     Returns a container object
 148
 149     If public_key isn't set, we'll use the Juju ssh key
 150
 151     :param public_key: The public key to inject into the container
 152     :param name: The name of the test being run
 153     """
 154     container = None
 155
 156     # Format name so it's valid
 157     name = name.replace("_", "-").replace(".", "")
 158
 159     client = get_lxd_client()
 160     test_machine = "test-{}-{}".format(
 161         uuid.uuid4().hex[-4:],
 162         name,
 163     )
 164
 165     private_key_path, public_key_path = find_n2vc_ssh_keys()
 166
 167     try:
 168         # create profile w/cloud-init and juju ssh key
 169         if not public_key:
 170             public_key = ""
 171             with open(public_key_path, "r") as f:
 172                 public_key = f.readline()
 173
 174         client.profiles.create(
 175             test_machine,
 176             config={
 177                 'user.user-data': '#cloud-config\nssh_authorized_keys:\n- {}'.format(public_key)},
 178             devices={
 179                 'root': {'path': '/', 'pool': 'default', 'type': 'disk'},
 180                 'eth0': {
 181                     'nictype': 'bridged',
 182                     'parent': 'lxdbr0',
 183                     'type': 'nic'
 184                 }
 185             }
 186         )
 187     except Exception as ex:
 188         debug("Error creating lxd profile {}: {}".format(test_machine, ex))
 189         raise ex
 190
 191     try:
 192         # create lxc machine
 193         config = {
 194             'name': test_machine,
 195             'source': {
 196                 'type': 'image',
 197                 'alias': 'xenial',
 198                 'mode': 'pull',
 199                 'protocol': 'simplestreams',
 200                 'server': 'https://cloud-images.ubuntu.com/releases',
 201             },
 202             'profiles': [test_machine],
 203         }
 204         container = client.containers.create(config, wait=True)
 205         container.start(wait=True)
 206     except Exception as ex:
 207         debug("Error creating lxd container {}: {}".format(test_machine, ex))
 208         # This is a test-ending failure.
 209         raise ex
 210
 211     def wait_for_network(container, timeout=30):
 212         """Wait for eth0 to have an ipv4 address."""
 213         starttime = time.time()
 214         while(time.time() < starttime + timeout):
 215             time.sleep(1)
 216             if 'eth0' in container.state().network:
 217                 addresses = container.state().network['eth0']['addresses']
 218                 if len(addresses) > 0:
 219                     if addresses[0]['family'] == 'inet':
 220                         return addresses[0]
 221         return None
 222
 223     try:
 224         wait_for_network(container)
 225     except Exception as ex:
 226         debug(
 227             "Error waiting for container {} network: {}".format(
 228                 test_machine,
 229                 ex,
 230             )
 231         )
 232
 233     # HACK: We need to give sshd a chance to bind to the interface,
 234     # and pylxd's container.execute seems to be broken and fails and/or
 235     # hangs trying to properly check if the service is up.
 236     (exit_code, stdout, stderr) = container.execute([
 237         'ping',
 238         '-c', '5',   # Wait for 5 ECHO_REPLY
 239         '8.8.8.8',   # Ping Google's public DNS
 240         '-W', '15',  # Set a 15 second deadline
 241     ])
 242     if exit_code > 0:
 243         # The network failed
 244         raise Exception("Unable to verify container network")
 245
 246     return container
 247
 248
 249 def destroy_lxd_container(container):
 250     """Stop and delete a LXD container.
 251
 252     Sometimes we see errors talking to LXD -- ephemerial issues like
 253     load or a bug that's killed the API. We'll do our best to clean
 254     up here, and we should run a cleanup after all tests are finished
 255     to remove any extra containers and profiles belonging to us.
 256     """
 257
 258     if type(container) is bool:
 259         return
 260
 261     name = container.name
 262     debug("Destroying container {}".format(name))
 263
 264     client = get_lxd_client()
 265
 266     def wait_for_stop(timeout=30):
 267         """Wait for eth0 to have an ipv4 address."""
 268         starttime = time.time()
 269         while(time.time() < starttime + timeout):
 270             time.sleep(1)
 271             if container.state == "Stopped":
 272                 return
 273
 274     def wait_for_delete(timeout=30):
 275         starttime = time.time()
 276         while(time.time() < starttime + timeout):
 277             time.sleep(1)
 278             if client.containers.exists(name) is False:
 279                 return
 280
 281     try:
 282         container.stop(wait=False)
 283         wait_for_stop()
 284     except Exception as ex:
 285         debug(
 286             "Error stopping container {}: {}".format(
 287                 name,
 288                 ex,
 289             )
 290         )
 291
 292     try:
 293         container.delete(wait=False)
 294         wait_for_delete()
 295     except Exception as ex:
 296         debug(
 297             "Error deleting container {}: {}".format(
 298                 name,
 299                 ex,
 300             )
 301         )
 302
 303     try:
 304         # Delete the profile created for this container
 305         profile = client.profiles.get(name)
 306         if profile:
 307             profile.delete()
 308     except Exception as ex:
 309         debug(
 310             "Error deleting profile {}: {}".format(
 311                 name,
 312                 ex,
 313             )
 314         )
 315
 316
 317 def find_lxd_config():
 318     """Find the LXD configuration directory."""
 319     paths = []
 320     paths.append(os.path.expanduser("~/.config/lxc"))
 321     paths.append(os.path.expanduser("~/snap/lxd/current/.config/lxc"))
 322
 323     for path in paths:
 324         if os.path.exists(path):
 325             crt = os.path.expanduser("{}/client.crt".format(path))
 326             key = os.path.expanduser("{}/client.key".format(path))
 327             if os.path.exists(crt) and os.path.exists(key):
 328                 return (crt, key)
 329     return (None, None)
 330
 331
 332 def find_n2vc_ssh_keys():
 333     """Find the N2VC ssh keys."""
 334
 335     paths = []
 336     paths.append(os.path.expanduser("~/.ssh/"))
 337
 338     for path in paths:
 339         if os.path.exists(path):
 340             private = os.path.expanduser("{}/id_n2vc_rsa".format(path))
 341             public = os.path.expanduser("{}/id_n2vc_rsa.pub".format(path))
 342             if os.path.exists(private) and os.path.exists(public):
 343                 return (private, public)
 344     return (None, None)
 345
 346
 347 def find_juju_ssh_keys():
 348     """Find the Juju ssh keys."""
 349
 350     paths = []
 351     paths.append(os.path.expanduser("~/.local/share/juju/ssh/"))
 352
 353     for path in paths:
 354         if os.path.exists(path):
 355             private = os.path.expanduser("{}/juju_id_rsa".format(path))
 356             public = os.path.expanduser("{}/juju_id_rsa.pub".format(path))
 357             if os.path.exists(private) and os.path.exists(public):
 358                 return (private, public)
 359     return (None, None)
 360
 361
 362 def get_juju_private_key():
 363     keys = find_juju_ssh_keys()
 364     return keys[0]
 365
 366
 367 def get_lxd_client(host="127.0.0.1", port="8443", verify=False):
 368     """ Get the LXD client."""
 369     client = None
 370     (crt, key) = find_lxd_config()
 371
 372     if crt and key:
 373         client = pylxd.Client(
 374             endpoint="https://{}:{}".format(host, port),
 375             cert=(crt, key),
 376             verify=verify,
 377         )
 378
 379     return client
 380
 381
 382 # TODO: This is marked serial but can be run in parallel with work, including:
 383 # - Fixing an event loop issue; seems that all tests stop when one test stops?
 384
 385
 386 @pytest.mark.serial
 387 class TestN2VC(object):
 388     """TODO:
 389     1. Validator Validation
 390
 391     Automatically validate the descriptors we're using here, unless the test author explicitly wants to skip them. Useful to make sure tests aren't being run against invalid descriptors, validating functionality that may fail against a properly written descriptor.
 392
 393     We need to have a flag (instance variable) that controls this behavior. It may be necessary to skip validation and run against a descriptor implementing features that have not yet been released in the Information Model.
 394     """
 395
 396     """
 397     The six phases of integration testing, for the test itself and each charm?:
 398
 399     setup/teardown_class:
 400     1. Prepare      - Verify the environment and create a new model
 401     2. Deploy       - Mark the test as ready to execute
 402     3. Configure    - Configuration to reach Active state
 403     4. Test         - Execute primitive(s) to verify success
 404     5. Collect      - Collect any useful artifacts for debugging (charm, logs)
 405     6. Destroy      - Destroy the model
 406
 407
 408     1. Prepare      - Building of charm
 409     2. Deploy       - Deploying charm
 410     3. Configure    - Configuration to reach Active state
 411     4. Test         - Execute primitive(s) to verify success
 412     5. Collect      - Collect any useful artifacts for debugging (charm, logs)
 413     6. Destroy      - Destroy the charm
 414
 415     """
 416     @classmethod
 417     def setup_class(self):
 418         """ setup any state specific to the execution of the given class (which
 419         usually contains tests).
 420         """
 421         # Initialize instance variable(s)
 422         self.n2vc = None
 423
 424         # Track internal state for each test run
 425         self.state = {}
 426
 427         # Parse the test's descriptors
 428         self.nsd = get_descriptor(self.NSD_YAML)
 429         self.vnfd = get_descriptor(self.VNFD_YAML)
 430
 431         self.ns_name = self.nsd['name']
 432         self.vnf_name = self.vnfd['name']
 433
 434         self.charms = {}
 435         self.parse_vnf_descriptor()
 436         assert self.charms is not {}
 437
 438         # Track artifacts, like compiled charms, that will need to be removed
 439         self.artifacts = {}
 440
 441         # Build the charm(s) needed for this test
 442         for charm in self.get_charm_names():
 443             self.get_charm(charm)
 444
 445         # A bit of a hack, in order to allow the N2VC callback to run parallel
 446         # to pytest. Test(s) should wait for this flag to change to False
 447         # before returning.
 448         self._running = True
 449         self._stopping = False
 450
 451     @classmethod
 452     def teardown_class(self):
 453         """ teardown any state that was previously setup with a call to
 454         setup_class.
 455         """
 456         debug("Running teardown_class...")
 457         try:
 458
 459             debug("Destroying LXD containers...")
 460             for application in self.state:
 461                 if self.state[application]['container']:
 462                     destroy_lxd_container(self.state[application]['container'])
 463             debug("Destroying LXD containers...done.")
 464
 465             # Logout of N2VC
 466             if self.n2vc:
 467                 debug("teardown_class(): Logging out of N2VC...")
 468                 yield from self.n2vc.logout()
 469                 debug("teardown_class(): Logging out of N2VC...done.")
 470
 471             debug("Running teardown_class...done.")
 472         except Exception as ex:
 473             debug("Exception in teardown_class: {}".format(ex))
 474
 475     @classmethod
 476     def all_charms_active(self):
 477         """Determine if the all deployed charms are active."""
 478         active = 0
 479
 480         for application in self.state:
 481             if 'status' in self.state[application]:
 482                 debug("status of {} is '{}'".format(
 483                     application,
 484                     self.state[application]['status'],
 485                 ))
 486                 if self.state[application]['status'] == 'active':
 487                     active += 1
 488
 489         debug("Active charms: {}/{}".format(
 490             active,
 491             len(self.charms),
 492         ))
 493
 494         if active == len(self.charms):
 495             return True
 496
 497         return False
 498
 499     @classmethod
 500     def are_tests_finished(self):
 501         appcount = len(self.state)
 502
 503         # If we don't have state yet, keep running.
 504         if appcount == 0:
 505             debug("No applications")
 506             return False
 507
 508         if self._stopping:
 509             debug("_stopping is True")
 510             return True
 511
 512         appdone = 0
 513         for application in self.state:
 514             if self.state[application]['done']:
 515                 appdone += 1
 516
 517         debug("{}/{} charms tested".format(appdone, appcount))
 518
 519         if appcount == appdone:
 520             return True
 521
 522         return False
 523
 524     @classmethod
 525     async def running(self, timeout=600):
 526         """Returns if the test is still running.
 527
 528         @param timeout The time, in seconds, to wait for the test to complete.
 529         """
 530         if self.are_tests_finished():
 531             await self.stop()
 532             return False
 533
 534         await asyncio.sleep(30)
 535
 536         return self._running
 537
 538     @classmethod
 539     def get_charm(self, charm):
 540         """Build and return the path to the test charm.
 541
 542         Builds one of the charms in tests/charms/layers and returns the path
 543         to the compiled charm. The charm will automatically be removed when
 544         when the test is complete.
 545
 546         Returns: The path to the built charm or None if `charm build` failed.
 547         """
 548
 549         # Make sure the charm snap is installed
 550         try:
 551             subprocess.check_call(['which', 'charm'])
 552         except subprocess.CalledProcessError:
 553             raise Exception("charm snap not installed.")
 554
 555         if charm not in self.artifacts:
 556             try:
 557                 # Note: This builds the charm under N2VC/tests/charms/builds/
 558                 # Currently, the snap-installed command only has write access
 559                 # to the $HOME (changing in an upcoming release) so writing to
 560                 # /tmp isn't possible at the moment.
 561                 builds = get_charm_path()
 562
 563                 if not os.path.exists("{}/builds/{}".format(builds, charm)):
 564                     cmd = "charm build --no-local-layers {}/{} -o {}/".format(
 565                         get_layer_path(),
 566                         charm,
 567                         builds,
 568                     )
 569                     subprocess.check_call(shlex.split(cmd))
 570
 571                 self.artifacts[charm] = {
 572                     'tmpdir': builds,
 573                     'charm': "{}/builds/{}".format(builds, charm),
 574                 }
 575             except subprocess.CalledProcessError as e:
 576                 raise Exception("charm build failed: {}.".format(e))
 577
 578         return self.artifacts[charm]['charm']
 579
 580     @classmethod
 581     async def deploy(self, vnf_index, charm, params, loop):
 582         """An inner function to do the deployment of a charm from
 583         either a vdu or vnf.
 584         """
 585
 586         if not self.n2vc:
 587             self.n2vc = get_n2vc(loop=loop)
 588
 589         application = self.n2vc.FormatApplicationName(
 590             self.ns_name,
 591             self.vnf_name,
 592             str(vnf_index),
 593         )
 594
 595         # Initialize the state of the application
 596         self.state[application] = {
 597             'status': None,     # Juju status
 598             'container': None,  # lxd container, for proxy charms
 599             'actions': {},      # Actions we've executed
 600             'done': False,      # Are we done testing this charm?
 601             'phase': "deploy",  # What phase is this application in?
 602         }
 603
 604         debug("Deploying charm at {}".format(self.artifacts[charm]))
 605
 606         # If this is a native charm, we need to provision the underlying
 607         # machine ala an LXC container.
 608         machine_spec = {}
 609
 610         if not self.isproxy(application):
 611             debug("Creating container for native charm")
 612             # args = ("default", application, None, None)
 613             self.state[application]['container'] = create_lxd_container(
 614                 name=os.path.basename(__file__)
 615             )
 616
 617             hostname = self.get_container_ip(
 618                 self.state[application]['container'],
 619             )
 620
 621             machine_spec = {
 622                 'host': hostname,
 623                 'user': 'ubuntu',
 624             }
 625
 626         await self.n2vc.DeployCharms(
 627             self.ns_name,
 628             application,
 629             self.vnfd,
 630             self.get_charm(charm),
 631             params,
 632             machine_spec,
 633             self.n2vc_callback,
 634         )
 635
 636     @classmethod
 637     def parse_vnf_descriptor(self):
 638         """Parse the VNF descriptor to make running tests easier.
 639
 640         Parse the charm information in the descriptor to make it easy to write
 641         tests to run again it.
 642
 643         Each charm becomes a dictionary in a list:
 644         [
 645             'is-proxy': True,
 646             'vnf-member-index': 1,
 647             'vnf-name': '',
 648             'charm-name': '',
 649             'initial-config-primitive': {},
 650             'config-primitive': {}
 651         ]
 652         - charm name
 653         - is this a proxy charm?
 654         - what are the initial-config-primitives (day 1)?
 655         - what are the config primitives (day 2)?
 656
 657         """
 658         charms = {}
 659
 660         # You'd think this would be explicit, but it's just an incremental
 661         # value that should be consistent.
 662         vnf_member_index = 0
 663
 664         """Get all vdu and/or vdu config in a descriptor."""
 665         config = self.get_config()
 666         for cfg in config:
 667             if 'juju' in cfg:
 668
 669                 # Get the name to be used for the deployed application
 670                 application_name = n2vc.vnf.N2VC().FormatApplicationName(
 671                     self.ns_name,
 672                     self.vnf_name,
 673                     str(vnf_member_index),
 674                 )
 675
 676                 charm = {
 677                     'application-name': application_name,
 678                     'proxy': True,
 679                     'vnf-member-index': vnf_member_index,
 680                     'vnf-name': self.vnf_name,
 681                     'name': None,
 682                     'initial-config-primitive': {},
 683                     'config-primitive': {},
 684                 }
 685
 686                 juju = cfg['juju']
 687                 charm['name'] = juju['charm']
 688
 689                 if 'proxy' in juju:
 690                     charm['proxy'] = juju['proxy']
 691
 692                 if 'initial-config-primitive' in cfg:
 693                     charm['initial-config-primitive'] = \
 694                         cfg['initial-config-primitive']
 695
 696                 if 'config-primitive' in cfg:
 697                     charm['config-primitive'] = cfg['config-primitive']
 698
 699                 charms[application_name] = charm
 700
 701             # Increment the vnf-member-index
 702             vnf_member_index += 1
 703
 704         self.charms = charms
 705
 706     @classmethod
 707     def isproxy(self, application_name):
 708
 709         assert application_name in self.charms
 710         assert 'proxy' in self.charms[application_name]
 711         assert type(self.charms[application_name]['proxy']) is bool
 712
 713         # debug(self.charms[application_name])
 714         return self.charms[application_name]['proxy']
 715
 716     @classmethod
 717     def get_config(self):
 718         """Return an iterable list of config items (vdu and vnf).
 719
 720         As far as N2VC is concerned, the config section for vdu and vnf are
 721         identical. This joins them together so tests only need to iterate
 722         through one list.
 723         """
 724         configs = []
 725
 726         """Get all vdu and/or vdu config in a descriptor."""
 727         vnf_config = self.vnfd.get("vnf-configuration")
 728         if vnf_config:
 729             juju = vnf_config['juju']
 730             if juju:
 731                 configs.append(vnf_config)
 732
 733         for vdu in self.vnfd['vdu']:
 734             vdu_config = vdu.get('vdu-configuration')
 735             if vdu_config:
 736                 juju = vdu_config['juju']
 737                 if juju:
 738                     configs.append(vdu_config)
 739
 740         return configs
 741
 742     @classmethod
 743     def get_charm_names(self):
 744         """Return a list of charms used by the test descriptor."""
 745
 746         charms = {}
 747
 748         # Check if the VDUs in this VNF have a charm
 749         for config in self.get_config():
 750             juju = config['juju']
 751
 752             name = juju['charm']
 753             if name not in charms:
 754                 charms[name] = 1
 755
 756         return charms.keys()
 757
 758     @classmethod
 759     def get_phase(self, application):
 760         return self.state[application]['phase']
 761
 762     @classmethod
 763     def set_phase(self, application, phase):
 764         self.state[application]['phase'] = phase
 765
 766     @classmethod
 767     async def configure_proxy_charm(self, *args):
 768         """Configure a container for use via ssh."""
 769         (model, application, _, _) = args
 770
 771         try:
 772             if self.get_phase(application) == "deploy":
 773                 self.set_phase(application, "configure")
 774
 775                 debug("Start CreateContainer for {}".format(application))
 776                 self.state[application]['container'] = \
 777                     await self.CreateContainer(*args)
 778                 debug("Done CreateContainer for {}".format(application))
 779
 780                 if self.state[application]['container']:
 781                     debug("Configure {} for container".format(application))
 782                     if await self.configure_ssh_proxy(application):
 783                         await asyncio.sleep(0.1)
 784                         return True
 785                 else:
 786                     debug("Failed to configure container for {}".format(application))
 787             else:
 788                 debug("skipping CreateContainer for {}: {}".format(
 789                     application,
 790                     self.get_phase(application),
 791                 ))
 792
 793         except Exception as ex:
 794             debug("configure_proxy_charm exception: {}".format(ex))
 795         finally:
 796             await asyncio.sleep(0.1)
 797
 798         return False
 799
 800     @classmethod
 801     async def execute_charm_tests(self, *args):
 802         (model, application, _, _) = args
 803
 804         debug("Executing charm test(s) for {}".format(application))
 805
 806         if self.state[application]['done']:
 807             debug("Trying to execute tests against finished charm...aborting")
 808             return False
 809
 810         try:
 811             phase = self.get_phase(application)
 812             # We enter the test phase when after deploy (for native charms) or
 813             # configure, for proxy charms.
 814             if phase in ["deploy", "configure"]:
 815                 self.set_phase(application, "test")
 816                 if self.are_tests_finished():
 817                     raise Exception("Trying to execute init-config on finished test")
 818
 819                 if await self.execute_initial_config_primitives(application):
 820                     # check for metrics
 821                     await self.check_metrics(application)
 822
 823                     debug("Done testing {}".format(application))
 824                     self.state[application]['done'] = True
 825
 826         except Exception as ex:
 827             debug("Exception in execute_charm_tests: {}".format(ex))
 828         finally:
 829             await asyncio.sleep(0.1)
 830
 831         return True
 832
 833     @classmethod
 834     async def CreateContainer(self, *args):
 835         """Create a LXD container for use with a proxy charm.abs
 836
 837         1. Get the public key from the charm via `get-ssh-public-key` action
 838         2. Create container with said key injected for the ubuntu user
 839
 840         Returns a Container object
 841         """
 842         # Create and configure a LXD container for use with a proxy charm.
 843         (model, application, _, _) = args
 844
 845         debug("[CreateContainer] {}".format(args))
 846         container = None
 847
 848         try:
 849             # Execute 'get-ssh-public-key' primitive and get returned value
 850             uuid = await self.n2vc.ExecutePrimitive(
 851                 model,
 852                 application,
 853                 "get-ssh-public-key",
 854                 None,
 855             )
 856
 857             result = await self.n2vc.GetPrimitiveOutput(model, uuid)
 858             pubkey = result['pubkey']
 859
 860             container = create_lxd_container(
 861                 public_key=pubkey,
 862                 name=os.path.basename(__file__)
 863             )
 864
 865             return container
 866         except Exception as ex:
 867             debug("Error creating container: {}".format(ex))
 868             pass
 869
 870         return None
 871
 872     @classmethod
 873     async def stop(self):
 874         """Stop the test.
 875
 876         - Remove charms
 877         - Stop and delete containers
 878         - Logout of N2VC
 879
 880         TODO: Clean up duplicate code between teardown_class() and stop()
 881         """
 882         debug("stop() called")
 883
 884         if self.n2vc and self._running and not self._stopping:
 885             self._running = False
 886             self._stopping = True
 887
 888             for application in self.charms:
 889                 try:
 890                     await self.n2vc.RemoveCharms(self.ns_name, application)
 891
 892                     while True:
 893                         # Wait for the application to be removed
 894                         await asyncio.sleep(10)
 895                         if not await self.n2vc.HasApplication(
 896                             self.ns_name,
 897                             application,
 898                         ):
 899                             break
 900
 901                     # Need to wait for the charm to finish, because native charms
 902                     if self.state[application]['container']:
 903                         debug("Deleting LXD container...")
 904                         destroy_lxd_container(
 905                             self.state[application]['container']
 906                         )
 907                         self.state[application]['container'] = None
 908                         debug("Deleting LXD container...done.")
 909                     else:
 910                         debug("No container found for {}".format(application))
 911                 except Exception as e:
 912                     debug("Error while deleting container: {}".format(e))
 913
 914             # Logout of N2VC
 915             try:
 916                 debug("stop(): Logging out of N2VC...")
 917                 await self.n2vc.logout()
 918                 self.n2vc = None
 919                 debug("stop(): Logging out of N2VC...Done.")
 920             except Exception as ex:
 921                 debug(ex)
 922
 923             # Let the test know we're finished.
 924             debug("Marking test as finished.")
 925             # self._running = False
 926         else:
 927             debug("Skipping stop()")
 928
 929     @classmethod
 930     def get_container_ip(self, container):
 931         """Return the IPv4 address of container's eth0 interface."""
 932         ipaddr = None
 933         if container:
 934             addresses = container.state().network['eth0']['addresses']
 935             # The interface may have more than one address, but we only need
 936             # the first one for testing purposes.
 937             ipaddr = addresses[0]['address']
 938
 939         return ipaddr
 940
 941     @classmethod
 942     async def configure_ssh_proxy(self, application, task=None):
 943         """Configure the proxy charm to use the lxd container.
 944
 945         Configure the charm to use a LXD container as it's VNF.
 946         """
 947         debug("Configuring ssh proxy for {}".format(application))
 948
 949         mgmtaddr = self.get_container_ip(
 950             self.state[application]['container'],
 951         )
 952
 953         debug(
 954             "Setting ssh-hostname for {} to {}".format(
 955                 application,
 956                 mgmtaddr,
 957             )
 958         )
 959
 960         await self.n2vc.ExecutePrimitive(
 961             self.ns_name,
 962             application,
 963             "config",
 964             None,
 965             params={
 966                 'ssh-hostname': mgmtaddr,
 967                 'ssh-username': 'ubuntu',
 968             }
 969         )
 970
 971         return True
 972
 973     @classmethod
 974     async def execute_initial_config_primitives(self, application, task=None):
 975         debug("Executing initial_config_primitives for {}".format(application))
 976         try:
 977             init_config = self.charms[application]
 978
 979             """
 980             The initial-config-primitive is run during deploy but may fail
 981              on some steps because proxy charm access isn't configured.
 982
 983             Re-run those actions so we can inspect the status.
 984             """
 985             uuids = await self.n2vc.ExecuteInitialPrimitives(
 986                 self.ns_name,
 987                 application,
 988                 init_config,
 989             )
 990
 991             """
 992             ExecutePrimitives will return a list of uuids. We need to check the
 993              status of each. The test continues if all Actions succeed, and
 994              fails if any of them fail.
 995             """
 996             await self.wait_for_uuids(application, uuids)
 997             debug("Primitives for {} finished.".format(application))
 998
 999             return True
1000         except Exception as ex:
1001             debug("execute_initial_config_primitives exception: {}".format(ex))
1002
1003         return False
1004
1005     @classmethod
1006     async def check_metrics(self, application, task=None):
1007         """Check and run metrics, if present.
1008
1009         Checks to see if metrics are specified by the charm. If so, collects
1010         the metrics.
1011
1012         If no metrics, then mark the test as finished.
1013         """
1014         if has_metrics(self.charms[application]['name']):
1015             debug("Collecting metrics for {}".format(application))
1016
1017             metrics = await self.n2vc.GetMetrics(
1018                 self.ns_name,
1019                 application,
1020             )
1021
1022             return await self.verify_metrics(application, metrics)
1023
1024     @classmethod
1025     async def verify_metrics(self, application, metrics):
1026         """Verify the charm's metrics.
1027
1028         Verify that the charm has sent metrics successfully.
1029
1030         Stops the test when finished.
1031         """
1032         debug("Verifying metrics for {}: {}".format(application, metrics))
1033
1034         if len(metrics):
1035             return True
1036
1037         else:
1038             # TODO: Ran into a case where it took 9 attempts before metrics
1039             # were available; the controller is slow sometimes.
1040             await asyncio.sleep(30)
1041             return await self.check_metrics(application)
1042
1043     @classmethod
1044     async def wait_for_uuids(self, application, uuids):
1045         """Wait for primitives to execute.
1046
1047         The task will provide a list of uuids representing primitives that are
1048         queued to run.
1049         """
1050         debug("Waiting for uuids for {}: {}".format(application, uuids))
1051         waitfor = len(uuids)
1052         finished = 0
1053
1054         while waitfor > finished:
1055             for uid in uuids:
1056                 await asyncio.sleep(10)
1057
1058                 if uuid not in self.state[application]['actions']:
1059                     self.state[application]['actions'][uid] = "pending"
1060
1061                 status = self.state[application]['actions'][uid]
1062
1063                 # Have we already marked this as done?
1064                 if status in ["pending", "running"]:
1065
1066                     debug("Getting status of {} ({})...".format(uid, status))
1067                     status = await self.n2vc.GetPrimitiveStatus(
1068                         self.ns_name,
1069                         uid,
1070                     )
1071                     debug("...state of {} is {}".format(uid, status))
1072                     self.state[application]['actions'][uid] = status
1073
1074                     if status in ['completed', 'failed']:
1075                         finished += 1
1076
1077             debug("{}/{} actions complete".format(finished, waitfor))
1078
1079             # Wait for the primitive to finish and try again
1080             if waitfor > finished:
1081                 debug("Waiting 10s for action to finish...")
1082                 await asyncio.sleep(10)
1083
1084     @classmethod
1085     def n2vc_callback(self, *args, **kwargs):
1086         (model, application, status, message) = args
1087         # debug("callback: {}".format(args))
1088
1089         if application not in self.state:
1090             # Initialize the state of the application
1091             self.state[application] = {
1092                 'status': None,     # Juju status
1093                 'container': None,  # lxd container, for proxy charms
1094                 'actions': {},      # Actions we've executed
1095                 'done': False,      # Are we done testing this charm?
1096                 'phase': "deploy",  # What phase is this application in?
1097             }
1098
1099         self.state[application]['status'] = status
1100
1101         if status in ['waiting', 'maintenance', 'unknown']:
1102             # Nothing to do for these
1103             return
1104
1105         debug("callback: {}".format(args))
1106
1107         if self.state[application]['done']:
1108             debug("{} is done".format(application))
1109             return
1110
1111         if status in ["blocked"] and self.isproxy(application):
1112             if self.state[application]['phase'] == "deploy":
1113                 debug("Configuring proxy charm for {}".format(application))
1114                 asyncio.ensure_future(self.configure_proxy_charm(*args))
1115
1116         elif status in ["active"]:
1117             """When a charm is active, we can assume that it has been properly
1118             configured (not blocked), regardless of if it's a proxy or not.
1119
1120             All primitives should be complete by init_config_primitive
1121             """
1122             asyncio.ensure_future(self.execute_charm_tests(*args))