tests/base.py

   1 #!/usr/bin/env python3
   2 import asyncio
   3 import datetime
   4 import logging
   5 import n2vc.vnf
   6 import pylxd
   7 import pytest
   8 import os
   9 import shlex
  10 import subprocess
  11 import time
  12 import uuid
  13 import yaml
  14
  15 from juju.controller import Controller
  16
  17 # Disable InsecureRequestWarning w/LXD
  18 import urllib3
  19 urllib3.disable_warnings()
  20 logging.getLogger("urllib3").setLevel(logging.WARNING)
  21
  22 here = os.path.dirname(os.path.realpath(__file__))
  23
  24
  25 def is_bootstrapped():
  26     result = subprocess.run(['juju', 'switch'], stdout=subprocess.PIPE)
  27     return (
  28         result.returncode == 0 and
  29         len(result.stdout.decode().strip()) > 0)
  30
  31
  32 bootstrapped = pytest.mark.skipif(
  33     not is_bootstrapped(),
  34     reason='bootstrapped Juju environment required')
  35
  36
  37 class CleanController():
  38     """
  39     Context manager that automatically connects and disconnects from
  40     the currently active controller.
  41
  42     Note: Unlike CleanModel, this will not create a new controller for you,
  43     and an active controller must already be available.
  44     """
  45     def __init__(self):
  46         self._controller = None
  47
  48     async def __aenter__(self):
  49         self._controller = Controller()
  50         await self._controller.connect()
  51         return self._controller
  52
  53     async def __aexit__(self, exc_type, exc, tb):
  54         await self._controller.disconnect()
  55
  56
  57 def debug(msg):
  58     """Format debug messages in a consistent way."""
  59     now = datetime.datetime.now()
  60
  61     # TODO: Decide on the best way to log. Output from `logging.debug` shows up
  62     # when a test fails, but print() will always show up when running tox with
  63     # `-s`, which is really useful for debugging single tests without having to
  64     # insert a False assert to see the log.
  65     logging.debug(
  66         "[{}] {}".format(now.strftime('%Y-%m-%dT%H:%M:%S'), msg)
  67     )
  68     # print(
  69     #     "[{}] {}".format(now.strftime('%Y-%m-%dT%H:%M:%S'), msg)
  70     # )
  71
  72
  73 def get_charm_path():
  74     return "{}/charms".format(here)
  75
  76
  77 def get_layer_path():
  78     return "{}/charms/layers".format(here)
  79
  80
  81 def collect_metrics(application):
  82     """Invoke Juju's metrics collector.
  83
  84     Caveat: this shells out to the `juju collect-metrics` command, rather than
  85     making an API call. At the time of writing, that API is not exposed through
  86     the client library.
  87     """
  88
  89     try:
  90         subprocess.check_call(['juju', 'collect-metrics', application])
  91     except subprocess.CalledProcessError as e:
  92         raise Exception("Unable to collect metrics: {}".format(e))
  93
  94
  95 def has_metrics(charm):
  96     """Check if a charm has metrics defined."""
  97     metricsyaml = "{}/{}/metrics.yaml".format(
  98         get_layer_path(),
  99         charm,
 100     )
 101     if os.path.exists(metricsyaml):
 102         return True
 103     return False
 104
 105
 106 def get_descriptor(descriptor):
 107     desc = None
 108     try:
 109         tmp = yaml.load(descriptor)
 110
 111         # Remove the envelope
 112         root = list(tmp.keys())[0]
 113         if root == "nsd:nsd-catalog":
 114             desc = tmp['nsd:nsd-catalog']['nsd'][0]
 115         elif root == "vnfd:vnfd-catalog":
 116             desc = tmp['vnfd:vnfd-catalog']['vnfd'][0]
 117     except ValueError:
 118         assert False
 119     return desc
 120
 121
 122 def get_n2vc(loop=None):
 123     """Return an instance of N2VC.VNF."""
 124     log = logging.getLogger()
 125     log.level = logging.DEBUG
 126
 127     # Extract parameters from the environment in order to run our test
 128     vca_host = os.getenv('VCA_HOST', '127.0.0.1')
 129     vca_port = os.getenv('VCA_PORT', 17070)
 130     vca_user = os.getenv('VCA_USER', 'admin')
 131     vca_charms = os.getenv('VCA_CHARMS', None)
 132     vca_secret = os.getenv('VCA_SECRET', None)
 133
 134     client = n2vc.vnf.N2VC(
 135         log=log,
 136         server=vca_host,
 137         port=vca_port,
 138         user=vca_user,
 139         secret=vca_secret,
 140         artifacts=vca_charms,
 141         loop=loop
 142     )
 143     return client
 144
 145
 146 def create_lxd_container(public_key=None, name="test_name"):
 147     """
 148     Returns a container object
 149
 150     If public_key isn't set, we'll use the Juju ssh key
 151
 152     :param public_key: The public key to inject into the container
 153     :param name: The name of the test being run
 154     """
 155     container = None
 156
 157     # Format name so it's valid
 158     name = name.replace("_", "-").replace(".", "")
 159
 160     client = get_lxd_client()
 161     test_machine = "test-{}-{}".format(
 162         uuid.uuid4().hex[-4:],
 163         name,
 164     )
 165
 166     private_key_path, public_key_path = find_n2vc_ssh_keys()
 167
 168     try:
 169         # create profile w/cloud-init and juju ssh key
 170         if not public_key:
 171             public_key = ""
 172             with open(public_key_path, "r") as f:
 173                 public_key = f.readline()
 174
 175         client.profiles.create(
 176             test_machine,
 177             config={
 178                 'user.user-data': '#cloud-config\nssh_authorized_keys:\n- {}'.format(public_key)},
 179             devices={
 180                 'root': {'path': '/', 'pool': 'default', 'type': 'disk'},
 181                 'eth0': {
 182                     'nictype': 'bridged',
 183                     'parent': 'lxdbr0',
 184                     'type': 'nic'
 185                 }
 186             }
 187         )
 188     except Exception as ex:
 189         debug("Error creating lxd profile {}: {}".format(test_machine, ex))
 190         raise ex
 191
 192     try:
 193         # create lxc machine
 194         config = {
 195             'name': test_machine,
 196             'source': {
 197                 'type': 'image',
 198                 'alias': 'xenial',
 199                 'mode': 'pull',
 200                 'protocol': 'simplestreams',
 201                 'server': 'https://cloud-images.ubuntu.com/releases',
 202             },
 203             'profiles': [test_machine],
 204         }
 205         container = client.containers.create(config, wait=True)
 206         container.start(wait=True)
 207     except Exception as ex:
 208         debug("Error creating lxd container {}: {}".format(test_machine, ex))
 209         # This is a test-ending failure.
 210         raise ex
 211
 212     def wait_for_network(container, timeout=30):
 213         """Wait for eth0 to have an ipv4 address."""
 214         starttime = time.time()
 215         while(time.time() < starttime + timeout):
 216             time.sleep(1)
 217             if 'eth0' in container.state().network:
 218                 addresses = container.state().network['eth0']['addresses']
 219                 if len(addresses) > 0:
 220                     if addresses[0]['family'] == 'inet':
 221                         return addresses[0]
 222         return None
 223
 224     try:
 225         wait_for_network(container)
 226     except Exception as ex:
 227         debug(
 228             "Error waiting for container {} network: {}".format(
 229                 test_machine,
 230                 ex,
 231             )
 232         )
 233
 234     # HACK: We need to give sshd a chance to bind to the interface,
 235     # and pylxd's container.execute seems to be broken and fails and/or
 236     # hangs trying to properly check if the service is up.
 237     (exit_code, stdout, stderr) = container.execute([
 238         'ping',
 239         '-c', '5',   # Wait for 5 ECHO_REPLY
 240         '8.8.8.8',   # Ping Google's public DNS
 241         '-W', '15',  # Set a 15 second deadline
 242     ])
 243     if exit_code > 0:
 244         # The network failed
 245         raise Exception("Unable to verify container network")
 246
 247     return container
 248
 249
 250 def destroy_lxd_container(container):
 251     """Stop and delete a LXD container.
 252
 253     Sometimes we see errors talking to LXD -- ephemerial issues like
 254     load or a bug that's killed the API. We'll do our best to clean
 255     up here, and we should run a cleanup after all tests are finished
 256     to remove any extra containers and profiles belonging to us.
 257     """
 258
 259     if type(container) is bool:
 260         return
 261
 262     name = container.name
 263     debug("Destroying container {}".format(name))
 264
 265     client = get_lxd_client()
 266
 267     def wait_for_stop(timeout=30):
 268         """Wait for eth0 to have an ipv4 address."""
 269         starttime = time.time()
 270         while(time.time() < starttime + timeout):
 271             time.sleep(1)
 272             if container.state == "Stopped":
 273                 return
 274
 275     def wait_for_delete(timeout=30):
 276         starttime = time.time()
 277         while(time.time() < starttime + timeout):
 278             time.sleep(1)
 279             if client.containers.exists(name) is False:
 280                 return
 281
 282     try:
 283         container.stop(wait=False)
 284         wait_for_stop()
 285     except Exception as ex:
 286         debug(
 287             "Error stopping container {}: {}".format(
 288                 name,
 289                 ex,
 290             )
 291         )
 292
 293     try:
 294         container.delete(wait=False)
 295         wait_for_delete()
 296     except Exception as ex:
 297         debug(
 298             "Error deleting container {}: {}".format(
 299                 name,
 300                 ex,
 301             )
 302         )
 303
 304     try:
 305         # Delete the profile created for this container
 306         profile = client.profiles.get(name)
 307         if profile:
 308             profile.delete()
 309     except Exception as ex:
 310         debug(
 311             "Error deleting profile {}: {}".format(
 312                 name,
 313                 ex,
 314             )
 315         )
 316
 317
 318 def find_lxd_config():
 319     """Find the LXD configuration directory."""
 320     paths = []
 321     paths.append(os.path.expanduser("~/.config/lxc"))
 322     paths.append(os.path.expanduser("~/snap/lxd/current/.config/lxc"))
 323
 324     for path in paths:
 325         if os.path.exists(path):
 326             crt = os.path.expanduser("{}/client.crt".format(path))
 327             key = os.path.expanduser("{}/client.key".format(path))
 328             if os.path.exists(crt) and os.path.exists(key):
 329                 return (crt, key)
 330     return (None, None)
 331
 332
 333 def find_n2vc_ssh_keys():
 334     """Find the N2VC ssh keys."""
 335
 336     paths = []
 337     paths.append(os.path.expanduser("~/.ssh/"))
 338
 339     for path in paths:
 340         if os.path.exists(path):
 341             private = os.path.expanduser("{}/id_n2vc_rsa".format(path))
 342             public = os.path.expanduser("{}/id_n2vc_rsa.pub".format(path))
 343             if os.path.exists(private) and os.path.exists(public):
 344                 return (private, public)
 345     return (None, None)
 346
 347
 348 def find_juju_ssh_keys():
 349     """Find the Juju ssh keys."""
 350
 351     paths = []
 352     paths.append(os.path.expanduser("~/.local/share/juju/ssh/"))
 353
 354     for path in paths:
 355         if os.path.exists(path):
 356             private = os.path.expanduser("{}/juju_id_rsa".format(path))
 357             public = os.path.expanduser("{}/juju_id_rsa.pub".format(path))
 358             if os.path.exists(private) and os.path.exists(public):
 359                 return (private, public)
 360     return (None, None)
 361
 362
 363 def get_juju_private_key():
 364     keys = find_juju_ssh_keys()
 365     return keys[0]
 366
 367
 368 def get_lxd_client(host="127.0.0.1", port="8443", verify=False):
 369     """ Get the LXD client."""
 370     client = None
 371     (crt, key) = find_lxd_config()
 372
 373     if crt and key:
 374         client = pylxd.Client(
 375             endpoint="https://{}:{}".format(host, port),
 376             cert=(crt, key),
 377             verify=verify,
 378         )
 379
 380     return client
 381
 382
 383 # TODO: This is marked serial but can be run in parallel with work, including:
 384 # - Fixing an event loop issue; seems that all tests stop when one test stops?
 385
 386
 387 @pytest.mark.serial
 388 class TestN2VC(object):
 389     """TODO:
 390     1. Validator Validation
 391
 392     Automatically validate the descriptors we're using here, unless the test author explicitly wants to skip them. Useful to make sure tests aren't being run against invalid descriptors, validating functionality that may fail against a properly written descriptor.
 393
 394     We need to have a flag (instance variable) that controls this behavior. It may be necessary to skip validation and run against a descriptor implementing features that have not yet been released in the Information Model.
 395     """
 396
 397     """
 398     The six phases of integration testing, for the test itself and each charm?:
 399
 400     setup/teardown_class:
 401     1. Prepare      - Verify the environment and create a new model
 402     2. Deploy       - Mark the test as ready to execute
 403     3. Configure    - Configuration to reach Active state
 404     4. Test         - Execute primitive(s) to verify success
 405     5. Collect      - Collect any useful artifacts for debugging (charm, logs)
 406     6. Destroy      - Destroy the model
 407
 408
 409     1. Prepare      - Building of charm
 410     2. Deploy       - Deploying charm
 411     3. Configure    - Configuration to reach Active state
 412     4. Test         - Execute primitive(s) to verify success
 413     5. Collect      - Collect any useful artifacts for debugging (charm, logs)
 414     6. Destroy      - Destroy the charm
 415
 416     """
 417     @classmethod
 418     def setup_class(self):
 419         """ setup any state specific to the execution of the given class (which
 420         usually contains tests).
 421         """
 422         # Initialize instance variable(s)
 423         self.n2vc = None
 424
 425         # Track internal state for each test run
 426         self.state = {}
 427
 428         # Parse the test's descriptors
 429         self.nsd = get_descriptor(self.NSD_YAML)
 430         self.vnfd = get_descriptor(self.VNFD_YAML)
 431
 432         self.ns_name = self.nsd['name']
 433         self.vnf_name = self.vnfd['name']
 434
 435         # Hard-coded to default for now, but this may change in the future.
 436         self.model = "default"
 437
 438         self.charms = {}
 439         self.parse_vnf_descriptor()
 440         assert self.charms is not {}
 441
 442         # Track artifacts, like compiled charms, that will need to be removed
 443         self.artifacts = {}
 444
 445         # Build the charm(s) needed for this test
 446         for charm in self.get_charm_names():
 447             self.get_charm(charm)
 448
 449         # A bit of a hack, in order to allow the N2VC callback to run parallel
 450         # to pytest. Test(s) should wait for this flag to change to False
 451         # before returning.
 452         self._running = True
 453         self._stopping = False
 454
 455     @classmethod
 456     def teardown_class(self):
 457         """ teardown any state that was previously setup with a call to
 458         setup_class.
 459         """
 460         debug("Running teardown_class...")
 461         try:
 462
 463             debug("Destroying LXD containers...")
 464             for application in self.state:
 465                 if self.state[application]['container']:
 466                     destroy_lxd_container(self.state[application]['container'])
 467             debug("Destroying LXD containers...done.")
 468
 469             # Logout of N2VC
 470             if self.n2vc:
 471                 debug("teardown_class(): Logging out of N2VC...")
 472                 yield from self.n2vc.logout()
 473                 debug("teardown_class(): Logging out of N2VC...done.")
 474
 475             debug("Running teardown_class...done.")
 476         except Exception as ex:
 477             debug("Exception in teardown_class: {}".format(ex))
 478
 479     @classmethod
 480     def all_charms_active(self):
 481         """Determine if the all deployed charms are active."""
 482         active = 0
 483
 484         for application in self.state:
 485             if 'status' in self.state[application]:
 486                 debug("status of {} is '{}'".format(
 487                     application,
 488                     self.state[application]['status'],
 489                 ))
 490                 if self.state[application]['status'] == 'active':
 491                     active += 1
 492
 493         debug("Active charms: {}/{}".format(
 494             active,
 495             len(self.charms),
 496         ))
 497
 498         if active == len(self.charms):
 499             return True
 500
 501         return False
 502
 503     @classmethod
 504     def are_tests_finished(self):
 505         appcount = len(self.state)
 506
 507         # If we don't have state yet, keep running.
 508         if appcount == 0:
 509             debug("No applications")
 510             return False
 511
 512         if self._stopping:
 513             debug("_stopping is True")
 514             return True
 515
 516         appdone = 0
 517         for application in self.state:
 518             if self.state[application]['done']:
 519                 appdone += 1
 520
 521         debug("{}/{} charms tested".format(appdone, appcount))
 522
 523         if appcount == appdone:
 524             return True
 525
 526         return False
 527
 528     @classmethod
 529     async def running(self, timeout=600):
 530         """Returns if the test is still running.
 531
 532         @param timeout The time, in seconds, to wait for the test to complete.
 533         """
 534         if self.are_tests_finished():
 535             await self.stop()
 536             return False
 537
 538         await asyncio.sleep(30)
 539
 540         return self._running
 541
 542     @classmethod
 543     def get_charm(self, charm):
 544         """Build and return the path to the test charm.
 545
 546         Builds one of the charms in tests/charms/layers and returns the path
 547         to the compiled charm. The charm will automatically be removed when
 548         when the test is complete.
 549
 550         Returns: The path to the built charm or None if `charm build` failed.
 551         """
 552
 553         # Make sure the charm snap is installed
 554         try:
 555             subprocess.check_call(['which', 'charm'])
 556         except subprocess.CalledProcessError as e:
 557             raise Exception("charm snap not installed.")
 558
 559         if charm not in self.artifacts:
 560             try:
 561                 # Note: This builds the charm under N2VC/tests/charms/builds/
 562                 # Currently, the snap-installed command only has write access
 563                 # to the $HOME (changing in an upcoming release) so writing to
 564                 # /tmp isn't possible at the moment.
 565                 builds = get_charm_path()
 566
 567                 if not os.path.exists("{}/builds/{}".format(builds, charm)):
 568                     cmd = "charm build {}/{} -o {}/".format(
 569                         get_layer_path(),
 570                         charm,
 571                         builds,
 572                     )
 573                     subprocess.check_call(shlex.split(cmd))
 574
 575                 self.artifacts[charm] = {
 576                     'tmpdir': builds,
 577                     'charm': "{}/builds/{}".format(builds, charm),
 578                 }
 579             except subprocess.CalledProcessError as e:
 580                 raise Exception("charm build failed: {}.".format(e))
 581
 582         return self.artifacts[charm]['charm']
 583
 584     @classmethod
 585     async def deploy(self, vnf_index, charm, params, loop):
 586         """An inner function to do the deployment of a charm from
 587         either a vdu or vnf.
 588         """
 589
 590         if not self.n2vc:
 591             self.n2vc = get_n2vc(loop=loop)
 592
 593         application = self.n2vc.FormatApplicationName(
 594             self.ns_name,
 595             self.vnf_name,
 596             str(vnf_index),
 597         )
 598
 599         # Initialize the state of the application
 600         self.state[application] = {
 601             'status': None,     # Juju status
 602             'container': None,  # lxd container, for proxy charms
 603             'actions': {},      # Actions we've executed
 604             'done': False,      # Are we done testing this charm?
 605             'phase': "deploy",  # What phase is this application in?
 606         }
 607
 608         debug("Deploying charm at {}".format(self.artifacts[charm]))
 609
 610         # If this is a native charm, we need to provision the underlying
 611         # machine ala an LXC container.
 612         machine_spec = {}
 613
 614         if not self.isproxy(application):
 615             debug("Creating container for native charm")
 616             # args = ("default", application, None, None)
 617             self.state[application]['container'] = create_lxd_container(
 618                 name=os.path.basename(__file__)
 619             )
 620
 621             hostname = self.get_container_ip(
 622                 self.state[application]['container'],
 623             )
 624
 625             machine_spec = {
 626                 'host': hostname,
 627                 'user': 'ubuntu',
 628             }
 629
 630         await self.n2vc.DeployCharms(
 631             self.ns_name,
 632             application,
 633             self.vnfd,
 634             self.get_charm(charm),
 635             params,
 636             machine_spec,
 637             self.n2vc_callback,
 638         )
 639
 640     @classmethod
 641     def parse_vnf_descriptor(self):
 642         """Parse the VNF descriptor to make running tests easier.
 643
 644         Parse the charm information in the descriptor to make it easy to write
 645         tests to run again it.
 646
 647         Each charm becomes a dictionary in a list:
 648         [
 649             'is-proxy': True,
 650             'vnf-member-index': 1,
 651             'vnf-name': '',
 652             'charm-name': '',
 653             'initial-config-primitive': {},
 654             'config-primitive': {}
 655         ]
 656         - charm name
 657         - is this a proxy charm?
 658         - what are the initial-config-primitives (day 1)?
 659         - what are the config primitives (day 2)?
 660
 661         """
 662         charms = {}
 663
 664         # You'd think this would be explicit, but it's just an incremental
 665         # value that should be consistent.
 666         vnf_member_index = 0
 667
 668         """Get all vdu and/or vdu config in a descriptor."""
 669         config = self.get_config()
 670         for cfg in config:
 671             if 'juju' in cfg:
 672
 673                 # Get the name to be used for the deployed application
 674                 application_name = n2vc.vnf.N2VC().FormatApplicationName(
 675                     self.ns_name,
 676                     self.vnf_name,
 677                     str(vnf_member_index),
 678                 )
 679
 680                 charm = {
 681                     'application-name': application_name,
 682                     'proxy': True,
 683                     'vnf-member-index': vnf_member_index,
 684                     'vnf-name': self.vnf_name,
 685                     'name': None,
 686                     'initial-config-primitive': {},
 687                     'config-primitive': {},
 688                 }
 689
 690                 juju = cfg['juju']
 691                 charm['name'] = juju['charm']
 692
 693                 if 'proxy' in juju:
 694                     charm['proxy'] = juju['proxy']
 695
 696                 if 'initial-config-primitive' in cfg:
 697                     charm['initial-config-primitive'] = \
 698                         cfg['initial-config-primitive']
 699
 700                 if 'config-primitive' in cfg:
 701                     charm['config-primitive'] = cfg['config-primitive']
 702
 703                 charms[application_name] = charm
 704
 705             # Increment the vnf-member-index
 706             vnf_member_index += 1
 707
 708         self.charms = charms
 709
 710     @classmethod
 711     def isproxy(self, application_name):
 712
 713         assert application_name in self.charms
 714         assert 'proxy' in self.charms[application_name]
 715         assert type(self.charms[application_name]['proxy']) is bool
 716
 717         # debug(self.charms[application_name])
 718         return self.charms[application_name]['proxy']
 719
 720     @classmethod
 721     def get_config(self):
 722         """Return an iterable list of config items (vdu and vnf).
 723
 724         As far as N2VC is concerned, the config section for vdu and vnf are
 725         identical. This joins them together so tests only need to iterate
 726         through one list.
 727         """
 728         configs = []
 729
 730         """Get all vdu and/or vdu config in a descriptor."""
 731         vnf_config = self.vnfd.get("vnf-configuration")
 732         if vnf_config:
 733             juju = vnf_config['juju']
 734             if juju:
 735                 configs.append(vnf_config)
 736
 737         for vdu in self.vnfd['vdu']:
 738             vdu_config = vdu.get('vdu-configuration')
 739             if vdu_config:
 740                 juju = vdu_config['juju']
 741                 if juju:
 742                     configs.append(vdu_config)
 743
 744         return configs
 745
 746     @classmethod
 747     def get_charm_names(self):
 748         """Return a list of charms used by the test descriptor."""
 749
 750         charms = {}
 751
 752         # Check if the VDUs in this VNF have a charm
 753         for config in self.get_config():
 754             juju = config['juju']
 755
 756             name = juju['charm']
 757             if name not in charms:
 758                 charms[name] = 1
 759
 760         return charms.keys()
 761
 762     @classmethod
 763     def get_phase(self, application):
 764         return self.state[application]['phase']
 765
 766     @classmethod
 767     def set_phase(self, application, phase):
 768         self.state[application]['phase'] = phase
 769
 770     @classmethod
 771     async def configure_proxy_charm(self, *args):
 772         """Configure a container for use via ssh."""
 773         (model, application, _, _) = args
 774
 775         try:
 776             if self.get_phase(application) == "deploy":
 777                 self.set_phase(application, "configure")
 778
 779                 debug("Start CreateContainer for {}".format(application))
 780                 self.state[application]['container'] = \
 781                     await self.CreateContainer(*args)
 782                 debug("Done CreateContainer for {}".format(application))
 783
 784                 if self.state[application]['container']:
 785                     debug("Configure {} for container".format(application))
 786                     if await self.configure_ssh_proxy(application):
 787                         await asyncio.sleep(0.1)
 788                         return True
 789                 else:
 790                     debug("Failed to configure container for {}".format(application))
 791             else:
 792                 debug("skipping CreateContainer for {}: {}".format(
 793                     application,
 794                     self.get_phase(application),
 795                 ))
 796
 797         except Exception as ex:
 798             debug("configure_proxy_charm exception: {}".format(ex))
 799         finally:
 800             await asyncio.sleep(0.1)
 801
 802         return False
 803
 804     @classmethod
 805     async def execute_charm_tests(self, *args):
 806         (model, application, _, _) = args
 807
 808         debug("Executing charm test(s) for {}".format(application))
 809
 810         if self.state[application]['done']:
 811             debug("Trying to execute tests against finished charm...aborting")
 812             return False
 813
 814         try:
 815             phase = self.get_phase(application)
 816             # We enter the test phase when after deploy (for native charms) or
 817             # configure, for proxy charms.
 818             if phase in ["deploy", "configure"]:
 819                 self.set_phase(application, "test")
 820                 if self.are_tests_finished():
 821                     raise Exception("Trying to execute init-config on finished test")
 822
 823                 if await self.execute_initial_config_primitives(application):
 824                     # check for metrics
 825                     await self.check_metrics(application)
 826
 827                     debug("Done testing {}".format(application))
 828                     self.state[application]['done'] = True
 829
 830         except Exception as ex:
 831             debug("Exception in execute_charm_tests: {}".format(ex))
 832         finally:
 833             await asyncio.sleep(0.1)
 834
 835         return True
 836
 837     @classmethod
 838     async def CreateContainer(self, *args):
 839         """Create a LXD container for use with a proxy charm.abs
 840
 841         1. Get the public key from the charm via `get-ssh-public-key` action
 842         2. Create container with said key injected for the ubuntu user
 843
 844         Returns a Container object
 845         """
 846         # Create and configure a LXD container for use with a proxy charm.
 847         (model, application, _, _) = args
 848
 849         debug("[CreateContainer] {}".format(args))
 850         container = None
 851
 852         try:
 853             # Execute 'get-ssh-public-key' primitive and get returned value
 854             uuid = await self.n2vc.ExecutePrimitive(
 855                 model,
 856                 application,
 857                 "get-ssh-public-key",
 858                 None,
 859             )
 860
 861             result = await self.n2vc.GetPrimitiveOutput(model, uuid)
 862             pubkey = result['pubkey']
 863
 864             container = create_lxd_container(
 865                 public_key=pubkey,
 866                 name=os.path.basename(__file__)
 867             )
 868
 869             return container
 870         except Exception as ex:
 871             debug("Error creating container: {}".format(ex))
 872             pass
 873
 874         return None
 875
 876     @classmethod
 877     async def stop(self):
 878         """Stop the test.
 879
 880         - Remove charms
 881         - Stop and delete containers
 882         - Logout of N2VC
 883
 884         TODO: Clean up duplicate code between teardown_class() and stop()
 885         """
 886         debug("stop() called")
 887
 888         if self.n2vc and self._running and not self._stopping:
 889             self._running = False
 890             self._stopping = True
 891
 892             for application in self.charms:
 893                 try:
 894                     await self.n2vc.RemoveCharms(self.model, application)
 895
 896                     while True:
 897                         # Wait for the application to be removed
 898                         await asyncio.sleep(10)
 899                         if not await self.n2vc.HasApplication(
 900                             self.model,
 901                             application,
 902                         ):
 903                             break
 904
 905                     # Need to wait for the charm to finish, because native charms
 906                     if self.state[application]['container']:
 907                         debug("Deleting LXD container...")
 908                         destroy_lxd_container(
 909                             self.state[application]['container']
 910                         )
 911                         self.state[application]['container'] = None
 912                         debug("Deleting LXD container...done.")
 913                     else:
 914                         debug("No container found for {}".format(application))
 915                 except Exception as e:
 916                     debug("Error while deleting container: {}".format(e))
 917
 918             # Logout of N2VC
 919             try:
 920                 debug("stop(): Logging out of N2VC...")
 921                 await self.n2vc.logout()
 922                 self.n2vc = None
 923                 debug("stop(): Logging out of N2VC...Done.")
 924             except Exception as ex:
 925                 debug(ex)
 926
 927             # Let the test know we're finished.
 928             debug("Marking test as finished.")
 929             # self._running = False
 930         else:
 931             debug("Skipping stop()")
 932
 933     @classmethod
 934     def get_container_ip(self, container):
 935         """Return the IPv4 address of container's eth0 interface."""
 936         ipaddr = None
 937         if container:
 938             addresses = container.state().network['eth0']['addresses']
 939             # The interface may have more than one address, but we only need
 940             # the first one for testing purposes.
 941             ipaddr = addresses[0]['address']
 942
 943         return ipaddr
 944
 945     @classmethod
 946     async def configure_ssh_proxy(self, application, task=None):
 947         """Configure the proxy charm to use the lxd container.
 948
 949         Configure the charm to use a LXD container as it's VNF.
 950         """
 951         debug("Configuring ssh proxy for {}".format(application))
 952
 953         mgmtaddr = self.get_container_ip(
 954             self.state[application]['container'],
 955         )
 956
 957         debug(
 958             "Setting ssh-hostname for {} to {}".format(
 959                 application,
 960                 mgmtaddr,
 961             )
 962         )
 963
 964         await self.n2vc.ExecutePrimitive(
 965             self.model,
 966             application,
 967             "config",
 968             None,
 969             params={
 970                 'ssh-hostname': mgmtaddr,
 971                 'ssh-username': 'ubuntu',
 972             }
 973         )
 974
 975         return True
 976
 977     @classmethod
 978     async def execute_initial_config_primitives(self, application, task=None):
 979         debug("Executing initial_config_primitives for {}".format(application))
 980         try:
 981             init_config = self.charms[application]
 982
 983             """
 984             The initial-config-primitive is run during deploy but may fail
 985              on some steps because proxy charm access isn't configured.
 986
 987             Re-run those actions so we can inspect the status.
 988             """
 989             uuids = await self.n2vc.ExecuteInitialPrimitives(
 990                 self.model,
 991                 application,
 992                 init_config,
 993             )
 994
 995             """
 996             ExecutePrimitives will return a list of uuids. We need to check the
 997              status of each. The test continues if all Actions succeed, and
 998              fails if any of them fail.
 999             """
1000             await self.wait_for_uuids(application, uuids)
1001             debug("Primitives for {} finished.".format(application))
1002
1003             return True
1004         except Exception as ex:
1005             debug("execute_initial_config_primitives exception: {}".format(ex))
1006
1007         return False
1008
1009     @classmethod
1010     async def check_metrics(self, application, task=None):
1011         """Check and run metrics, if present.
1012
1013         Checks to see if metrics are specified by the charm. If so, collects
1014         the metrics.
1015
1016         If no metrics, then mark the test as finished.
1017         """
1018         if has_metrics(self.charms[application]['name']):
1019             debug("Collecting metrics for {}".format(application))
1020
1021             metrics = await self.n2vc.GetMetrics(
1022                 self.model,
1023                 application,
1024             )
1025
1026             return await self.verify_metrics(application, metrics)
1027
1028     @classmethod
1029     async def verify_metrics(self, application, metrics):
1030         """Verify the charm's metrics.
1031
1032         Verify that the charm has sent metrics successfully.
1033
1034         Stops the test when finished.
1035         """
1036         debug("Verifying metrics for {}: {}".format(application, metrics))
1037
1038         if len(metrics):
1039             return True
1040
1041         else:
1042             # TODO: Ran into a case where it took 9 attempts before metrics
1043             # were available; the controller is slow sometimes.
1044             await asyncio.sleep(30)
1045             return await self.check_metrics(application)
1046
1047     @classmethod
1048     async def wait_for_uuids(self, application, uuids):
1049         """Wait for primitives to execute.
1050
1051         The task will provide a list of uuids representing primitives that are
1052         queued to run.
1053         """
1054         debug("Waiting for uuids for {}: {}".format(application, uuids))
1055         waitfor = len(uuids)
1056         finished = 0
1057
1058         while waitfor > finished:
1059             for uid in uuids:
1060                 await asyncio.sleep(10)
1061
1062                 if uuid not in self.state[application]['actions']:
1063                     self.state[application]['actions'][uid] = "pending"
1064
1065                 status = self.state[application]['actions'][uid]
1066
1067                 # Have we already marked this as done?
1068                 if status in ["pending", "running"]:
1069
1070                     debug("Getting status of {} ({})...".format(uid, status))
1071                     status = await self.n2vc.GetPrimitiveStatus(
1072                         self.model,
1073                         uid,
1074                     )
1075                     debug("...state of {} is {}".format(uid, status))
1076                     self.state[application]['actions'][uid] = status
1077
1078                     if status in ['completed', 'failed']:
1079                         finished += 1
1080
1081             debug("{}/{} actions complete".format(finished, waitfor))
1082
1083             # Wait for the primitive to finish and try again
1084             if waitfor > finished:
1085                 debug("Waiting 10s for action to finish...")
1086                 await asyncio.sleep(10)
1087
1088     @classmethod
1089     def n2vc_callback(self, *args, **kwargs):
1090         (model, application, status, message) = args
1091         # debug("callback: {}".format(args))
1092
1093         if application not in self.state:
1094             # Initialize the state of the application
1095             self.state[application] = {
1096                 'status': None,     # Juju status
1097                 'container': None,  # lxd container, for proxy charms
1098                 'actions': {},      # Actions we've executed
1099                 'done': False,      # Are we done testing this charm?
1100                 'phase': "deploy",  # What phase is this application in?
1101             }
1102
1103         self.state[application]['status'] = status
1104
1105         if status in ['waiting', 'maintenance', 'unknown']:
1106             # Nothing to do for these
1107             return
1108
1109         debug("callback: {}".format(args))
1110
1111         if self.state[application]['done']:
1112             debug("{} is done".format(application))
1113             return
1114
1115         if status in ["blocked"] and self.isproxy(application):
1116             if self.state[application]['phase'] == "deploy":
1117                 debug("Configuring proxy charm for {}".format(application))
1118                 asyncio.ensure_future(self.configure_proxy_charm(*args))
1119
1120         elif status in ["active"]:
1121             """When a charm is active, we can assume that it has been properly
1122             configured (not blocked), regardless of if it's a proxy or not.
1123
1124             All primitives should be complete by init_config_primitive
1125             """
1126             asyncio.ensure_future(self.execute_charm_tests(*args))