tests/base.py

   1 #!/usr/bin/env python3
   2 import asyncio
   3 import datetime
   4 import logging
   5 import n2vc.vnf
   6 import pylxd
   7 import pytest
   8 import os
   9 import shlex
  10 import subprocess
  11 import time
  12 import uuid
  13 import yaml
  14
  15 from juju.controller import Controller
  16
  17 # Disable InsecureRequestWarning w/LXD
  18 import urllib3
  19 urllib3.disable_warnings()
  20 logging.getLogger("urllib3").setLevel(logging.WARNING)
  21
  22 here = os.path.dirname(os.path.realpath(__file__))
  23
  24
  25 def is_bootstrapped():
  26     result = subprocess.run(['juju', 'switch'], stdout=subprocess.PIPE)
  27     return (
  28         result.returncode == 0 and len(result.stdout.decode().strip()) > 0)
  29
  30
  31 bootstrapped = pytest.mark.skipif(
  32     not is_bootstrapped(),
  33     reason='bootstrapped Juju environment required')
  34
  35
  36 class CleanController():
  37     """
  38     Context manager that automatically connects and disconnects from
  39     the currently active controller.
  40
  41     Note: Unlike CleanModel, this will not create a new controller for you,
  42     and an active controller must already be available.
  43     """
  44     def __init__(self):
  45         self._controller = None
  46
  47     async def __aenter__(self):
  48         self._controller = Controller()
  49         await self._controller.connect()
  50         return self._controller
  51
  52     async def __aexit__(self, exc_type, exc, tb):
  53         await self._controller.disconnect()
  54
  55
  56 def debug(msg):
  57     """Format debug messages in a consistent way."""
  58     now = datetime.datetime.now()
  59
  60     # TODO: Decide on the best way to log. Output from `logging.debug` shows up
  61     # when a test fails, but print() will always show up when running tox with
  62     # `-s`, which is really useful for debugging single tests without having to
  63     # insert a False assert to see the log.
  64     logging.debug(
  65         "[{}] {}".format(now.strftime('%Y-%m-%dT%H:%M:%S'), msg)
  66     )
  67     print(
  68         "[{}] {}".format(now.strftime('%Y-%m-%dT%H:%M:%S'), msg)
  69     )
  70
  71
  72 def get_charm_path():
  73     return "{}/charms".format(here)
  74
  75
  76 def get_layer_path():
  77     return "{}/charms/layers".format(here)
  78
  79
  80 def collect_metrics(application):
  81     """Invoke Juju's metrics collector.
  82
  83     Caveat: this shells out to the `juju collect-metrics` command, rather than
  84     making an API call. At the time of writing, that API is not exposed through
  85     the client library.
  86     """
  87
  88     try:
  89         subprocess.check_call(['juju', 'collect-metrics', application])
  90     except subprocess.CalledProcessError as e:
  91         raise Exception("Unable to collect metrics: {}".format(e))
  92
  93
  94 def has_metrics(charm):
  95     """Check if a charm has metrics defined."""
  96     metricsyaml = "{}/{}/metrics.yaml".format(
  97         get_layer_path(),
  98         charm,
  99     )
 100     if os.path.exists(metricsyaml):
 101         return True
 102     return False
 103
 104
 105 def get_descriptor(descriptor):
 106     desc = None
 107     try:
 108         tmp = yaml.load(descriptor)
 109
 110         # Remove the envelope
 111         root = list(tmp.keys())[0]
 112         if root == "nsd:nsd-catalog":
 113             desc = tmp['nsd:nsd-catalog']['nsd'][0]
 114         elif root == "vnfd:vnfd-catalog":
 115             desc = tmp['vnfd:vnfd-catalog']['vnfd'][0]
 116     except ValueError:
 117         assert False
 118     return desc
 119
 120
 121 def get_n2vc(loop=None):
 122     """Return an instance of N2VC.VNF."""
 123     log = logging.getLogger()
 124     log.level = logging.DEBUG
 125
 126     # Extract parameters from the environment in order to run our test
 127     vca_host = os.getenv('VCA_HOST', '127.0.0.1')
 128     vca_port = os.getenv('VCA_PORT', 17070)
 129     vca_user = os.getenv('VCA_USER', 'admin')
 130     vca_charms = os.getenv('VCA_CHARMS', None)
 131     vca_secret = os.getenv('VCA_SECRET', None)
 132
 133     client = n2vc.vnf.N2VC(
 134         log=log,
 135         server=vca_host,
 136         port=vca_port,
 137         user=vca_user,
 138         secret=vca_secret,
 139         artifacts=vca_charms,
 140         loop=loop
 141     )
 142     return client
 143
 144
 145 def create_lxd_container(public_key=None, name="test_name"):
 146     """
 147     Returns a container object
 148
 149     If public_key isn't set, we'll use the Juju ssh key
 150
 151     :param public_key: The public key to inject into the container
 152     :param name: The name of the test being run
 153     """
 154     container = None
 155
 156     # Format name so it's valid
 157     name = name.replace("_", "-").replace(".", "")
 158
 159     client = get_lxd_client()
 160     test_machine = "test-{}-{}".format(
 161         uuid.uuid4().hex[-4:],
 162         name,
 163     )
 164
 165     private_key_path, public_key_path = find_n2vc_ssh_keys()
 166
 167     try:
 168         # create profile w/cloud-init and juju ssh key
 169         if not public_key:
 170             public_key = ""
 171             with open(public_key_path, "r") as f:
 172                 public_key = f.readline()
 173
 174         client.profiles.create(
 175             test_machine,
 176             config={
 177                 'user.user-data': '#cloud-config\nssh_authorized_keys:\n- {}'.format(public_key)},
 178             devices={
 179                 'root': {'path': '/', 'pool': 'default', 'type': 'disk'},
 180                 'eth0': {
 181                     'nictype': 'bridged',
 182                     'parent': 'lxdbr0',
 183                     'type': 'nic'
 184                 }
 185             }
 186         )
 187     except Exception as ex:
 188         debug("Error creating lxd profile {}: {}".format(test_machine, ex))
 189         raise ex
 190
 191     try:
 192         # create lxc machine
 193         config = {
 194             'name': test_machine,
 195             'source': {
 196                 'type': 'image',
 197                 'alias': 'xenial',
 198                 'mode': 'pull',
 199                 'protocol': 'simplestreams',
 200                 'server': 'https://cloud-images.ubuntu.com/releases',
 201             },
 202             'profiles': [test_machine],
 203         }
 204         container = client.containers.create(config, wait=True)
 205         container.start(wait=True)
 206     except Exception as ex:
 207         debug("Error creating lxd container {}: {}".format(test_machine, ex))
 208         # This is a test-ending failure.
 209         raise ex
 210
 211     def wait_for_network(container, timeout=30):
 212         """Wait for eth0 to have an ipv4 address."""
 213         starttime = time.time()
 214         while(time.time() < starttime + timeout):
 215             time.sleep(1)
 216             if 'eth0' in container.state().network:
 217                 addresses = container.state().network['eth0']['addresses']
 218                 if len(addresses) > 0:
 219                     if addresses[0]['family'] == 'inet':
 220                         return addresses[0]
 221         return None
 222
 223     try:
 224         wait_for_network(container)
 225     except Exception as ex:
 226         debug(
 227             "Error waiting for container {} network: {}".format(
 228                 test_machine,
 229                 ex,
 230             )
 231         )
 232
 233     try:
 234         waitcount = 0
 235         while waitcount <= 5:
 236             if is_sshd_running(container):
 237                 break
 238             waitcount += 1
 239             time.sleep(1)
 240         if waitcount >= 5:
 241             debug("couldn't detect sshd running")
 242             raise Exception("Unable to verify container sshd")
 243
 244     except Exception as ex:
 245         debug(
 246             "Error checking sshd status on {}: {}".format(
 247                 test_machine,
 248                 ex,
 249             )
 250         )
 251
 252     # HACK: We need to give sshd a chance to bind to the interface,
 253     # and pylxd's container.execute seems to be broken and fails and/or
 254     # hangs trying to properly check if the service is up.
 255     (exit_code, stdout, stderr) = container.execute([
 256         'ping',
 257         '-c', '5',   # Wait for 5 ECHO_REPLY
 258         '8.8.8.8',   # Ping Google's public DNS
 259         '-W', '15',  # Set a 15 second deadline
 260     ])
 261     if exit_code > 0:
 262         # The network failed
 263         raise Exception("Unable to verify container network")
 264
 265     return container
 266
 267
 268 def is_sshd_running(container):
 269     """Check if sshd is running in the container.
 270
 271     Check to see if the sshd process is running and listening on port 22.
 272
 273     :param container: The container to check
 274     :return boolean: True if sshd is running.
 275     """
 276     debug("Container: {}".format(container))
 277     try:
 278         (rc, stdout, stderr) = container.execute(
 279             ["service", "ssh", "status"]
 280         )
 281         # If the status is a) found and b) running, the exit code will be 0
 282         if rc == 0:
 283             return True
 284     except Exception as ex:
 285         debug("Failed to check sshd service status: {}".format(ex))
 286
 287     return False
 288
 289
 290 def destroy_lxd_container(container):
 291     """Stop and delete a LXD container.
 292
 293     Sometimes we see errors talking to LXD -- ephemerial issues like
 294     load or a bug that's killed the API. We'll do our best to clean
 295     up here, and we should run a cleanup after all tests are finished
 296     to remove any extra containers and profiles belonging to us.
 297     """
 298
 299     if type(container) is bool:
 300         return
 301
 302     name = container.name
 303     debug("Destroying container {}".format(name))
 304
 305     client = get_lxd_client()
 306
 307     def wait_for_stop(timeout=30):
 308         """Wait for eth0 to have an ipv4 address."""
 309         starttime = time.time()
 310         while(time.time() < starttime + timeout):
 311             time.sleep(1)
 312             if container.state == "Stopped":
 313                 return
 314
 315     def wait_for_delete(timeout=30):
 316         starttime = time.time()
 317         while(time.time() < starttime + timeout):
 318             time.sleep(1)
 319             if client.containers.exists(name) is False:
 320                 return
 321
 322     try:
 323         container.stop(wait=False)
 324         wait_for_stop()
 325     except Exception as ex:
 326         debug(
 327             "Error stopping container {}: {}".format(
 328                 name,
 329                 ex,
 330             )
 331         )
 332
 333     try:
 334         container.delete(wait=False)
 335         wait_for_delete()
 336     except Exception as ex:
 337         debug(
 338             "Error deleting container {}: {}".format(
 339                 name,
 340                 ex,
 341             )
 342         )
 343
 344     try:
 345         # Delete the profile created for this container
 346         profile = client.profiles.get(name)
 347         if profile:
 348             profile.delete()
 349     except Exception as ex:
 350         debug(
 351             "Error deleting profile {}: {}".format(
 352                 name,
 353                 ex,
 354             )
 355         )
 356
 357
 358 def find_lxd_config():
 359     """Find the LXD configuration directory."""
 360     paths = []
 361     paths.append(os.path.expanduser("~/.config/lxc"))
 362     paths.append(os.path.expanduser("~/snap/lxd/current/.config/lxc"))
 363
 364     for path in paths:
 365         if os.path.exists(path):
 366             crt = os.path.expanduser("{}/client.crt".format(path))
 367             key = os.path.expanduser("{}/client.key".format(path))
 368             if os.path.exists(crt) and os.path.exists(key):
 369                 return (crt, key)
 370     return (None, None)
 371
 372
 373 def find_n2vc_ssh_keys():
 374     """Find the N2VC ssh keys."""
 375
 376     paths = []
 377     paths.append(os.path.expanduser("~/.ssh/"))
 378
 379     for path in paths:
 380         if os.path.exists(path):
 381             private = os.path.expanduser("{}/id_n2vc_rsa".format(path))
 382             public = os.path.expanduser("{}/id_n2vc_rsa.pub".format(path))
 383             if os.path.exists(private) and os.path.exists(public):
 384                 return (private, public)
 385     return (None, None)
 386
 387
 388 def find_juju_ssh_keys():
 389     """Find the Juju ssh keys."""
 390
 391     paths = []
 392     paths.append(os.path.expanduser("~/.local/share/juju/ssh/"))
 393
 394     for path in paths:
 395         if os.path.exists(path):
 396             private = os.path.expanduser("{}/juju_id_rsa".format(path))
 397             public = os.path.expanduser("{}/juju_id_rsa.pub".format(path))
 398             if os.path.exists(private) and os.path.exists(public):
 399                 return (private, public)
 400     return (None, None)
 401
 402
 403 def get_juju_private_key():
 404     keys = find_juju_ssh_keys()
 405     return keys[0]
 406
 407
 408 def get_lxd_client(host="127.0.0.1", port="8443", verify=False):
 409     """ Get the LXD client."""
 410     client = None
 411     (crt, key) = find_lxd_config()
 412
 413     if crt and key:
 414         client = pylxd.Client(
 415             endpoint="https://{}:{}".format(host, port),
 416             cert=(crt, key),
 417             verify=verify,
 418         )
 419
 420     return client
 421
 422
 423 # TODO: This is marked serial but can be run in parallel with work, including:
 424 # - Fixing an event loop issue; seems that all tests stop when one test stops?
 425
 426
 427 @pytest.mark.serial
 428 class TestN2VC(object):
 429     """TODO:
 430     1. Validator Validation
 431
 432     Automatically validate the descriptors we're using here, unless the test author explicitly wants to skip them. Useful to make sure tests aren't being run against invalid descriptors, validating functionality that may fail against a properly written descriptor.
 433
 434     We need to have a flag (instance variable) that controls this behavior. It may be necessary to skip validation and run against a descriptor implementing features that have not yet been released in the Information Model.
 435     """
 436
 437     """
 438     The six phases of integration testing, for the test itself and each charm?:
 439
 440     setup/teardown_class:
 441     1. Prepare      - Verify the environment and create a new model
 442     2. Deploy       - Mark the test as ready to execute
 443     3. Configure    - Configuration to reach Active state
 444     4. Test         - Execute primitive(s) to verify success
 445     5. Collect      - Collect any useful artifacts for debugging (charm, logs)
 446     6. Destroy      - Destroy the model
 447
 448
 449     1. Prepare      - Building of charm
 450     2. Deploy       - Deploying charm
 451     3. Configure    - Configuration to reach Active state
 452     4. Test         - Execute primitive(s) to verify success
 453     5. Collect      - Collect any useful artifacts for debugging (charm, logs)
 454     6. Destroy      - Destroy the charm
 455
 456     """
 457     @classmethod
 458     def setup_class(self):
 459         """ setup any state specific to the execution of the given class (which
 460         usually contains tests).
 461         """
 462         # Initialize instance variable(s)
 463         self.n2vc = None
 464
 465         # Track internal state for each test run
 466         self.state = {}
 467
 468         # Parse the test's descriptors
 469         self.nsd = get_descriptor(self.NSD_YAML)
 470         self.vnfd = get_descriptor(self.VNFD_YAML)
 471
 472         self.ns_name = self.nsd['name']
 473         self.vnf_name = self.vnfd['name']
 474
 475         self.charms = {}
 476         self.parse_vnf_descriptor()
 477         assert self.charms is not {}
 478
 479         # Track artifacts, like compiled charms, that will need to be removed
 480         self.artifacts = {}
 481
 482         # Build the charm(s) needed for this test
 483         for charm in self.get_charm_names():
 484             self.get_charm(charm)
 485
 486         # A bit of a hack, in order to allow the N2VC callback to run parallel
 487         # to pytest. Test(s) should wait for this flag to change to False
 488         # before returning.
 489         self._running = True
 490         self._stopping = False
 491
 492     @classmethod
 493     def teardown_class(self):
 494         """ teardown any state that was previously setup with a call to
 495         setup_class.
 496         """
 497         debug("Running teardown_class...")
 498         try:
 499
 500             debug("Destroying LXD containers...")
 501             for application in self.state:
 502                 if self.state[application]['container']:
 503                     destroy_lxd_container(self.state[application]['container'])
 504             debug("Destroying LXD containers...done.")
 505
 506             # Logout of N2VC
 507             if self.n2vc:
 508                 debug("teardown_class(): Logging out of N2VC...")
 509                 yield from self.n2vc.logout()
 510                 debug("teardown_class(): Logging out of N2VC...done.")
 511
 512             debug("Running teardown_class...done.")
 513         except Exception as ex:
 514             debug("Exception in teardown_class: {}".format(ex))
 515
 516     @classmethod
 517     def all_charms_active(self):
 518         """Determine if the all deployed charms are active."""
 519         active = 0
 520
 521         for application in self.state:
 522             if 'status' in self.state[application]:
 523                 debug("status of {} is '{}'".format(
 524                     application,
 525                     self.state[application]['status'],
 526                 ))
 527                 if self.state[application]['status'] == 'active':
 528                     active += 1
 529
 530         debug("Active charms: {}/{}".format(
 531             active,
 532             len(self.charms),
 533         ))
 534
 535         if active == len(self.charms):
 536             return True
 537
 538         return False
 539
 540     @classmethod
 541     def are_tests_finished(self):
 542         appcount = len(self.state)
 543
 544         # If we don't have state yet, keep running.
 545         if appcount == 0:
 546             debug("No applications")
 547             return False
 548
 549         if self._stopping:
 550             debug("_stopping is True")
 551             return True
 552
 553         appdone = 0
 554         for application in self.state:
 555             if self.state[application]['done']:
 556                 appdone += 1
 557
 558         debug("{}/{} charms tested".format(appdone, appcount))
 559
 560         if appcount == appdone:
 561             return True
 562
 563         return False
 564
 565     @classmethod
 566     async def running(self, timeout=600):
 567         """Returns if the test is still running.
 568
 569         @param timeout The time, in seconds, to wait for the test to complete.
 570         """
 571         if self.are_tests_finished():
 572             await self.stop()
 573             return False
 574
 575         await asyncio.sleep(30)
 576
 577         return self._running
 578
 579     @classmethod
 580     def get_charm(self, charm):
 581         """Build and return the path to the test charm.
 582
 583         Builds one of the charms in tests/charms/layers and returns the path
 584         to the compiled charm. The charm will automatically be removed when
 585         when the test is complete.
 586
 587         Returns: The path to the built charm or None if `charm build` failed.
 588         """
 589
 590         # Make sure the charm snap is installed
 591         try:
 592             subprocess.check_call(['which', 'charm'])
 593         except subprocess.CalledProcessError:
 594             raise Exception("charm snap not installed.")
 595
 596         if charm not in self.artifacts:
 597             try:
 598                 # Note: This builds the charm under N2VC/tests/charms/builds/
 599                 # Currently, the snap-installed command only has write access
 600                 # to the $HOME (changing in an upcoming release) so writing to
 601                 # /tmp isn't possible at the moment.
 602                 builds = get_charm_path()
 603
 604                 if not os.path.exists("{}/builds/{}".format(builds, charm)):
 605                     cmd = "charm build --no-local-layers {}/{} -o {}/".format(
 606                         get_layer_path(),
 607                         charm,
 608                         builds,
 609                     )
 610                     subprocess.check_call(shlex.split(cmd))
 611
 612             except subprocess.CalledProcessError as e:
 613                 # charm build will return error code 100 if the charm fails
 614                 # the auto-run of charm proof, which we can safely ignore for
 615                 # our CI charms.
 616                 if e.returncode != 100:
 617                     raise Exception("charm build failed: {}.".format(e))
 618
 619             self.artifacts[charm] = {
 620                 'tmpdir': builds,
 621                 'charm': "{}/builds/{}".format(builds, charm),
 622             }
 623
 624         return self.artifacts[charm]['charm']
 625
 626     @classmethod
 627     async def deploy(self, vnf_index, charm, params, loop):
 628         """An inner function to do the deployment of a charm from
 629         either a vdu or vnf.
 630         """
 631
 632         if not self.n2vc:
 633             self.n2vc = get_n2vc(loop=loop)
 634
 635         debug("Creating model for Network Service {}".format(self.ns_name))
 636         await self.n2vc.CreateNetworkService(self.ns_name)
 637
 638         application = self.n2vc.FormatApplicationName(
 639             self.ns_name,
 640             self.vnf_name,
 641             str(vnf_index),
 642         )
 643
 644         # Initialize the state of the application
 645         self.state[application] = {
 646             'status': None,     # Juju status
 647             'container': None,  # lxd container, for proxy charms
 648             'actions': {},      # Actions we've executed
 649             'done': False,      # Are we done testing this charm?
 650             'phase': "deploy",  # What phase is this application in?
 651         }
 652
 653         debug("Deploying charm at {}".format(self.artifacts[charm]))
 654
 655         # If this is a native charm, we need to provision the underlying
 656         # machine ala an LXC container.
 657         machine_spec = {}
 658
 659         if not self.isproxy(application):
 660             debug("Creating container for native charm")
 661             # args = ("default", application, None, None)
 662             self.state[application]['container'] = create_lxd_container(
 663                 name=os.path.basename(__file__)
 664             )
 665
 666             hostname = self.get_container_ip(
 667                 self.state[application]['container'],
 668             )
 669
 670             machine_spec = {
 671                 'host': hostname,
 672                 'user': 'ubuntu',
 673             }
 674
 675         await self.n2vc.DeployCharms(
 676             self.ns_name,
 677             application,
 678             self.vnfd,
 679             self.get_charm(charm),
 680             params,
 681             machine_spec,
 682             self.n2vc_callback,
 683         )
 684
 685     @classmethod
 686     def parse_vnf_descriptor(self):
 687         """Parse the VNF descriptor to make running tests easier.
 688
 689         Parse the charm information in the descriptor to make it easy to write
 690         tests to run again it.
 691
 692         Each charm becomes a dictionary in a list:
 693         [
 694             'is-proxy': True,
 695             'vnf-member-index': 1,
 696             'vnf-name': '',
 697             'charm-name': '',
 698             'initial-config-primitive': {},
 699             'config-primitive': {}
 700         ]
 701         - charm name
 702         - is this a proxy charm?
 703         - what are the initial-config-primitives (day 1)?
 704         - what are the config primitives (day 2)?
 705
 706         """
 707         charms = {}
 708
 709         # You'd think this would be explicit, but it's just an incremental
 710         # value that should be consistent.
 711         vnf_member_index = 0
 712
 713         """Get all vdu and/or vdu config in a descriptor."""
 714         config = self.get_config()
 715         for cfg in config:
 716             if 'juju' in cfg:
 717
 718                 # Get the name to be used for the deployed application
 719                 application_name = n2vc.vnf.N2VC().FormatApplicationName(
 720                     self.ns_name,
 721                     self.vnf_name,
 722                     str(vnf_member_index),
 723                 )
 724
 725                 charm = {
 726                     'application-name': application_name,
 727                     'proxy': True,
 728                     'vnf-member-index': vnf_member_index,
 729                     'vnf-name': self.vnf_name,
 730                     'name': None,
 731                     'initial-config-primitive': {},
 732                     'config-primitive': {},
 733                 }
 734
 735                 juju = cfg['juju']
 736                 charm['name'] = juju['charm']
 737
 738                 if 'proxy' in juju:
 739                     charm['proxy'] = juju['proxy']
 740
 741                 if 'initial-config-primitive' in cfg:
 742                     charm['initial-config-primitive'] = \
 743                         cfg['initial-config-primitive']
 744
 745                 if 'config-primitive' in cfg:
 746                     charm['config-primitive'] = cfg['config-primitive']
 747
 748                 charms[application_name] = charm
 749
 750             # Increment the vnf-member-index
 751             vnf_member_index += 1
 752
 753         self.charms = charms
 754
 755     @classmethod
 756     def isproxy(self, application_name):
 757
 758         assert application_name in self.charms
 759         assert 'proxy' in self.charms[application_name]
 760         assert type(self.charms[application_name]['proxy']) is bool
 761
 762         # debug(self.charms[application_name])
 763         return self.charms[application_name]['proxy']
 764
 765     @classmethod
 766     def get_config(self):
 767         """Return an iterable list of config items (vdu and vnf).
 768
 769         As far as N2VC is concerned, the config section for vdu and vnf are
 770         identical. This joins them together so tests only need to iterate
 771         through one list.
 772         """
 773         configs = []
 774
 775         """Get all vdu and/or vdu config in a descriptor."""
 776         vnf_config = self.vnfd.get("vnf-configuration")
 777         if vnf_config:
 778             juju = vnf_config['juju']
 779             if juju:
 780                 configs.append(vnf_config)
 781
 782         for vdu in self.vnfd['vdu']:
 783             vdu_config = vdu.get('vdu-configuration')
 784             if vdu_config:
 785                 juju = vdu_config['juju']
 786                 if juju:
 787                     configs.append(vdu_config)
 788
 789         return configs
 790
 791     @classmethod
 792     def get_charm_names(self):
 793         """Return a list of charms used by the test descriptor."""
 794
 795         charms = {}
 796
 797         # Check if the VDUs in this VNF have a charm
 798         for config in self.get_config():
 799             juju = config['juju']
 800
 801             name = juju['charm']
 802             if name not in charms:
 803                 charms[name] = 1
 804
 805         return charms.keys()
 806
 807     @classmethod
 808     def get_phase(self, application):
 809         return self.state[application]['phase']
 810
 811     @classmethod
 812     def set_phase(self, application, phase):
 813         self.state[application]['phase'] = phase
 814
 815     @classmethod
 816     async def configure_proxy_charm(self, *args):
 817         """Configure a container for use via ssh."""
 818         (model, application, _, _) = args
 819
 820         try:
 821             if self.get_phase(application) == "deploy":
 822                 self.set_phase(application, "configure")
 823
 824                 debug("Start CreateContainer for {}".format(application))
 825                 self.state[application]['container'] = \
 826                     await self.CreateContainer(*args)
 827                 debug("Done CreateContainer for {}".format(application))
 828
 829                 if self.state[application]['container']:
 830                     debug("Configure {} for container".format(application))
 831                     if await self.configure_ssh_proxy(application):
 832                         await asyncio.sleep(0.1)
 833                         return True
 834                 else:
 835                     debug("Failed to configure container for {}".format(application))
 836             else:
 837                 debug("skipping CreateContainer for {}: {}".format(
 838                     application,
 839                     self.get_phase(application),
 840                 ))
 841
 842         except Exception as ex:
 843             debug("configure_proxy_charm exception: {}".format(ex))
 844         finally:
 845             await asyncio.sleep(0.1)
 846
 847         return False
 848
 849     @classmethod
 850     async def execute_charm_tests(self, *args):
 851         (model, application, _, _) = args
 852
 853         debug("Executing charm test(s) for {}".format(application))
 854
 855         if self.state[application]['done']:
 856             debug("Trying to execute tests against finished charm...aborting")
 857             return False
 858
 859         try:
 860             phase = self.get_phase(application)
 861             # We enter the test phase when after deploy (for native charms) or
 862             # configure, for proxy charms.
 863             if phase in ["deploy", "configure"]:
 864                 self.set_phase(application, "test")
 865                 if self.are_tests_finished():
 866                     raise Exception("Trying to execute init-config on finished test")
 867
 868                 if await self.execute_initial_config_primitives(application):
 869                     # check for metrics
 870                     await self.check_metrics(application)
 871
 872                     debug("Done testing {}".format(application))
 873                     self.state[application]['done'] = True
 874
 875         except Exception as ex:
 876             debug("Exception in execute_charm_tests: {}".format(ex))
 877         finally:
 878             await asyncio.sleep(0.1)
 879
 880         return True
 881
 882     @classmethod
 883     async def CreateContainer(self, *args):
 884         """Create a LXD container for use with a proxy charm.abs
 885
 886         1. Get the public key from the charm via `get-ssh-public-key` action
 887         2. Create container with said key injected for the ubuntu user
 888
 889         Returns a Container object
 890         """
 891         # Create and configure a LXD container for use with a proxy charm.
 892         (model, application, _, _) = args
 893
 894         debug("[CreateContainer] {}".format(args))
 895         container = None
 896
 897         try:
 898             # Execute 'get-ssh-public-key' primitive and get returned value
 899             uuid = await self.n2vc.ExecutePrimitive(
 900                 model,
 901                 application,
 902                 "get-ssh-public-key",
 903                 None,
 904             )
 905
 906             result = await self.n2vc.GetPrimitiveOutput(model, uuid)
 907             pubkey = result['pubkey']
 908
 909             container = create_lxd_container(
 910                 public_key=pubkey,
 911                 name=os.path.basename(__file__)
 912             )
 913
 914             return container
 915         except Exception as ex:
 916             debug("Error creating container: {}".format(ex))
 917             pass
 918
 919         return None
 920
 921     @classmethod
 922     async def stop(self):
 923         """Stop the test.
 924
 925         - Remove charms
 926         - Stop and delete containers
 927         - Logout of N2VC
 928
 929         TODO: Clean up duplicate code between teardown_class() and stop()
 930         """
 931         debug("stop() called")
 932
 933         if self.n2vc and self._running and not self._stopping:
 934             self._running = False
 935             self._stopping = True
 936
 937             # Destroy the network service
 938             try:
 939                 await self.n2vc.DestroyNetworkService(self.ns_name)
 940             except Exception as e:
 941                 debug(
 942                     "Error Destroying Network Service \"{}\": {}".format(
 943                         self.ns_name,
 944                         e,
 945                     )
 946                 )
 947
 948             # Wait for the applications to be removed and delete the containers
 949             for application in self.charms:
 950                 try:
 951
 952                     while True:
 953                         # Wait for the application to be removed
 954                         await asyncio.sleep(10)
 955                         if not await self.n2vc.HasApplication(
 956                             self.ns_name,
 957                             application,
 958                         ):
 959                             break
 960
 961                     # Need to wait for the charm to finish, because native charms
 962                     if self.state[application]['container']:
 963                         debug("Deleting LXD container...")
 964                         destroy_lxd_container(
 965                             self.state[application]['container']
 966                         )
 967                         self.state[application]['container'] = None
 968                         debug("Deleting LXD container...done.")
 969                     else:
 970                         debug("No container found for {}".format(application))
 971                 except Exception as e:
 972                     debug("Error while deleting container: {}".format(e))
 973
 974             # Logout of N2VC
 975             try:
 976                 debug("stop(): Logging out of N2VC...")
 977                 await self.n2vc.logout()
 978                 self.n2vc = None
 979                 debug("stop(): Logging out of N2VC...Done.")
 980             except Exception as ex:
 981                 debug(ex)
 982
 983             # Let the test know we're finished.
 984             debug("Marking test as finished.")
 985             # self._running = False
 986         else:
 987             debug("Skipping stop()")
 988
 989     @classmethod
 990     def get_container_ip(self, container):
 991         """Return the IPv4 address of container's eth0 interface."""
 992         ipaddr = None
 993         if container:
 994             addresses = container.state().network['eth0']['addresses']
 995             # The interface may have more than one address, but we only need
 996             # the first one for testing purposes.
 997             ipaddr = addresses[0]['address']
 998
 999         return ipaddr
1000
1001     @classmethod
1002     async def configure_ssh_proxy(self, application, task=None):
1003         """Configure the proxy charm to use the lxd container.
1004
1005         Configure the charm to use a LXD container as it's VNF.
1006         """
1007         debug("Configuring ssh proxy for {}".format(application))
1008
1009         mgmtaddr = self.get_container_ip(
1010             self.state[application]['container'],
1011         )
1012
1013         debug(
1014             "Setting ssh-hostname for {} to {}".format(
1015                 application,
1016                 mgmtaddr,
1017             )
1018         )
1019
1020         await self.n2vc.ExecutePrimitive(
1021             self.ns_name,
1022             application,
1023             "config",
1024             None,
1025             params={
1026                 'ssh-hostname': mgmtaddr,
1027                 'ssh-username': 'ubuntu',
1028             }
1029         )
1030
1031         return True
1032
1033     @classmethod
1034     async def execute_initial_config_primitives(self, application, task=None):
1035         debug("Executing initial_config_primitives for {}".format(application))
1036         try:
1037             init_config = self.charms[application]
1038
1039             """
1040             The initial-config-primitive is run during deploy but may fail
1041              on some steps because proxy charm access isn't configured.
1042
1043             Re-run those actions so we can inspect the status.
1044             """
1045             uuids = await self.n2vc.ExecuteInitialPrimitives(
1046                 self.ns_name,
1047                 application,
1048                 init_config,
1049             )
1050
1051             """
1052             ExecutePrimitives will return a list of uuids. We need to check the
1053              status of each. The test continues if all Actions succeed, and
1054              fails if any of them fail.
1055             """
1056             await self.wait_for_uuids(application, uuids)
1057             debug("Primitives for {} finished.".format(application))
1058
1059             return True
1060         except Exception as ex:
1061             debug("execute_initial_config_primitives exception: {}".format(ex))
1062
1063         return False
1064
1065     @classmethod
1066     async def check_metrics(self, application, task=None):
1067         """Check and run metrics, if present.
1068
1069         Checks to see if metrics are specified by the charm. If so, collects
1070         the metrics.
1071
1072         If no metrics, then mark the test as finished.
1073         """
1074         if has_metrics(self.charms[application]['name']):
1075             debug("Collecting metrics for {}".format(application))
1076
1077             metrics = await self.n2vc.GetMetrics(
1078                 self.ns_name,
1079                 application,
1080             )
1081
1082             return await self.verify_metrics(application, metrics)
1083
1084     @classmethod
1085     async def verify_metrics(self, application, metrics):
1086         """Verify the charm's metrics.
1087
1088         Verify that the charm has sent metrics successfully.
1089
1090         Stops the test when finished.
1091         """
1092         debug("Verifying metrics for {}: {}".format(application, metrics))
1093
1094         if len(metrics):
1095             return True
1096
1097         else:
1098             # TODO: Ran into a case where it took 9 attempts before metrics
1099             # were available; the controller is slow sometimes.
1100             await asyncio.sleep(30)
1101             return await self.check_metrics(application)
1102
1103     @classmethod
1104     async def wait_for_uuids(self, application, uuids):
1105         """Wait for primitives to execute.
1106
1107         The task will provide a list of uuids representing primitives that are
1108         queued to run.
1109         """
1110         debug("Waiting for uuids for {}: {}".format(application, uuids))
1111         waitfor = len(uuids)
1112         finished = 0
1113
1114         while waitfor > finished:
1115             for uid in uuids:
1116                 await asyncio.sleep(10)
1117
1118                 if uuid not in self.state[application]['actions']:
1119                     self.state[application]['actions'][uid] = "pending"
1120
1121                 status = self.state[application]['actions'][uid]
1122
1123                 # Have we already marked this as done?
1124                 if status in ["pending", "running"]:
1125
1126                     debug("Getting status of {} ({})...".format(uid, status))
1127                     status = await self.n2vc.GetPrimitiveStatus(
1128                         self.ns_name,
1129                         uid,
1130                     )
1131                     debug("...state of {} is {}".format(uid, status))
1132                     self.state[application]['actions'][uid] = status
1133
1134                     if status in ['completed', 'failed']:
1135                         finished += 1
1136
1137             debug("{}/{} actions complete".format(finished, waitfor))
1138
1139             # Wait for the primitive to finish and try again
1140             if waitfor > finished:
1141                 debug("Waiting 10s for action to finish...")
1142                 await asyncio.sleep(10)
1143
1144     @classmethod
1145     def n2vc_callback(self, *args, **kwargs):
1146         (model, application, status, message) = args
1147         # debug("callback: {}".format(args))
1148
1149         if application not in self.state:
1150             # Initialize the state of the application
1151             self.state[application] = {
1152                 'status': None,     # Juju status
1153                 'container': None,  # lxd container, for proxy charms
1154                 'actions': {},      # Actions we've executed
1155                 'done': False,      # Are we done testing this charm?
1156                 'phase': "deploy",  # What phase is this application in?
1157             }
1158
1159         self.state[application]['status'] = status
1160
1161         if status in ['waiting', 'maintenance', 'unknown']:
1162             # Nothing to do for these
1163             return
1164
1165         debug("callback: {}".format(args))
1166
1167         if self.state[application]['done']:
1168             debug("{} is done".format(application))
1169             return
1170
1171         if status in ['error']:
1172             # To test broken charms, if a charm enters an error state we should
1173             # end the test
1174             debug("{} is in an error state, stop the test.".format(application))
1175             # asyncio.ensure_future(self.stop())
1176             self.state[application]['done'] = True
1177             assert False
1178
1179         if status in ["blocked"] and self.isproxy(application):
1180             if self.state[application]['phase'] == "deploy":
1181                 debug("Configuring proxy charm for {}".format(application))
1182                 asyncio.ensure_future(self.configure_proxy_charm(*args))
1183
1184         elif status in ["active"]:
1185             """When a charm is active, we can assume that it has been properly
1186             configured (not blocked), regardless of if it's a proxy or not.
1187
1188             All primitives should be complete by init_config_primitive
1189             """
1190             asyncio.ensure_future(self.execute_charm_tests(*args))