tests/base.py

   1 #!/usr/bin/env python3
   2 import asyncio
   3 import datetime
   4 import logging
   5 import n2vc.vnf
   6 import pylxd
   7 import pytest
   8 import os
   9 import shlex
  10 import subprocess
  11 import time
  12 import uuid
  13 import yaml
  14
  15 from juju.controller import Controller
  16
  17 # Disable InsecureRequestWarning w/LXD
  18 import urllib3
  19 urllib3.disable_warnings()
  20 logging.getLogger("urllib3").setLevel(logging.WARNING)
  21
  22 here = os.path.dirname(os.path.realpath(__file__))
  23
  24
  25 def is_bootstrapped():
  26     result = subprocess.run(['juju', 'switch'], stdout=subprocess.PIPE)
  27     return (
  28         result.returncode == 0 and
  29         len(result.stdout.decode().strip()) > 0)
  30
  31
  32 bootstrapped = pytest.mark.skipif(
  33     not is_bootstrapped(),
  34     reason='bootstrapped Juju environment required')
  35
  36
  37 class CleanController():
  38     """
  39     Context manager that automatically connects and disconnects from
  40     the currently active controller.
  41
  42     Note: Unlike CleanModel, this will not create a new controller for you,
  43     and an active controller must already be available.
  44     """
  45     def __init__(self):
  46         self._controller = None
  47
  48     async def __aenter__(self):
  49         self._controller = Controller()
  50         await self._controller.connect()
  51         return self._controller
  52
  53     async def __aexit__(self, exc_type, exc, tb):
  54         await self._controller.disconnect()
  55
  56
  57 def debug(msg):
  58     """Format debug messages in a consistent way."""
  59     now = datetime.datetime.now()
  60
  61     # TODO: Decide on the best way to log. Output from `logging.debug` shows up
  62     # when a test fails, but print() will always show up when running tox with
  63     # `-s`, which is really useful for debugging single tests without having to
  64     # insert a False assert to see the log.
  65     logging.debug(
  66         "[{}] {}".format(now.strftime('%Y-%m-%dT%H:%M:%S'), msg)
  67     )
  68     print(
  69         "[{}] {}".format(now.strftime('%Y-%m-%dT%H:%M:%S'), msg)
  70     )
  71
  72
  73 def get_charm_path():
  74     return "{}/charms".format(here)
  75
  76
  77 def get_layer_path():
  78     return "{}/charms/layers".format(here)
  79
  80
  81 def collect_metrics(application):
  82     """Invoke Juju's metrics collector.
  83
  84     Caveat: this shells out to the `juju collect-metrics` command, rather than
  85     making an API call. At the time of writing, that API is not exposed through
  86     the client library.
  87     """
  88
  89     try:
  90         subprocess.check_call(['juju', 'collect-metrics', application])
  91     except subprocess.CalledProcessError as e:
  92         raise Exception("Unable to collect metrics: {}".format(e))
  93
  94
  95 def has_metrics(charm):
  96     """Check if a charm has metrics defined."""
  97     metricsyaml = "{}/{}/metrics.yaml".format(
  98         get_layer_path(),
  99         charm,
 100     )
 101     if os.path.exists(metricsyaml):
 102         return True
 103     return False
 104
 105
 106 def get_descriptor(descriptor):
 107     desc = None
 108     try:
 109         tmp = yaml.load(descriptor)
 110
 111         # Remove the envelope
 112         root = list(tmp.keys())[0]
 113         if root == "nsd:nsd-catalog":
 114             desc = tmp['nsd:nsd-catalog']['nsd'][0]
 115         elif root == "vnfd:vnfd-catalog":
 116             desc = tmp['vnfd:vnfd-catalog']['vnfd'][0]
 117     except ValueError:
 118         assert False
 119     return desc
 120
 121
 122 def get_n2vc(loop=None):
 123     """Return an instance of N2VC.VNF."""
 124     log = logging.getLogger()
 125     log.level = logging.DEBUG
 126
 127     # Extract parameters from the environment in order to run our test
 128     vca_host = os.getenv('VCA_HOST', '127.0.0.1')
 129     vca_port = os.getenv('VCA_PORT', 17070)
 130     vca_user = os.getenv('VCA_USER', 'admin')
 131     vca_charms = os.getenv('VCA_CHARMS', None)
 132     vca_secret = os.getenv('VCA_SECRET', None)
 133
 134     client = n2vc.vnf.N2VC(
 135         log=log,
 136         server=vca_host,
 137         port=vca_port,
 138         user=vca_user,
 139         secret=vca_secret,
 140         artifacts=vca_charms,
 141         loop=loop
 142     )
 143     return client
 144
 145
 146 def create_lxd_container(public_key=None, name="test_name"):
 147     """
 148     Returns a container object
 149
 150     If public_key isn't set, we'll use the Juju ssh key
 151
 152     :param public_key: The public key to inject into the container
 153     :param name: The name of the test being run
 154     """
 155     container = None
 156
 157     # Format name so it's valid
 158     name = name.replace("_", "-").replace(".", "")
 159
 160     client = get_lxd_client()
 161     test_machine = "test-{}-{}".format(
 162         uuid.uuid4().hex[-4:],
 163         name,
 164     )
 165
 166     private_key_path, public_key_path = find_n2vc_ssh_keys()
 167
 168     try:
 169         # create profile w/cloud-init and juju ssh key
 170         if not public_key:
 171             public_key = ""
 172             with open(public_key_path, "r") as f:
 173                 public_key = f.readline()
 174
 175         client.profiles.create(
 176             test_machine,
 177             config={
 178                 'user.user-data': '#cloud-config\nssh_authorized_keys:\n- {}'.format(public_key)},
 179             devices={
 180                 'root': {'path': '/', 'pool': 'default', 'type': 'disk'},
 181                 'eth0': {
 182                     'nictype': 'bridged',
 183                     'parent': 'lxdbr0',
 184                     'type': 'nic'
 185                 }
 186             }
 187         )
 188     except Exception as ex:
 189         debug("Error creating lxd profile {}: {}".format(test_machine, ex))
 190         raise ex
 191
 192     try:
 193         # create lxc machine
 194         config = {
 195             'name': test_machine,
 196             'source': {
 197                 'type': 'image',
 198                 'alias': 'xenial',
 199                 'mode': 'pull',
 200                 'protocol': 'simplestreams',
 201                 'server': 'https://cloud-images.ubuntu.com/releases',
 202             },
 203             'profiles': [test_machine],
 204         }
 205         container = client.containers.create(config, wait=True)
 206         container.start(wait=True)
 207     except Exception as ex:
 208         debug("Error creating lxd container {}: {}".format(test_machine, ex))
 209         # This is a test-ending failure.
 210         raise ex
 211
 212     def wait_for_network(container, timeout=30):
 213         """Wait for eth0 to have an ipv4 address."""
 214         starttime = time.time()
 215         while(time.time() < starttime + timeout):
 216             time.sleep(1)
 217             if 'eth0' in container.state().network:
 218                 addresses = container.state().network['eth0']['addresses']
 219                 if len(addresses) > 0:
 220                     if addresses[0]['family'] == 'inet':
 221                         return addresses[0]
 222         return None
 223
 224     try:
 225         wait_for_network(container)
 226     except Exception as ex:
 227         debug(
 228             "Error waiting for container {} network: {}".format(
 229                 test_machine,
 230                 ex,
 231             )
 232         )
 233
 234     # HACK: We need to give sshd a chance to bind to the interface,
 235     # and pylxd's container.execute seems to be broken and fails and/or
 236     # hangs trying to properly check if the service is up.
 237     (exit_code, stdout, stderr) = container.execute([
 238         'ping',
 239         '-c', '5',   # Wait for 5 ECHO_REPLY
 240         '8.8.8.8',   # Ping Google's public DNS
 241         '-W', '15',  # Set a 15 second deadline
 242     ])
 243     if exit_code > 0:
 244         # The network failed
 245         raise Exception("Unable to verify container network")
 246
 247     return container
 248
 249
 250 def destroy_lxd_container(container):
 251     """Stop and delete a LXD container.
 252
 253     Sometimes we see errors talking to LXD -- ephemerial issues like
 254     load or a bug that's killed the API. We'll do our best to clean
 255     up here, and we should run a cleanup after all tests are finished
 256     to remove any extra containers and profiles belonging to us.
 257     """
 258
 259     if type(container) is bool:
 260         return
 261
 262     name = container.name
 263     debug("Destroying container {}".format(name))
 264
 265     client = get_lxd_client()
 266
 267     def wait_for_stop(timeout=30):
 268         """Wait for eth0 to have an ipv4 address."""
 269         starttime = time.time()
 270         while(time.time() < starttime + timeout):
 271             time.sleep(1)
 272             if container.state == "Stopped":
 273                 return
 274
 275     def wait_for_delete(timeout=30):
 276         starttime = time.time()
 277         while(time.time() < starttime + timeout):
 278             time.sleep(1)
 279             if client.containers.exists(name) is False:
 280                 return
 281
 282     try:
 283         container.stop(wait=False)
 284         wait_for_stop()
 285     except Exception as ex:
 286         debug(
 287             "Error stopping container {}: {}".format(
 288                 name,
 289                 ex,
 290             )
 291         )
 292
 293     try:
 294         container.delete(wait=False)
 295         wait_for_delete()
 296     except Exception as ex:
 297         debug(
 298             "Error deleting container {}: {}".format(
 299                 name,
 300                 ex,
 301             )
 302         )
 303
 304     try:
 305         # Delete the profile created for this container
 306         profile = client.profiles.get(name)
 307         if profile:
 308             profile.delete()
 309     except Exception as ex:
 310         debug(
 311             "Error deleting profile {}: {}".format(
 312                 name,
 313                 ex,
 314             )
 315         )
 316
 317
 318 def find_lxd_config():
 319     """Find the LXD configuration directory."""
 320     paths = []
 321     paths.append(os.path.expanduser("~/.config/lxc"))
 322     paths.append(os.path.expanduser("~/snap/lxd/current/.config/lxc"))
 323
 324     for path in paths:
 325         if os.path.exists(path):
 326             crt = os.path.expanduser("{}/client.crt".format(path))
 327             key = os.path.expanduser("{}/client.key".format(path))
 328             if os.path.exists(crt) and os.path.exists(key):
 329                 return (crt, key)
 330     return (None, None)
 331
 332
 333 def find_n2vc_ssh_keys():
 334     """Find the N2VC ssh keys."""
 335
 336     paths = []
 337     paths.append(os.path.expanduser("~/.ssh/"))
 338
 339     for path in paths:
 340         if os.path.exists(path):
 341             private = os.path.expanduser("{}/id_n2vc_rsa".format(path))
 342             public = os.path.expanduser("{}/id_n2vc_rsa.pub".format(path))
 343             if os.path.exists(private) and os.path.exists(public):
 344                 return (private, public)
 345     return (None, None)
 346
 347
 348 def find_juju_ssh_keys():
 349     """Find the Juju ssh keys."""
 350
 351     paths = []
 352     paths.append(os.path.expanduser("~/.local/share/juju/ssh/"))
 353
 354     for path in paths:
 355         if os.path.exists(path):
 356             private = os.path.expanduser("{}/juju_id_rsa".format(path))
 357             public = os.path.expanduser("{}/juju_id_rsa.pub".format(path))
 358             if os.path.exists(private) and os.path.exists(public):
 359                 return (private, public)
 360     return (None, None)
 361
 362
 363 def get_juju_private_key():
 364     keys = find_juju_ssh_keys()
 365     return keys[0]
 366
 367
 368 def get_lxd_client(host="127.0.0.1", port="8443", verify=False):
 369     """ Get the LXD client."""
 370     client = None
 371     (crt, key) = find_lxd_config()
 372
 373     if crt and key:
 374         client = pylxd.Client(
 375             endpoint="https://{}:{}".format(host, port),
 376             cert=(crt, key),
 377             verify=verify,
 378         )
 379
 380     return client
 381
 382
 383 # TODO: This is marked serial but can be run in parallel with work, including:
 384 # - Fixing an event loop issue; seems that all tests stop when one test stops?
 385
 386
 387 @pytest.mark.serial
 388 class TestN2VC(object):
 389     """TODO:
 390     1. Validator Validation
 391
 392     Automatically validate the descriptors we're using here, unless the test author explicitly wants to skip them. Useful to make sure tests aren't being run against invalid descriptors, validating functionality that may fail against a properly written descriptor.
 393
 394     We need to have a flag (instance variable) that controls this behavior. It may be necessary to skip validation and run against a descriptor implementing features that have not yet been released in the Information Model.
 395     """
 396
 397     """
 398     The six phases of integration testing, for the test itself and each charm?:
 399
 400     setup/teardown_class:
 401     1. Prepare      - Verify the environment and create a new model
 402     2. Deploy       - Mark the test as ready to execute
 403     3. Configure    - Configuration to reach Active state
 404     4. Test         - Execute primitive(s) to verify success
 405     5. Collect      - Collect any useful artifacts for debugging (charm, logs)
 406     6. Destroy      - Destroy the model
 407
 408
 409     1. Prepare      - Building of charm
 410     2. Deploy       - Deploying charm
 411     3. Configure    - Configuration to reach Active state
 412     4. Test         - Execute primitive(s) to verify success
 413     5. Collect      - Collect any useful artifacts for debugging (charm, logs)
 414     6. Destroy      - Destroy the charm
 415
 416     """
 417     @classmethod
 418     def setup_class(self):
 419         """ setup any state specific to the execution of the given class (which
 420         usually contains tests).
 421         """
 422         # Initialize instance variable(s)
 423         self.n2vc = None
 424
 425         # Track internal state for each test run
 426         self.state = {}
 427
 428         # Parse the test's descriptors
 429         self.nsd = get_descriptor(self.NSD_YAML)
 430         self.vnfd = get_descriptor(self.VNFD_YAML)
 431
 432         self.ns_name = self.nsd['name']
 433         self.vnf_name = self.vnfd['name']
 434
 435         self.charms = {}
 436         self.parse_vnf_descriptor()
 437         assert self.charms is not {}
 438
 439         # Track artifacts, like compiled charms, that will need to be removed
 440         self.artifacts = {}
 441
 442         # Build the charm(s) needed for this test
 443         for charm in self.get_charm_names():
 444             self.get_charm(charm)
 445
 446         # A bit of a hack, in order to allow the N2VC callback to run parallel
 447         # to pytest. Test(s) should wait for this flag to change to False
 448         # before returning.
 449         self._running = True
 450         self._stopping = False
 451
 452     @classmethod
 453     def teardown_class(self):
 454         """ teardown any state that was previously setup with a call to
 455         setup_class.
 456         """
 457         debug("Running teardown_class...")
 458         try:
 459
 460             debug("Destroying LXD containers...")
 461             for application in self.state:
 462                 if self.state[application]['container']:
 463                     destroy_lxd_container(self.state[application]['container'])
 464             debug("Destroying LXD containers...done.")
 465
 466             # Logout of N2VC
 467             if self.n2vc:
 468                 debug("teardown_class(): Logging out of N2VC...")
 469                 yield from self.n2vc.logout()
 470                 debug("teardown_class(): Logging out of N2VC...done.")
 471
 472             debug("Running teardown_class...done.")
 473         except Exception as ex:
 474             debug("Exception in teardown_class: {}".format(ex))
 475
 476     @classmethod
 477     def all_charms_active(self):
 478         """Determine if the all deployed charms are active."""
 479         active = 0
 480
 481         for application in self.state:
 482             if 'status' in self.state[application]:
 483                 debug("status of {} is '{}'".format(
 484                     application,
 485                     self.state[application]['status'],
 486                 ))
 487                 if self.state[application]['status'] == 'active':
 488                     active += 1
 489
 490         debug("Active charms: {}/{}".format(
 491             active,
 492             len(self.charms),
 493         ))
 494
 495         if active == len(self.charms):
 496             return True
 497
 498         return False
 499
 500     @classmethod
 501     def are_tests_finished(self):
 502         appcount = len(self.state)
 503
 504         # If we don't have state yet, keep running.
 505         if appcount == 0:
 506             debug("No applications")
 507             return False
 508
 509         if self._stopping:
 510             debug("_stopping is True")
 511             return True
 512
 513         appdone = 0
 514         for application in self.state:
 515             if self.state[application]['done']:
 516                 appdone += 1
 517
 518         debug("{}/{} charms tested".format(appdone, appcount))
 519
 520         if appcount == appdone:
 521             return True
 522
 523         return False
 524
 525     @classmethod
 526     async def running(self, timeout=600):
 527         """Returns if the test is still running.
 528
 529         @param timeout The time, in seconds, to wait for the test to complete.
 530         """
 531         if self.are_tests_finished():
 532             await self.stop()
 533             return False
 534
 535         await asyncio.sleep(30)
 536
 537         return self._running
 538
 539     @classmethod
 540     def get_charm(self, charm):
 541         """Build and return the path to the test charm.
 542
 543         Builds one of the charms in tests/charms/layers and returns the path
 544         to the compiled charm. The charm will automatically be removed when
 545         when the test is complete.
 546
 547         Returns: The path to the built charm or None if `charm build` failed.
 548         """
 549
 550         # Make sure the charm snap is installed
 551         try:
 552             subprocess.check_call(['which', 'charm'])
 553         except subprocess.CalledProcessError:
 554             raise Exception("charm snap not installed.")
 555
 556         if charm not in self.artifacts:
 557             try:
 558                 # Note: This builds the charm under N2VC/tests/charms/builds/
 559                 # Currently, the snap-installed command only has write access
 560                 # to the $HOME (changing in an upcoming release) so writing to
 561                 # /tmp isn't possible at the moment.
 562                 builds = get_charm_path()
 563
 564                 if not os.path.exists("{}/builds/{}".format(builds, charm)):
 565                     cmd = "charm build --no-local-layers {}/{} -o {}/".format(
 566                         get_layer_path(),
 567                         charm,
 568                         builds,
 569                     )
 570                     subprocess.check_call(shlex.split(cmd))
 571
 572                 self.artifacts[charm] = {
 573                     'tmpdir': builds,
 574                     'charm': "{}/builds/{}".format(builds, charm),
 575                 }
 576             except subprocess.CalledProcessError as e:
 577                 raise Exception("charm build failed: {}.".format(e))
 578
 579         return self.artifacts[charm]['charm']
 580
 581     @classmethod
 582     async def deploy(self, vnf_index, charm, params, loop):
 583         """An inner function to do the deployment of a charm from
 584         either a vdu or vnf.
 585         """
 586
 587         if not self.n2vc:
 588             self.n2vc = get_n2vc(loop=loop)
 589
 590         application = self.n2vc.FormatApplicationName(
 591             self.ns_name,
 592             self.vnf_name,
 593             str(vnf_index),
 594         )
 595
 596         # Initialize the state of the application
 597         self.state[application] = {
 598             'status': None,     # Juju status
 599             'container': None,  # lxd container, for proxy charms
 600             'actions': {},      # Actions we've executed
 601             'done': False,      # Are we done testing this charm?
 602             'phase': "deploy",  # What phase is this application in?
 603         }
 604
 605         debug("Deploying charm at {}".format(self.artifacts[charm]))
 606
 607         # If this is a native charm, we need to provision the underlying
 608         # machine ala an LXC container.
 609         machine_spec = {}
 610
 611         if not self.isproxy(application):
 612             debug("Creating container for native charm")
 613             # args = ("default", application, None, None)
 614             self.state[application]['container'] = create_lxd_container(
 615                 name=os.path.basename(__file__)
 616             )
 617
 618             hostname = self.get_container_ip(
 619                 self.state[application]['container'],
 620             )
 621
 622             machine_spec = {
 623                 'host': hostname,
 624                 'user': 'ubuntu',
 625             }
 626
 627         await self.n2vc.DeployCharms(
 628             self.ns_name,
 629             application,
 630             self.vnfd,
 631             self.get_charm(charm),
 632             params,
 633             machine_spec,
 634             self.n2vc_callback,
 635         )
 636
 637     @classmethod
 638     def parse_vnf_descriptor(self):
 639         """Parse the VNF descriptor to make running tests easier.
 640
 641         Parse the charm information in the descriptor to make it easy to write
 642         tests to run again it.
 643
 644         Each charm becomes a dictionary in a list:
 645         [
 646             'is-proxy': True,
 647             'vnf-member-index': 1,
 648             'vnf-name': '',
 649             'charm-name': '',
 650             'initial-config-primitive': {},
 651             'config-primitive': {}
 652         ]
 653         - charm name
 654         - is this a proxy charm?
 655         - what are the initial-config-primitives (day 1)?
 656         - what are the config primitives (day 2)?
 657
 658         """
 659         charms = {}
 660
 661         # You'd think this would be explicit, but it's just an incremental
 662         # value that should be consistent.
 663         vnf_member_index = 0
 664
 665         """Get all vdu and/or vdu config in a descriptor."""
 666         config = self.get_config()
 667         for cfg in config:
 668             if 'juju' in cfg:
 669
 670                 # Get the name to be used for the deployed application
 671                 application_name = n2vc.vnf.N2VC().FormatApplicationName(
 672                     self.ns_name,
 673                     self.vnf_name,
 674                     str(vnf_member_index),
 675                 )
 676
 677                 charm = {
 678                     'application-name': application_name,
 679                     'proxy': True,
 680                     'vnf-member-index': vnf_member_index,
 681                     'vnf-name': self.vnf_name,
 682                     'name': None,
 683                     'initial-config-primitive': {},
 684                     'config-primitive': {},
 685                 }
 686
 687                 juju = cfg['juju']
 688                 charm['name'] = juju['charm']
 689
 690                 if 'proxy' in juju:
 691                     charm['proxy'] = juju['proxy']
 692
 693                 if 'initial-config-primitive' in cfg:
 694                     charm['initial-config-primitive'] = \
 695                         cfg['initial-config-primitive']
 696
 697                 if 'config-primitive' in cfg:
 698                     charm['config-primitive'] = cfg['config-primitive']
 699
 700                 charms[application_name] = charm
 701
 702             # Increment the vnf-member-index
 703             vnf_member_index += 1
 704
 705         self.charms = charms
 706
 707     @classmethod
 708     def isproxy(self, application_name):
 709
 710         assert application_name in self.charms
 711         assert 'proxy' in self.charms[application_name]
 712         assert type(self.charms[application_name]['proxy']) is bool
 713
 714         # debug(self.charms[application_name])
 715         return self.charms[application_name]['proxy']
 716
 717     @classmethod
 718     def get_config(self):
 719         """Return an iterable list of config items (vdu and vnf).
 720
 721         As far as N2VC is concerned, the config section for vdu and vnf are
 722         identical. This joins them together so tests only need to iterate
 723         through one list.
 724         """
 725         configs = []
 726
 727         """Get all vdu and/or vdu config in a descriptor."""
 728         vnf_config = self.vnfd.get("vnf-configuration")
 729         if vnf_config:
 730             juju = vnf_config['juju']
 731             if juju:
 732                 configs.append(vnf_config)
 733
 734         for vdu in self.vnfd['vdu']:
 735             vdu_config = vdu.get('vdu-configuration')
 736             if vdu_config:
 737                 juju = vdu_config['juju']
 738                 if juju:
 739                     configs.append(vdu_config)
 740
 741         return configs
 742
 743     @classmethod
 744     def get_charm_names(self):
 745         """Return a list of charms used by the test descriptor."""
 746
 747         charms = {}
 748
 749         # Check if the VDUs in this VNF have a charm
 750         for config in self.get_config():
 751             juju = config['juju']
 752
 753             name = juju['charm']
 754             if name not in charms:
 755                 charms[name] = 1
 756
 757         return charms.keys()
 758
 759     @classmethod
 760     def get_phase(self, application):
 761         return self.state[application]['phase']
 762
 763     @classmethod
 764     def set_phase(self, application, phase):
 765         self.state[application]['phase'] = phase
 766
 767     @classmethod
 768     async def configure_proxy_charm(self, *args):
 769         """Configure a container for use via ssh."""
 770         (model, application, _, _) = args
 771
 772         try:
 773             if self.get_phase(application) == "deploy":
 774                 self.set_phase(application, "configure")
 775
 776                 debug("Start CreateContainer for {}".format(application))
 777                 self.state[application]['container'] = \
 778                     await self.CreateContainer(*args)
 779                 debug("Done CreateContainer for {}".format(application))
 780
 781                 if self.state[application]['container']:
 782                     debug("Configure {} for container".format(application))
 783                     if await self.configure_ssh_proxy(application):
 784                         await asyncio.sleep(0.1)
 785                         return True
 786                 else:
 787                     debug("Failed to configure container for {}".format(application))
 788             else:
 789                 debug("skipping CreateContainer for {}: {}".format(
 790                     application,
 791                     self.get_phase(application),
 792                 ))
 793
 794         except Exception as ex:
 795             debug("configure_proxy_charm exception: {}".format(ex))
 796         finally:
 797             await asyncio.sleep(0.1)
 798
 799         return False
 800
 801     @classmethod
 802     async def execute_charm_tests(self, *args):
 803         (model, application, _, _) = args
 804
 805         debug("Executing charm test(s) for {}".format(application))
 806
 807         if self.state[application]['done']:
 808             debug("Trying to execute tests against finished charm...aborting")
 809             return False
 810
 811         try:
 812             phase = self.get_phase(application)
 813             # We enter the test phase when after deploy (for native charms) or
 814             # configure, for proxy charms.
 815             if phase in ["deploy", "configure"]:
 816                 self.set_phase(application, "test")
 817                 if self.are_tests_finished():
 818                     raise Exception("Trying to execute init-config on finished test")
 819
 820                 if await self.execute_initial_config_primitives(application):
 821                     # check for metrics
 822                     await self.check_metrics(application)
 823
 824                     debug("Done testing {}".format(application))
 825                     self.state[application]['done'] = True
 826
 827         except Exception as ex:
 828             debug("Exception in execute_charm_tests: {}".format(ex))
 829         finally:
 830             await asyncio.sleep(0.1)
 831
 832         return True
 833
 834     @classmethod
 835     async def CreateContainer(self, *args):
 836         """Create a LXD container for use with a proxy charm.abs
 837
 838         1. Get the public key from the charm via `get-ssh-public-key` action
 839         2. Create container with said key injected for the ubuntu user
 840
 841         Returns a Container object
 842         """
 843         # Create and configure a LXD container for use with a proxy charm.
 844         (model, application, _, _) = args
 845
 846         debug("[CreateContainer] {}".format(args))
 847         container = None
 848
 849         try:
 850             # Execute 'get-ssh-public-key' primitive and get returned value
 851             uuid = await self.n2vc.ExecutePrimitive(
 852                 model,
 853                 application,
 854                 "get-ssh-public-key",
 855                 None,
 856             )
 857
 858             result = await self.n2vc.GetPrimitiveOutput(model, uuid)
 859             pubkey = result['pubkey']
 860
 861             container = create_lxd_container(
 862                 public_key=pubkey,
 863                 name=os.path.basename(__file__)
 864             )
 865
 866             return container
 867         except Exception as ex:
 868             debug("Error creating container: {}".format(ex))
 869             pass
 870
 871         return None
 872
 873     @classmethod
 874     async def stop(self):
 875         """Stop the test.
 876
 877         - Remove charms
 878         - Stop and delete containers
 879         - Logout of N2VC
 880
 881         TODO: Clean up duplicate code between teardown_class() and stop()
 882         """
 883         debug("stop() called")
 884
 885         if self.n2vc and self._running and not self._stopping:
 886             self._running = False
 887             self._stopping = True
 888
 889             for application in self.charms:
 890                 try:
 891                     await self.n2vc.RemoveCharms(self.ns_name, application)
 892
 893                     while True:
 894                         # Wait for the application to be removed
 895                         await asyncio.sleep(10)
 896                         if not await self.n2vc.HasApplication(
 897                             self.ns_name,
 898                             application,
 899                         ):
 900                             break
 901
 902                     # Need to wait for the charm to finish, because native charms
 903                     if self.state[application]['container']:
 904                         debug("Deleting LXD container...")
 905                         destroy_lxd_container(
 906                             self.state[application]['container']
 907                         )
 908                         self.state[application]['container'] = None
 909                         debug("Deleting LXD container...done.")
 910                     else:
 911                         debug("No container found for {}".format(application))
 912                 except Exception as e:
 913                     debug("Error while deleting container: {}".format(e))
 914
 915             # Logout of N2VC
 916             try:
 917                 debug("stop(): Logging out of N2VC...")
 918                 await self.n2vc.logout()
 919                 self.n2vc = None
 920                 debug("stop(): Logging out of N2VC...Done.")
 921             except Exception as ex:
 922                 debug(ex)
 923
 924             # Let the test know we're finished.
 925             debug("Marking test as finished.")
 926             # self._running = False
 927         else:
 928             debug("Skipping stop()")
 929
 930     @classmethod
 931     def get_container_ip(self, container):
 932         """Return the IPv4 address of container's eth0 interface."""
 933         ipaddr = None
 934         if container:
 935             addresses = container.state().network['eth0']['addresses']
 936             # The interface may have more than one address, but we only need
 937             # the first one for testing purposes.
 938             ipaddr = addresses[0]['address']
 939
 940         return ipaddr
 941
 942     @classmethod
 943     async def configure_ssh_proxy(self, application, task=None):
 944         """Configure the proxy charm to use the lxd container.
 945
 946         Configure the charm to use a LXD container as it's VNF.
 947         """
 948         debug("Configuring ssh proxy for {}".format(application))
 949
 950         mgmtaddr = self.get_container_ip(
 951             self.state[application]['container'],
 952         )
 953
 954         debug(
 955             "Setting ssh-hostname for {} to {}".format(
 956                 application,
 957                 mgmtaddr,
 958             )
 959         )
 960
 961         await self.n2vc.ExecutePrimitive(
 962             self.ns_name,
 963             application,
 964             "config",
 965             None,
 966             params={
 967                 'ssh-hostname': mgmtaddr,
 968                 'ssh-username': 'ubuntu',
 969             }
 970         )
 971
 972         return True
 973
 974     @classmethod
 975     async def execute_initial_config_primitives(self, application, task=None):
 976         debug("Executing initial_config_primitives for {}".format(application))
 977         try:
 978             init_config = self.charms[application]
 979
 980             """
 981             The initial-config-primitive is run during deploy but may fail
 982              on some steps because proxy charm access isn't configured.
 983
 984             Re-run those actions so we can inspect the status.
 985             """
 986             uuids = await self.n2vc.ExecuteInitialPrimitives(
 987                 self.ns_name,
 988                 application,
 989                 init_config,
 990             )
 991
 992             """
 993             ExecutePrimitives will return a list of uuids. We need to check the
 994              status of each. The test continues if all Actions succeed, and
 995              fails if any of them fail.
 996             """
 997             await self.wait_for_uuids(application, uuids)
 998             debug("Primitives for {} finished.".format(application))
 999
1000             return True
1001         except Exception as ex:
1002             debug("execute_initial_config_primitives exception: {}".format(ex))
1003
1004         return False
1005
1006     @classmethod
1007     async def check_metrics(self, application, task=None):
1008         """Check and run metrics, if present.
1009
1010         Checks to see if metrics are specified by the charm. If so, collects
1011         the metrics.
1012
1013         If no metrics, then mark the test as finished.
1014         """
1015         if has_metrics(self.charms[application]['name']):
1016             debug("Collecting metrics for {}".format(application))
1017
1018             metrics = await self.n2vc.GetMetrics(
1019                 self.ns_name,
1020                 application,
1021             )
1022
1023             return await self.verify_metrics(application, metrics)
1024
1025     @classmethod
1026     async def verify_metrics(self, application, metrics):
1027         """Verify the charm's metrics.
1028
1029         Verify that the charm has sent metrics successfully.
1030
1031         Stops the test when finished.
1032         """
1033         debug("Verifying metrics for {}: {}".format(application, metrics))
1034
1035         if len(metrics):
1036             return True
1037
1038         else:
1039             # TODO: Ran into a case where it took 9 attempts before metrics
1040             # were available; the controller is slow sometimes.
1041             await asyncio.sleep(30)
1042             return await self.check_metrics(application)
1043
1044     @classmethod
1045     async def wait_for_uuids(self, application, uuids):
1046         """Wait for primitives to execute.
1047
1048         The task will provide a list of uuids representing primitives that are
1049         queued to run.
1050         """
1051         debug("Waiting for uuids for {}: {}".format(application, uuids))
1052         waitfor = len(uuids)
1053         finished = 0
1054
1055         while waitfor > finished:
1056             for uid in uuids:
1057                 await asyncio.sleep(10)
1058
1059                 if uuid not in self.state[application]['actions']:
1060                     self.state[application]['actions'][uid] = "pending"
1061
1062                 status = self.state[application]['actions'][uid]
1063
1064                 # Have we already marked this as done?
1065                 if status in ["pending", "running"]:
1066
1067                     debug("Getting status of {} ({})...".format(uid, status))
1068                     status = await self.n2vc.GetPrimitiveStatus(
1069                         self.ns_name,
1070                         uid,
1071                     )
1072                     debug("...state of {} is {}".format(uid, status))
1073                     self.state[application]['actions'][uid] = status
1074
1075                     if status in ['completed', 'failed']:
1076                         finished += 1
1077
1078             debug("{}/{} actions complete".format(finished, waitfor))
1079
1080             # Wait for the primitive to finish and try again
1081             if waitfor > finished:
1082                 debug("Waiting 10s for action to finish...")
1083                 await asyncio.sleep(10)
1084
1085     @classmethod
1086     def n2vc_callback(self, *args, **kwargs):
1087         (model, application, status, message) = args
1088         # debug("callback: {}".format(args))
1089
1090         if application not in self.state:
1091             # Initialize the state of the application
1092             self.state[application] = {
1093                 'status': None,     # Juju status
1094                 'container': None,  # lxd container, for proxy charms
1095                 'actions': {},      # Actions we've executed
1096                 'done': False,      # Are we done testing this charm?
1097                 'phase': "deploy",  # What phase is this application in?
1098             }
1099
1100         self.state[application]['status'] = status
1101
1102         if status in ['waiting', 'maintenance', 'unknown']:
1103             # Nothing to do for these
1104             return
1105
1106         debug("callback: {}".format(args))
1107
1108         if self.state[application]['done']:
1109             debug("{} is done".format(application))
1110             return
1111
1112         if status in ["blocked"] and self.isproxy(application):
1113             if self.state[application]['phase'] == "deploy":
1114                 debug("Configuring proxy charm for {}".format(application))
1115                 asyncio.ensure_future(self.configure_proxy_charm(*args))
1116
1117         elif status in ["active"]:
1118             """When a charm is active, we can assume that it has been properly
1119             configured (not blocked), regardless of if it's a proxy or not.
1120
1121             All primitives should be complete by init_config_primitive
1122             """
1123             asyncio.ensure_future(self.execute_charm_tests(*args))