tests/base.py

   1 #!/usr/bin/env python3
   2 import asyncio
   3 import datetime
   4 import logging
   5 import n2vc.vnf
   6 import pylxd
   7 import pytest
   8 import os
   9 import shlex
  10 import subprocess
  11 import time
  12 import uuid
  13 import yaml
  14
  15 from juju.controller import Controller
  16
  17 # Disable InsecureRequestWarning w/LXD
  18 import urllib3
  19 urllib3.disable_warnings()
  20 logging.getLogger("urllib3").setLevel(logging.WARNING)
  21
  22 here = os.path.dirname(os.path.realpath(__file__))
  23
  24
  25 class CleanController():
  26     """
  27     Context manager that automatically connects and disconnects from
  28     the currently active controller.
  29
  30     Note: Unlike CleanModel, this will not create a new controller for you,
  31     and an active controller must already be available.
  32     """
  33     def __init__(self):
  34         self._controller = None
  35
  36     async def __aenter__(self):
  37         self._controller = Controller()
  38         await self._controller.connect()
  39         return self._controller
  40
  41     async def __aexit__(self, exc_type, exc, tb):
  42         await self._controller.disconnect()
  43
  44
  45 def debug(msg):
  46     """Format debug messages in a consistent way."""
  47     now = datetime.datetime.now()
  48
  49     # TODO: Decide on the best way to log. Output from `logging.debug` shows up
  50     # when a test fails, but print() will always show up when running tox with
  51     # `-s`, which is really useful for debugging single tests without having to
  52     # insert a False assert to see the log.
  53     logging.debug(
  54         "[{}] {}".format(now.strftime('%Y-%m-%dT%H:%M:%S'), msg)
  55     )
  56     print(
  57         "[{}] {}".format(now.strftime('%Y-%m-%dT%H:%M:%S'), msg)
  58     )
  59
  60
  61 def get_charm_path():
  62     return "{}/charms".format(here)
  63
  64
  65 def get_layer_path():
  66     return "{}/charms/layers".format(here)
  67
  68
  69 def collect_metrics(application):
  70     """Invoke Juju's metrics collector.
  71
  72     Caveat: this shells out to the `juju collect-metrics` command, rather than
  73     making an API call. At the time of writing, that API is not exposed through
  74     the client library.
  75     """
  76
  77     try:
  78         subprocess.check_call(['juju', 'collect-metrics', application])
  79     except subprocess.CalledProcessError as e:
  80         raise Exception("Unable to collect metrics: {}".format(e))
  81
  82
  83 def has_metrics(charm):
  84     """Check if a charm has metrics defined."""
  85     metricsyaml = "{}/{}/metrics.yaml".format(
  86         get_layer_path(),
  87         charm,
  88     )
  89     if os.path.exists(metricsyaml):
  90         return True
  91     return False
  92
  93
  94 def get_descriptor(descriptor):
  95     desc = None
  96     try:
  97         tmp = yaml.load(descriptor)
  98
  99         # Remove the envelope
 100         root = list(tmp.keys())[0]
 101         if root == "nsd:nsd-catalog":
 102             desc = tmp['nsd:nsd-catalog']['nsd'][0]
 103         elif root == "vnfd:vnfd-catalog":
 104             desc = tmp['vnfd:vnfd-catalog']['vnfd'][0]
 105     except ValueError:
 106         assert False
 107     return desc
 108
 109
 110 def get_n2vc(loop=None):
 111     """Return an instance of N2VC.VNF."""
 112     log = logging.getLogger()
 113     log.level = logging.DEBUG
 114
 115     # Extract parameters from the environment in order to run our test
 116     vca_host = os.getenv('VCA_HOST', '127.0.0.1')
 117     vca_port = os.getenv('VCA_PORT', 17070)
 118     vca_user = os.getenv('VCA_USER', 'admin')
 119     vca_charms = os.getenv('VCA_CHARMS', None)
 120     vca_secret = os.getenv('VCA_SECRET', None)
 121
 122     # Get the Juju Public key
 123     juju_public_key = get_juju_public_key()
 124     if juju_public_key:
 125         debug("Reading Juju public key @ {}".format(juju_public_key))
 126         with open(juju_public_key, 'r') as f:
 127             juju_public_key = f.read()
 128         debug("Found public key: {}".format(juju_public_key))
 129     else:
 130         raise Exception("No Juju Public Key found")
 131
 132     # Get the ca-cert
 133     # os.path.expanduser("~/.config/lxc")
 134     # with open("{}/agent.conf".format(AGENT_PATH), "r") as f:
 135     #     try:
 136     #         y = yaml.safe_load(f)
 137     #         self.cacert = y['cacert']
 138     #     except yaml.YAMLError as exc:
 139     #         log("Unable to find Juju ca-cert.")
 140     #         raise exc
 141
 142     client = n2vc.vnf.N2VC(
 143         log=log,
 144         server=vca_host,
 145         port=vca_port,
 146         user=vca_user,
 147         secret=vca_secret,
 148         artifacts=vca_charms,
 149         loop=loop,
 150         juju_public_key=juju_public_key,
 151     )
 152     return client
 153
 154
 155 def create_lxd_container(public_key=None, name="test_name"):
 156     """
 157     Returns a container object
 158
 159     If public_key isn't set, we'll use the Juju ssh key
 160
 161     :param public_key: The public key to inject into the container
 162     :param name: The name of the test being run
 163     """
 164     container = None
 165
 166     # Format name so it's valid
 167     name = name.replace("_", "-").replace(".", "")
 168
 169     client = get_lxd_client()
 170     if not client:
 171         raise Exception("Unable to connect to LXD")
 172
 173     test_machine = "test-{}-{}".format(
 174         uuid.uuid4().hex[-4:],
 175         name,
 176     )
 177
 178     private_key_path, public_key_path = find_n2vc_ssh_keys()
 179
 180     try:
 181         # create profile w/cloud-init and juju ssh key
 182         if not public_key:
 183             public_key = ""
 184             with open(public_key_path, "r") as f:
 185                 public_key = f.readline()
 186
 187         client.profiles.create(
 188             test_machine,
 189             config={
 190                 'user.user-data': '#cloud-config\nssh_authorized_keys:\n- {}'.format(public_key)},
 191             devices={
 192                 'root': {'path': '/', 'pool': 'default', 'type': 'disk'},
 193                 'eth0': {
 194                     'nictype': 'bridged',
 195                     'parent': 'lxdbr0',
 196                     'type': 'nic'
 197                 }
 198             }
 199         )
 200     except Exception as ex:
 201         debug("Error creating lxd profile {}: {}".format(test_machine, ex))
 202         raise ex
 203
 204     try:
 205         # create lxc machine
 206         config = {
 207             'name': test_machine,
 208             'source': {
 209                 'type': 'image',
 210                 'alias': 'xenial',
 211                 'mode': 'pull',
 212                 'protocol': 'simplestreams',
 213                 'server': 'https://cloud-images.ubuntu.com/releases',
 214             },
 215             'profiles': [test_machine],
 216         }
 217         container = client.containers.create(config, wait=True)
 218         container.start(wait=True)
 219     except Exception as ex:
 220         debug("Error creating lxd container {}: {}".format(test_machine, ex))
 221         # This is a test-ending failure.
 222         raise ex
 223
 224     def wait_for_network(container, timeout=30):
 225         """Wait for eth0 to have an ipv4 address."""
 226         starttime = time.time()
 227         while(time.time() < starttime + timeout):
 228             time.sleep(1)
 229             if 'eth0' in container.state().network:
 230                 addresses = container.state().network['eth0']['addresses']
 231                 if len(addresses) > 0:
 232                     if addresses[0]['family'] == 'inet':
 233                         return addresses[0]
 234         return None
 235
 236     try:
 237         wait_for_network(container)
 238     except Exception as ex:
 239         debug(
 240             "Error waiting for container {} network: {}".format(
 241                 test_machine,
 242                 ex,
 243             )
 244         )
 245
 246     try:
 247         waitcount = 0
 248         while waitcount <= 5:
 249             if is_sshd_running(container):
 250                 break
 251             waitcount += 1
 252             time.sleep(1)
 253         if waitcount >= 5:
 254             debug("couldn't detect sshd running")
 255             raise Exception("Unable to verify container sshd")
 256
 257     except Exception as ex:
 258         debug(
 259             "Error checking sshd status on {}: {}".format(
 260                 test_machine,
 261                 ex,
 262             )
 263         )
 264
 265     # HACK: We need to give sshd a chance to bind to the interface,
 266     # and pylxd's container.execute seems to be broken and fails and/or
 267     # hangs trying to properly check if the service is up.
 268     (exit_code, stdout, stderr) = container.execute([
 269         'ping',
 270         '-c', '5',   # Wait for 5 ECHO_REPLY
 271         '8.8.8.8',   # Ping Google's public DNS
 272         '-W', '15',  # Set a 15 second deadline
 273     ])
 274     if exit_code > 0:
 275         # The network failed
 276         raise Exception("Unable to verify container network")
 277
 278     return container
 279
 280
 281 def is_sshd_running(container):
 282     """Check if sshd is running in the container.
 283
 284     Check to see if the sshd process is running and listening on port 22.
 285
 286     :param container: The container to check
 287     :return boolean: True if sshd is running.
 288     """
 289     debug("Container: {}".format(container))
 290     try:
 291         (rc, stdout, stderr) = container.execute(
 292             ["service", "ssh", "status"]
 293         )
 294         # If the status is a) found and b) running, the exit code will be 0
 295         if rc == 0:
 296             return True
 297     except Exception as ex:
 298         debug("Failed to check sshd service status: {}".format(ex))
 299
 300     return False
 301
 302
 303 def destroy_lxd_container(container):
 304     """Stop and delete a LXD container.
 305
 306     Sometimes we see errors talking to LXD -- ephemerial issues like
 307     load or a bug that's killed the API. We'll do our best to clean
 308     up here, and we should run a cleanup after all tests are finished
 309     to remove any extra containers and profiles belonging to us.
 310     """
 311
 312     if type(container) is bool:
 313         return
 314
 315     name = container.name
 316     debug("Destroying container {}".format(name))
 317
 318     client = get_lxd_client()
 319
 320     def wait_for_stop(timeout=30):
 321         """Wait for eth0 to have an ipv4 address."""
 322         starttime = time.time()
 323         while(time.time() < starttime + timeout):
 324             time.sleep(1)
 325             if container.state == "Stopped":
 326                 return
 327
 328     def wait_for_delete(timeout=30):
 329         starttime = time.time()
 330         while(time.time() < starttime + timeout):
 331             time.sleep(1)
 332             if client.containers.exists(name) is False:
 333                 return
 334
 335     try:
 336         container.stop(wait=False)
 337         wait_for_stop()
 338     except Exception as ex:
 339         debug(
 340             "Error stopping container {}: {}".format(
 341                 name,
 342                 ex,
 343             )
 344         )
 345
 346     try:
 347         container.delete(wait=False)
 348         wait_for_delete()
 349     except Exception as ex:
 350         debug(
 351             "Error deleting container {}: {}".format(
 352                 name,
 353                 ex,
 354             )
 355         )
 356
 357     try:
 358         # Delete the profile created for this container
 359         profile = client.profiles.get(name)
 360         if profile:
 361             profile.delete()
 362     except Exception as ex:
 363         debug(
 364             "Error deleting profile {}: {}".format(
 365                 name,
 366                 ex,
 367             )
 368         )
 369
 370
 371 def find_lxd_config():
 372     """Find the LXD configuration directory."""
 373     paths = []
 374     paths.append(os.path.expanduser("~/.config/lxc"))
 375     paths.append(os.path.expanduser("~/snap/lxd/current/.config/lxc"))
 376
 377     for path in paths:
 378         if os.path.exists(path):
 379             crt = os.path.expanduser("{}/client.crt".format(path))
 380             key = os.path.expanduser("{}/client.key".format(path))
 381             if os.path.exists(crt) and os.path.exists(key):
 382                 return (crt, key)
 383     return (None, None)
 384
 385
 386 def find_n2vc_ssh_keys():
 387     """Find the N2VC ssh keys."""
 388
 389     paths = []
 390     paths.append(os.path.expanduser("~/.ssh/"))
 391
 392     for path in paths:
 393         if os.path.exists(path):
 394             private = os.path.expanduser("{}/id_n2vc_rsa".format(path))
 395             public = os.path.expanduser("{}/id_n2vc_rsa.pub".format(path))
 396             if os.path.exists(private) and os.path.exists(public):
 397                 return (private, public)
 398     return (None, None)
 399
 400
 401 def find_juju_ssh_keys():
 402     """Find the Juju ssh keys."""
 403
 404     paths = []
 405     paths.append(os.path.expanduser("~/.local/share/juju/ssh"))
 406
 407     for path in paths:
 408         if os.path.exists(path):
 409             private = os.path.expanduser("{}/juju_id_rsa".format(path))
 410             public = os.path.expanduser("{}/juju_id_rsa.pub".format(path))
 411             if os.path.exists(private) and os.path.exists(public):
 412                 return (private, public)
 413     return (None, None)
 414
 415
 416 def get_juju_private_key():
 417     keys = find_juju_ssh_keys()
 418     return keys[0]
 419
 420
 421 def get_juju_public_key():
 422     """Find the Juju public key."""
 423     paths = []
 424
 425     if 'VCA_PATH' in os.environ:
 426         paths.append("{}/ssh".format(os.environ["VCA_PATH"]))
 427
 428     paths.append(os.path.expanduser("~/.local/share/juju/ssh"))
 429     paths.append("/root/.local/share/juju/ssh")
 430
 431     for path in paths:
 432         if os.path.exists(path):
 433             public = os.path.expanduser("{}/juju_id_rsa.pub".format(path))
 434             if os.path.exists(public):
 435                 return public
 436     return None
 437
 438
 439 def get_lxd_client(host=None, port="8443", verify=False):
 440     """ Get the LXD client."""
 441
 442     if host is None:
 443         if 'LXD_HOST' in os.environ:
 444             host = os.environ['LXD_HOST']
 445         else:
 446             host = '127.0.0.1'
 447
 448     passwd = None
 449     if 'LXD_SECRET' in os.environ:
 450         passwd = os.environ['LXD_SECRET']
 451
 452     # debug("Connecting to LXD remote {} w/authentication ({})".format(
 453     #     host,
 454     #     passwd
 455     # ))
 456     client = None
 457     (crt, key) = find_lxd_config()
 458
 459     if crt and key:
 460         client = pylxd.Client(
 461             endpoint="https://{}:{}".format(host, port),
 462             cert=(crt, key),
 463             verify=verify,
 464         )
 465
 466         # If the LXD server has a pasword set, authenticate with it.
 467         if not client.trusted and passwd:
 468             try:
 469                 client.authenticate(passwd)
 470                 if not client.trusted:
 471                     raise Exception("Unable to authenticate with LXD remote")
 472             except pylxd.exceptions.LXDAPIException as ex:
 473                 if 'Certificate already in trust store' in ex:
 474                     pass
 475
 476     return client
 477
 478
 479 # TODO: This is marked serial but can be run in parallel with work, including:
 480 # - Fixing an event loop issue; seems that all tests stop when one test stops?
 481
 482
 483 @pytest.mark.serial
 484 class TestN2VC(object):
 485     """TODO:
 486     1. Validator Validation
 487
 488     Automatically validate the descriptors we're using here, unless the test
 489     author explicitly wants to skip them. Useful to make sure tests aren't
 490     being run against invalid descriptors, validating functionality that may
 491     fail against a properly written descriptor.
 492
 493     We need to have a flag (instance variable) that controls this behavior. It
 494     may be necessary to skip validation and run against a descriptor
 495     implementing features that have not yet been released in the Information
 496     Model.
 497     """
 498
 499     """
 500     The six phases of integration testing, for the test itself and each charm?:
 501
 502     setup/teardown_class:
 503     1. Prepare      - Verify the environment and create a new model
 504     2. Deploy       - Mark the test as ready to execute
 505     3. Configure    - Configuration to reach Active state
 506     4. Test         - Execute primitive(s) to verify success
 507     5. Collect      - Collect any useful artifacts for debugging (charm, logs)
 508     6. Destroy      - Destroy the model
 509
 510
 511     1. Prepare      - Building of charm
 512     2. Deploy       - Deploying charm
 513     3. Configure    - Configuration to reach Active state
 514     4. Test         - Execute primitive(s) to verify success
 515     5. Collect      - Collect any useful artifacts for debugging (charm, logs)
 516     6. Destroy      - Destroy the charm
 517
 518     """
 519     @classmethod
 520     def setup_class(self):
 521         """ setup any state specific to the execution of the given class (which
 522         usually contains tests).
 523         """
 524         # Initialize instance variable(s)
 525         self.n2vc = None
 526
 527         # Track internal state for each test run
 528         self.state = {}
 529
 530         # Parse the test's descriptors
 531         self.nsd = get_descriptor(self.NSD_YAML)
 532         self.vnfd = get_descriptor(self.VNFD_YAML)
 533
 534         self.ns_name = self.nsd['name']
 535         self.vnf_name = self.vnfd['name']
 536
 537         self.charms = {}
 538         self.parse_vnf_descriptor()
 539         assert self.charms is not {}
 540
 541         # Track artifacts, like compiled charms, that will need to be removed
 542         self.artifacts = {}
 543
 544         # Build the charm(s) needed for this test
 545         for charm in self.get_charm_names():
 546             # debug("Building charm {}".format(charm))
 547             self.get_charm(charm)
 548
 549         # A bit of a hack, in order to allow the N2VC callback to run parallel
 550         # to pytest. Test(s) should wait for this flag to change to False
 551         # before returning.
 552         self._running = True
 553         self._stopping = False
 554
 555     @classmethod
 556     def teardown_class(self):
 557         """ teardown any state that was previously setup with a call to
 558         setup_class.
 559         """
 560         debug("Running teardown_class...")
 561         try:
 562
 563             debug("Destroying LXD containers...")
 564             for application in self.state:
 565                 if self.state[application]['container']:
 566                     destroy_lxd_container(self.state[application]['container'])
 567             debug("Destroying LXD containers...done.")
 568
 569             # Logout of N2VC
 570             if self.n2vc:
 571                 debug("teardown_class(): Logging out of N2VC...")
 572                 yield from self.n2vc.logout()
 573                 debug("teardown_class(): Logging out of N2VC...done.")
 574
 575             debug("Running teardown_class...done.")
 576         except Exception as ex:
 577             debug("Exception in teardown_class: {}".format(ex))
 578
 579     @classmethod
 580     def all_charms_active(self):
 581         """Determine if the all deployed charms are active."""
 582         active = 0
 583
 584         for application in self.state:
 585             if 'status' in self.state[application]:
 586                 debug("status of {} is '{}'".format(
 587                     application,
 588                     self.state[application]['status'],
 589                 ))
 590                 if self.state[application]['status'] == 'active':
 591                     active += 1
 592
 593         debug("Active charms: {}/{}".format(
 594             active,
 595             len(self.charms),
 596         ))
 597
 598         if active == len(self.charms):
 599             return True
 600
 601         return False
 602
 603     @classmethod
 604     def are_tests_finished(self):
 605         appcount = len(self.state)
 606
 607         # If we don't have state yet, keep running.
 608         if appcount == 0:
 609             debug("No applications")
 610             return False
 611
 612         if self._stopping:
 613             debug("_stopping is True")
 614             return True
 615
 616         appdone = 0
 617         for application in self.state:
 618             if self.state[application]['done']:
 619                 appdone += 1
 620
 621         debug("{}/{} charms tested".format(appdone, appcount))
 622
 623         if appcount == appdone:
 624             return True
 625
 626         return False
 627
 628     @classmethod
 629     async def running(self, timeout=600):
 630         """Returns if the test is still running.
 631
 632         @param timeout The time, in seconds, to wait for the test to complete.
 633         """
 634         if self.are_tests_finished():
 635             await self.stop()
 636             return False
 637
 638         await asyncio.sleep(30)
 639
 640         return self._running
 641
 642     @classmethod
 643     def get_charm(self, charm):
 644         """Build and return the path to the test charm.
 645
 646         Builds one of the charms in tests/charms/layers and returns the path
 647         to the compiled charm. The charm will automatically be removed when
 648         when the test is complete.
 649
 650         Returns: The path to the built charm or None if `charm build` failed.
 651         """
 652         # Make sure the charm snap is installed
 653         charm_cmd = None
 654         try:
 655             subprocess.check_call(['which', 'charm'])
 656             charm_cmd = "charm build"
 657         except subprocess.CalledProcessError:
 658             # charm_cmd = "charm-build"
 659             # debug("Using legacy charm-build")
 660             raise Exception("charm snap not installed.")
 661
 662         if charm not in self.artifacts:
 663             try:
 664                 # Note: This builds the charm under N2VC/tests/charms/builds/
 665                 # Currently, the snap-installed command only has write access
 666                 # to the $HOME (changing in an upcoming release) so writing to
 667                 # /tmp isn't possible at the moment.
 668
 669                 builds = get_charm_path()
 670                 if not os.path.exists("{}/builds/{}".format(builds, charm)):
 671                     cmd = "{} --no-local-layers {}/{} -o {}/".format(
 672                         charm_cmd,
 673                         get_layer_path(),
 674                         charm,
 675                         builds,
 676                     )
 677                     # debug(cmd)
 678
 679                     env = os.environ.copy()
 680                     env["CHARM_BUILD_DIR"] = builds
 681
 682                     subprocess.check_call(shlex.split(cmd), env=env)
 683
 684             except subprocess.CalledProcessError as e:
 685                 # charm build will return error code 100 if the charm fails
 686                 # the auto-run of charm proof, which we can safely ignore for
 687                 # our CI charms.
 688                 if e.returncode != 100:
 689                     raise Exception("charm build failed: {}.".format(e))
 690
 691             self.artifacts[charm] = {
 692                 'tmpdir': builds,
 693                 'charm': "{}/builds/{}".format(builds, charm),
 694             }
 695
 696         return self.artifacts[charm]['charm']
 697
 698     @classmethod
 699     async def deploy(self, vnf_index, charm, params, loop):
 700         """An inner function to do the deployment of a charm from
 701         either a vdu or vnf.
 702         """
 703
 704         if not self.n2vc:
 705             self.n2vc = get_n2vc(loop=loop)
 706
 707         debug("Creating model for Network Service {}".format(self.ns_name))
 708         await self.n2vc.CreateNetworkService(self.ns_name)
 709
 710         application = self.n2vc.FormatApplicationName(
 711             self.ns_name,
 712             self.vnf_name,
 713             str(vnf_index),
 714         )
 715
 716         # Initialize the state of the application
 717         self.state[application] = {
 718             'status': None,     # Juju status
 719             'container': None,  # lxd container, for proxy charms
 720             'actions': {},      # Actions we've executed
 721             'done': False,      # Are we done testing this charm?
 722             'phase': "deploy",  # What phase is this application in?
 723         }
 724
 725         debug("Deploying charm at {}".format(self.artifacts[charm]))
 726
 727         # If this is a native charm, we need to provision the underlying
 728         # machine ala an LXC container.
 729         machine_spec = {}
 730
 731         if not self.isproxy(application):
 732             debug("Creating container for native charm")
 733             # args = ("default", application, None, None)
 734             self.state[application]['container'] = create_lxd_container(
 735                 name=os.path.basename(__file__)
 736             )
 737
 738             hostname = self.get_container_ip(
 739                 self.state[application]['container'],
 740             )
 741
 742             machine_spec = {
 743                 'hostname': hostname,
 744                 'username': 'ubuntu',
 745             }
 746
 747         await self.n2vc.DeployCharms(
 748             self.ns_name,
 749             application,
 750             self.vnfd,
 751             self.get_charm(charm),
 752             params,
 753             machine_spec,
 754             self.n2vc_callback,
 755         )
 756
 757     @classmethod
 758     def parse_vnf_descriptor(self):
 759         """Parse the VNF descriptor to make running tests easier.
 760
 761         Parse the charm information in the descriptor to make it easy to write
 762         tests to run again it.
 763
 764         Each charm becomes a dictionary in a list:
 765         [
 766             'is-proxy': True,
 767             'vnf-member-index': 1,
 768             'vnf-name': '',
 769             'charm-name': '',
 770             'initial-config-primitive': {},
 771             'config-primitive': {}
 772         ]
 773         - charm name
 774         - is this a proxy charm?
 775         - what are the initial-config-primitives (day 1)?
 776         - what are the config primitives (day 2)?
 777
 778         """
 779         charms = {}
 780
 781         # You'd think this would be explicit, but it's just an incremental
 782         # value that should be consistent.
 783         vnf_member_index = 0
 784
 785         """Get all vdu and/or vdu config in a descriptor."""
 786         config = self.get_config()
 787         for cfg in config:
 788             if 'juju' in cfg:
 789
 790                 # Get the name to be used for the deployed application
 791                 application_name = n2vc.vnf.N2VC().FormatApplicationName(
 792                     self.ns_name,
 793                     self.vnf_name,
 794                     str(vnf_member_index),
 795                 )
 796
 797                 charm = {
 798                     'application-name': application_name,
 799                     'proxy': True,
 800                     'vnf-member-index': vnf_member_index,
 801                     'vnf-name': self.vnf_name,
 802                     'name': None,
 803                     'initial-config-primitive': {},
 804                     'config-primitive': {},
 805                 }
 806
 807                 juju = cfg['juju']
 808                 charm['name'] = juju['charm']
 809
 810                 if 'proxy' in juju:
 811                     charm['proxy'] = juju['proxy']
 812
 813                 if 'initial-config-primitive' in cfg:
 814                     charm['initial-config-primitive'] = \
 815                         cfg['initial-config-primitive']
 816
 817                 if 'config-primitive' in cfg:
 818                     charm['config-primitive'] = cfg['config-primitive']
 819
 820                 charms[application_name] = charm
 821
 822             # Increment the vnf-member-index
 823             vnf_member_index += 1
 824
 825         self.charms = charms
 826
 827     @classmethod
 828     def isproxy(self, application_name):
 829
 830         assert application_name in self.charms
 831         assert 'proxy' in self.charms[application_name]
 832         assert type(self.charms[application_name]['proxy']) is bool
 833
 834         # debug(self.charms[application_name])
 835         return self.charms[application_name]['proxy']
 836
 837     @classmethod
 838     def get_config(self):
 839         """Return an iterable list of config items (vdu and vnf).
 840
 841         As far as N2VC is concerned, the config section for vdu and vnf are
 842         identical. This joins them together so tests only need to iterate
 843         through one list.
 844         """
 845         configs = []
 846
 847         """Get all vdu and/or vdu config in a descriptor."""
 848         vnf_config = self.vnfd.get("vnf-configuration")
 849         if vnf_config:
 850             juju = vnf_config['juju']
 851             if juju:
 852                 configs.append(vnf_config)
 853
 854         for vdu in self.vnfd['vdu']:
 855             vdu_config = vdu.get('vdu-configuration')
 856             if vdu_config:
 857                 juju = vdu_config['juju']
 858                 if juju:
 859                     configs.append(vdu_config)
 860
 861         return configs
 862
 863     @classmethod
 864     def get_charm_names(self):
 865         """Return a list of charms used by the test descriptor."""
 866
 867         charms = {}
 868
 869         # Check if the VDUs in this VNF have a charm
 870         for config in self.get_config():
 871             juju = config['juju']
 872
 873             name = juju['charm']
 874             if name not in charms:
 875                 charms[name] = 1
 876
 877         return charms.keys()
 878
 879     @classmethod
 880     def get_phase(self, application):
 881         return self.state[application]['phase']
 882
 883     @classmethod
 884     def set_phase(self, application, phase):
 885         self.state[application]['phase'] = phase
 886
 887     @classmethod
 888     async def configure_proxy_charm(self, *args):
 889         """Configure a container for use via ssh."""
 890         (model, application, _, _) = args
 891
 892         try:
 893             if self.get_phase(application) == "deploy":
 894                 self.set_phase(application, "configure")
 895
 896                 debug("Start CreateContainer for {}".format(application))
 897                 self.state[application]['container'] = \
 898                     await self.CreateContainer(*args)
 899                 debug("Done CreateContainer for {}".format(application))
 900
 901                 if self.state[application]['container']:
 902                     debug("Configure {} for container".format(application))
 903                     if await self.configure_ssh_proxy(application):
 904                         await asyncio.sleep(0.1)
 905                         return True
 906                 else:
 907                     debug("Failed to configure container for {}".format(application))
 908             else:
 909                 debug("skipping CreateContainer for {}: {}".format(
 910                     application,
 911                     self.get_phase(application),
 912                 ))
 913
 914         except Exception as ex:
 915             debug("configure_proxy_charm exception: {}".format(ex))
 916         finally:
 917             await asyncio.sleep(0.1)
 918
 919         return False
 920
 921     @classmethod
 922     async def execute_charm_tests(self, *args):
 923         (model, application, _, _) = args
 924
 925         debug("Executing charm test(s) for {}".format(application))
 926
 927         if self.state[application]['done']:
 928             debug("Trying to execute tests against finished charm...aborting")
 929             return False
 930
 931         try:
 932             phase = self.get_phase(application)
 933             # We enter the test phase when after deploy (for native charms) or
 934             # configure, for proxy charms.
 935             if phase in ["deploy", "configure"]:
 936                 self.set_phase(application, "test")
 937                 if self.are_tests_finished():
 938                     raise Exception("Trying to execute init-config on finished test")
 939
 940                 if await self.execute_initial_config_primitives(application):
 941                     # check for metrics
 942                     await self.check_metrics(application)
 943
 944                     debug("Done testing {}".format(application))
 945                     self.state[application]['done'] = True
 946
 947         except Exception as ex:
 948             debug("Exception in execute_charm_tests: {}".format(ex))
 949         finally:
 950             await asyncio.sleep(0.1)
 951
 952         return True
 953
 954     @classmethod
 955     async def CreateContainer(self, *args):
 956         """Create a LXD container for use with a proxy charm.abs
 957
 958         1. Get the public key from the charm via `get-ssh-public-key` action
 959         2. Create container with said key injected for the ubuntu user
 960
 961         Returns a Container object
 962         """
 963         # Create and configure a LXD container for use with a proxy charm.
 964         (model, application, _, _) = args
 965
 966         debug("[CreateContainer] {}".format(args))
 967         container = None
 968
 969         try:
 970             # Execute 'get-ssh-public-key' primitive and get returned value
 971             uuid = await self.n2vc.ExecutePrimitive(
 972                 model,
 973                 application,
 974                 "get-ssh-public-key",
 975                 None,
 976             )
 977
 978             result = await self.n2vc.GetPrimitiveOutput(model, uuid)
 979             pubkey = result['pubkey']
 980
 981             container = create_lxd_container(
 982                 public_key=pubkey,
 983                 name=os.path.basename(__file__)
 984             )
 985
 986             return container
 987         except Exception as ex:
 988             debug("Error creating container: {}".format(ex))
 989             pass
 990
 991         return None
 992
 993     @classmethod
 994     async def stop(self):
 995         """Stop the test.
 996
 997         - Remove charms
 998         - Stop and delete containers
 999         - Logout of N2VC
1000
1001         TODO: Clean up duplicate code between teardown_class() and stop()
1002         """
1003         debug("stop() called")
1004
1005         if self.n2vc and self._running and not self._stopping:
1006             self._running = False
1007             self._stopping = True
1008
1009             # Destroy the network service
1010             try:
1011                 await self.n2vc.DestroyNetworkService(self.ns_name)
1012             except Exception as e:
1013                 debug(
1014                     "Error Destroying Network Service \"{}\": {}".format(
1015                         self.ns_name,
1016                         e,
1017                     )
1018                 )
1019
1020             # Wait for the applications to be removed and delete the containers
1021             for application in self.charms:
1022                 try:
1023
1024                     while True:
1025                         # Wait for the application to be removed
1026                         await asyncio.sleep(10)
1027                         if not await self.n2vc.HasApplication(
1028                             self.ns_name,
1029                             application,
1030                         ):
1031                             break
1032
1033                     # Need to wait for the charm to finish, because native charms
1034                     if self.state[application]['container']:
1035                         debug("Deleting LXD container...")
1036                         destroy_lxd_container(
1037                             self.state[application]['container']
1038                         )
1039                         self.state[application]['container'] = None
1040                         debug("Deleting LXD container...done.")
1041                     else:
1042                         debug("No container found for {}".format(application))
1043                 except Exception as e:
1044                     debug("Error while deleting container: {}".format(e))
1045
1046             # Logout of N2VC
1047             try:
1048                 debug("stop(): Logging out of N2VC...")
1049                 await self.n2vc.logout()
1050                 self.n2vc = None
1051                 debug("stop(): Logging out of N2VC...Done.")
1052             except Exception as ex:
1053                 debug(ex)
1054
1055             # Let the test know we're finished.
1056             debug("Marking test as finished.")
1057             # self._running = False
1058         else:
1059             debug("Skipping stop()")
1060
1061     @classmethod
1062     def get_container_ip(self, container):
1063         """Return the IPv4 address of container's eth0 interface."""
1064         ipaddr = None
1065         if container:
1066             addresses = container.state().network['eth0']['addresses']
1067             # The interface may have more than one address, but we only need
1068             # the first one for testing purposes.
1069             ipaddr = addresses[0]['address']
1070
1071         return ipaddr
1072
1073     @classmethod
1074     async def configure_ssh_proxy(self, application, task=None):
1075         """Configure the proxy charm to use the lxd container.
1076
1077         Configure the charm to use a LXD container as it's VNF.
1078         """
1079         debug("Configuring ssh proxy for {}".format(application))
1080
1081         mgmtaddr = self.get_container_ip(
1082             self.state[application]['container'],
1083         )
1084
1085         debug(
1086             "Setting ssh-hostname for {} to {}".format(
1087                 application,
1088                 mgmtaddr,
1089             )
1090         )
1091
1092         await self.n2vc.ExecutePrimitive(
1093             self.ns_name,
1094             application,
1095             "config",
1096             None,
1097             params={
1098                 'ssh-hostname': mgmtaddr,
1099                 'ssh-username': 'ubuntu',
1100             }
1101         )
1102
1103         return True
1104
1105     @classmethod
1106     async def execute_initial_config_primitives(self, application, task=None):
1107         debug("Executing initial_config_primitives for {}".format(application))
1108         try:
1109             init_config = self.charms[application]
1110
1111             """
1112             The initial-config-primitive is run during deploy but may fail
1113              on some steps because proxy charm access isn't configured.
1114
1115             Re-run those actions so we can inspect the status.
1116             """
1117             uuids = await self.n2vc.ExecuteInitialPrimitives(
1118                 self.ns_name,
1119                 application,
1120                 init_config,
1121             )
1122
1123             """
1124             ExecutePrimitives will return a list of uuids. We need to check the
1125              status of each. The test continues if all Actions succeed, and
1126              fails if any of them fail.
1127             """
1128             await self.wait_for_uuids(application, uuids)
1129             debug("Primitives for {} finished.".format(application))
1130
1131             return True
1132         except Exception as ex:
1133             debug("execute_initial_config_primitives exception: {}".format(ex))
1134             raise ex
1135
1136         return False
1137
1138     @classmethod
1139     async def check_metrics(self, application, task=None):
1140         """Check and run metrics, if present.
1141
1142         Checks to see if metrics are specified by the charm. If so, collects
1143         the metrics.
1144
1145         If no metrics, then mark the test as finished.
1146         """
1147         if has_metrics(self.charms[application]['name']):
1148             debug("Collecting metrics for {}".format(application))
1149
1150             metrics = await self.n2vc.GetMetrics(
1151                 self.ns_name,
1152                 application,
1153             )
1154
1155             return await self.verify_metrics(application, metrics)
1156
1157     @classmethod
1158     async def verify_metrics(self, application, metrics):
1159         """Verify the charm's metrics.
1160
1161         Verify that the charm has sent metrics successfully.
1162
1163         Stops the test when finished.
1164         """
1165         debug("Verifying metrics for {}: {}".format(application, metrics))
1166
1167         if len(metrics):
1168             return True
1169
1170         else:
1171             # TODO: Ran into a case where it took 9 attempts before metrics
1172             # were available; the controller is slow sometimes.
1173             await asyncio.sleep(30)
1174             return await self.check_metrics(application)
1175
1176     @classmethod
1177     async def wait_for_uuids(self, application, uuids):
1178         """Wait for primitives to execute.
1179
1180         The task will provide a list of uuids representing primitives that are
1181         queued to run.
1182         """
1183         debug("Waiting for uuids for {}: {}".format(application, uuids))
1184         waitfor = len(uuids)
1185         finished = 0
1186
1187         while waitfor > finished:
1188             for uid in uuids:
1189                 await asyncio.sleep(10)
1190
1191                 if uuid not in self.state[application]['actions']:
1192                     self.state[application]['actions'][uid] = "pending"
1193
1194                 status = self.state[application]['actions'][uid]
1195
1196                 # Have we already marked this as done?
1197                 if status in ["pending", "running"]:
1198
1199                     debug("Getting status of {} ({})...".format(uid, status))
1200                     status = await self.n2vc.GetPrimitiveStatus(
1201                         self.ns_name,
1202                         uid,
1203                     )
1204                     debug("...state of {} is {}".format(uid, status))
1205                     self.state[application]['actions'][uid] = status
1206
1207                     if status in ['completed', 'failed']:
1208                         finished += 1
1209
1210             debug("{}/{} actions complete".format(finished, waitfor))
1211
1212             # Wait for the primitive to finish and try again
1213             if waitfor > finished:
1214                 debug("Waiting 10s for action to finish...")
1215                 await asyncio.sleep(10)
1216
1217     @classmethod
1218     def n2vc_callback(self, *args, **kwargs):
1219         (model, application, status, message) = args
1220         # debug("callback: {}".format(args))
1221
1222         if application not in self.state:
1223             # Initialize the state of the application
1224             self.state[application] = {
1225                 'status': None,     # Juju status
1226                 'container': None,  # lxd container, for proxy charms
1227                 'actions': {},      # Actions we've executed
1228                 'done': False,      # Are we done testing this charm?
1229                 'phase': "deploy",  # What phase is this application in?
1230             }
1231
1232         self.state[application]['status'] = status
1233
1234         if status in ['waiting', 'maintenance', 'unknown']:
1235             # Nothing to do for these
1236             return
1237
1238         debug("callback: {}".format(args))
1239
1240         if self.state[application]['done']:
1241             debug("{} is done".format(application))
1242             return
1243
1244         if status in ['error']:
1245             # To test broken charms, if a charm enters an error state we should
1246             # end the test
1247             debug("{} is in an error state, stop the test.".format(application))
1248             # asyncio.ensure_future(self.stop())
1249             self.state[application]['done'] = True
1250             assert False
1251
1252         if status in ["blocked"] and self.isproxy(application):
1253             if self.state[application]['phase'] == "deploy":
1254                 debug("Configuring proxy charm for {}".format(application))
1255                 asyncio.ensure_future(self.configure_proxy_charm(*args))
1256
1257         elif status in ["active"]:
1258             """When a charm is active, we can assume that it has been properly
1259             configured (not blocked), regardless of if it's a proxy or not.
1260
1261             All primitives should be complete by init_config_primitive
1262             """
1263             asyncio.ensure_future(self.execute_charm_tests(*args))