2 # Copyright 2019 Canonical Ltd.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
29 from juju
.controller
import Controller
31 # Disable InsecureRequestWarning w/LXD
33 urllib3
.disable_warnings()
34 logging
.getLogger("urllib3").setLevel(logging
.WARNING
)
36 here
= os
.path
.dirname(os
.path
.realpath(__file__
))
39 class CleanController():
41 Context manager that automatically connects and disconnects from
42 the currently active controller.
44 Note: Unlike CleanModel, this will not create a new controller for you,
45 and an active controller must already be available.
48 self
._controller
= None
50 async def __aenter__(self
):
51 self
._controller
= Controller()
52 await self
._controller
.connect()
53 return self
._controller
55 async def __aexit__(self
, exc_type
, exc
, tb
):
56 await self
._controller
.disconnect()
60 """Format debug messages in a consistent way."""
61 now
= datetime
.datetime
.now()
63 # TODO: Decide on the best way to log. Output from `logging.debug` shows up
64 # when a test fails, but print() will always show up when running tox with
65 # `-s`, which is really useful for debugging single tests without having to
66 # insert a False assert to see the log.
68 "[{}] {}".format(now
.strftime('%Y-%m-%dT%H:%M:%S'), msg
)
71 "[{}] {}".format(now
.strftime('%Y-%m-%dT%H:%M:%S'), msg
)
76 return "{}/charms".format(here
)
80 return "{}/charms/layers".format(here
)
83 def collect_metrics(application
):
84 """Invoke Juju's metrics collector.
86 Caveat: this shells out to the `juju collect-metrics` command, rather than
87 making an API call. At the time of writing, that API is not exposed through
92 subprocess
.check_call(['juju', 'collect-metrics', application
])
93 except subprocess
.CalledProcessError
as e
:
94 raise Exception("Unable to collect metrics: {}".format(e
))
97 def has_metrics(charm
):
98 """Check if a charm has metrics defined."""
99 metricsyaml
= "{}/{}/metrics.yaml".format(
103 if os
.path
.exists(metricsyaml
):
108 def get_descriptor(descriptor
):
111 tmp
= yaml
.safe_load(descriptor
)
113 # Remove the envelope
114 root
= list(tmp
.keys())[0]
115 if root
== "nsd:nsd-catalog":
116 desc
= tmp
['nsd:nsd-catalog']['nsd'][0]
117 elif root
== "vnfd:vnfd-catalog":
118 desc
= tmp
['vnfd:vnfd-catalog']['vnfd'][0]
124 def get_n2vc(loop
=None):
125 """Return an instance of N2VC.VNF."""
126 log
= logging
.getLogger()
127 log
.level
= logging
.DEBUG
129 # Extract parameters from the environment in order to run our test
130 vca_host
= os
.getenv('VCA_HOST', '127.0.0.1')
131 vca_port
= os
.getenv('VCA_PORT', 17070)
132 vca_user
= os
.getenv('VCA_USER', 'admin')
133 vca_charms
= os
.getenv('VCA_CHARMS', None)
134 vca_secret
= os
.getenv('VCA_SECRET', None)
135 vca_cacert
= os
.getenv('VCA_CACERT', None)
137 # Get the Juju Public key
138 juju_public_key
= get_juju_public_key()
140 debug("Reading Juju public key @ {}".format(juju_public_key
))
141 with
open(juju_public_key
, 'r') as f
:
142 juju_public_key
= f
.read()
143 debug("Found public key: {}".format(juju_public_key
))
145 raise Exception("No Juju Public Key found")
148 # os.path.expanduser("~/.config/lxc")
149 # with open("{}/agent.conf".format(AGENT_PATH), "r") as f:
151 # y = yaml.safe_load(f)
152 # self.cacert = y['cacert']
153 # except yaml.YAMLError as exc:
154 # log("Unable to find Juju ca-cert.")
157 client
= n2vc
.vnf
.N2VC(
163 artifacts
=vca_charms
,
165 juju_public_key
=juju_public_key
,
171 def create_lxd_container(public_key
=None, name
="test_name"):
173 Returns a container object
175 If public_key isn't set, we'll use the Juju ssh key
177 :param public_key: The public key to inject into the container
178 :param name: The name of the test being run
182 # Format name so it's valid
183 name
= name
.replace("_", "-").replace(".", "")
185 client
= get_lxd_client()
187 raise Exception("Unable to connect to LXD")
189 test_machine
= "test-{}-{}".format(
190 uuid
.uuid4().hex[-4:],
194 private_key_path
, public_key_path
= find_n2vc_ssh_keys()
197 # create profile w/cloud-init and juju ssh key
200 with
open(public_key_path
, "r") as f
:
201 public_key
= f
.readline()
203 client
.profiles
.create(
206 'user.user-data': '#cloud-config\nssh_authorized_keys:\n- {}'.format(public_key
)},
208 'root': {'path': '/', 'pool': 'default', 'type': 'disk'},
210 'nictype': 'bridged',
216 except Exception as ex
:
217 debug("Error creating lxd profile {}: {}".format(test_machine
, ex
))
223 'name': test_machine
,
228 'protocol': 'simplestreams',
229 'server': 'https://cloud-images.ubuntu.com/releases',
231 'profiles': [test_machine
],
233 container
= client
.containers
.create(config
, wait
=True)
234 container
.start(wait
=True)
235 except Exception as ex
:
236 debug("Error creating lxd container {}: {}".format(test_machine
, ex
))
237 # This is a test-ending failure.
240 def wait_for_network(container
, timeout
=30):
241 """Wait for eth0 to have an ipv4 address."""
242 starttime
= time
.time()
243 while(time
.time() < starttime
+ timeout
):
245 if 'eth0' in container
.state().network
:
246 addresses
= container
.state().network
['eth0']['addresses']
247 if len(addresses
) > 0:
248 if addresses
[0]['family'] == 'inet':
253 wait_for_network(container
)
254 except Exception as ex
:
256 "Error waiting for container {} network: {}".format(
264 while waitcount
<= 5:
265 if is_sshd_running(container
):
270 debug("couldn't detect sshd running")
271 raise Exception("Unable to verify container sshd")
273 except Exception as ex
:
275 "Error checking sshd status on {}: {}".format(
281 # HACK: We need to give sshd a chance to bind to the interface,
282 # and pylxd's container.execute seems to be broken and fails and/or
283 # hangs trying to properly check if the service is up.
284 (exit_code
, stdout
, stderr
) = container
.execute([
286 '-c', '5', # Wait for 5 ECHO_REPLY
287 '8.8.8.8', # Ping Google's public DNS
288 '-W', '15', # Set a 15 second deadline
292 raise Exception("Unable to verify container network")
297 def is_sshd_running(container
):
298 """Check if sshd is running in the container.
300 Check to see if the sshd process is running and listening on port 22.
302 :param container: The container to check
303 :return boolean: True if sshd is running.
305 debug("Container: {}".format(container
))
307 (rc
, stdout
, stderr
) = container
.execute(
308 ["service", "ssh", "status"]
310 # If the status is a) found and b) running, the exit code will be 0
313 except Exception as ex
:
314 debug("Failed to check sshd service status: {}".format(ex
))
319 def destroy_lxd_container(container
):
320 """Stop and delete a LXD container.
322 Sometimes we see errors talking to LXD -- ephemerial issues like
323 load or a bug that's killed the API. We'll do our best to clean
324 up here, and we should run a cleanup after all tests are finished
325 to remove any extra containers and profiles belonging to us.
328 if type(container
) is bool:
331 name
= container
.name
332 debug("Destroying container {}".format(name
))
334 client
= get_lxd_client()
336 def wait_for_stop(timeout
=30):
337 """Wait for eth0 to have an ipv4 address."""
338 starttime
= time
.time()
339 while(time
.time() < starttime
+ timeout
):
341 if container
.state
== "Stopped":
344 def wait_for_delete(timeout
=30):
345 starttime
= time
.time()
346 while(time
.time() < starttime
+ timeout
):
348 if client
.containers
.exists(name
) is False:
352 container
.stop(wait
=False)
354 except Exception as ex
:
356 "Error stopping container {}: {}".format(
363 container
.delete(wait
=False)
365 except Exception as ex
:
367 "Error deleting container {}: {}".format(
374 # Delete the profile created for this container
375 profile
= client
.profiles
.get(name
)
378 except Exception as ex
:
380 "Error deleting profile {}: {}".format(
387 def find_lxd_config():
388 """Find the LXD configuration directory."""
390 paths
.append(os
.path
.expanduser("~/.config/lxc"))
391 paths
.append(os
.path
.expanduser("~/snap/lxd/current/.config/lxc"))
394 if os
.path
.exists(path
):
395 crt
= os
.path
.expanduser("{}/client.crt".format(path
))
396 key
= os
.path
.expanduser("{}/client.key".format(path
))
397 if os
.path
.exists(crt
) and os
.path
.exists(key
):
402 def find_n2vc_ssh_keys():
403 """Find the N2VC ssh keys."""
406 paths
.append(os
.path
.expanduser("~/.ssh/"))
409 if os
.path
.exists(path
):
410 private
= os
.path
.expanduser("{}/id_n2vc_rsa".format(path
))
411 public
= os
.path
.expanduser("{}/id_n2vc_rsa.pub".format(path
))
412 if os
.path
.exists(private
) and os
.path
.exists(public
):
413 return (private
, public
)
417 def find_juju_ssh_keys():
418 """Find the Juju ssh keys."""
421 paths
.append(os
.path
.expanduser("~/.local/share/juju/ssh"))
424 if os
.path
.exists(path
):
425 private
= os
.path
.expanduser("{}/juju_id_rsa".format(path
))
426 public
= os
.path
.expanduser("{}/juju_id_rsa.pub".format(path
))
427 if os
.path
.exists(private
) and os
.path
.exists(public
):
428 return (private
, public
)
432 def get_juju_private_key():
433 keys
= find_juju_ssh_keys()
437 def get_juju_public_key():
438 """Find the Juju public key."""
441 if 'VCA_PATH' in os
.environ
:
442 paths
.append("{}/ssh".format(os
.environ
["VCA_PATH"]))
444 paths
.append(os
.path
.expanduser("~/.local/share/juju/ssh"))
445 paths
.append("/root/.local/share/juju/ssh")
448 if os
.path
.exists(path
):
449 public
= os
.path
.expanduser("{}/juju_id_rsa.pub".format(path
))
450 if os
.path
.exists(public
):
455 def get_lxd_client(host
=None, port
="8443", verify
=False):
456 """ Get the LXD client."""
459 if 'LXD_HOST' in os
.environ
:
460 host
= os
.environ
['LXD_HOST']
465 if 'LXD_SECRET' in os
.environ
:
466 passwd
= os
.environ
['LXD_SECRET']
468 # debug("Connecting to LXD remote {} w/authentication ({})".format(
473 (crt
, key
) = find_lxd_config()
476 client
= pylxd
.Client(
477 endpoint
="https://{}:{}".format(host
, port
),
482 # If the LXD server has a pasword set, authenticate with it.
483 if not client
.trusted
and passwd
:
485 client
.authenticate(passwd
)
486 if not client
.trusted
:
487 raise Exception("Unable to authenticate with LXD remote")
488 except pylxd
.exceptions
.LXDAPIException
as ex
:
489 if 'Certificate already in trust store' in ex
:
495 # TODO: This is marked serial but can be run in parallel with work, including:
496 # - Fixing an event loop issue; seems that all tests stop when one test stops?
500 class TestN2VC(object):
502 1. Validator Validation
504 Automatically validate the descriptors we're using here, unless the test
505 author explicitly wants to skip them. Useful to make sure tests aren't
506 being run against invalid descriptors, validating functionality that may
507 fail against a properly written descriptor.
509 We need to have a flag (instance variable) that controls this behavior. It
510 may be necessary to skip validation and run against a descriptor
511 implementing features that have not yet been released in the Information
516 The six phases of integration testing, for the test itself and each charm?:
518 setup/teardown_class:
519 1. Prepare - Verify the environment and create a new model
520 2. Deploy - Mark the test as ready to execute
521 3. Configure - Configuration to reach Active state
522 4. Test - Execute primitive(s) to verify success
523 5. Collect - Collect any useful artifacts for debugging (charm, logs)
524 6. Destroy - Destroy the model
527 1. Prepare - Building of charm
528 2. Deploy - Deploying charm
529 3. Configure - Configuration to reach Active state
530 4. Test - Execute primitive(s) to verify success
531 5. Collect - Collect any useful artifacts for debugging (charm, logs)
532 6. Destroy - Destroy the charm
536 def setup_class(self
):
537 """ setup any state specific to the execution of the given class (which
538 usually contains tests).
540 # Initialize instance variable(s)
543 # Track internal state for each test run
546 # Parse the test's descriptors
547 self
.nsd
= get_descriptor(self
.NSD_YAML
)
548 self
.vnfd
= get_descriptor(self
.VNFD_YAML
)
550 self
.ns_name
= self
.nsd
['name']
551 self
.vnf_name
= self
.vnfd
['name']
554 self
.parse_vnf_descriptor()
555 assert self
.charms
is not {}
557 # Track artifacts, like compiled charms, that will need to be removed
560 # Build the charm(s) needed for this test
561 for charm
in self
.get_charm_names():
562 # debug("Building charm {}".format(charm))
563 self
.get_charm(charm
)
565 # A bit of a hack, in order to allow the N2VC callback to run parallel
566 # to pytest. Test(s) should wait for this flag to change to False
569 self
._stopping
= False
572 def teardown_class(self
):
573 """ teardown any state that was previously setup with a call to
576 debug("Running teardown_class...")
579 debug("Destroying LXD containers...")
580 for application
in self
.state
:
581 if self
.state
[application
]['container']:
582 destroy_lxd_container(self
.state
[application
]['container'])
583 debug("Destroying LXD containers...done.")
587 debug("teardown_class(): Logging out of N2VC...")
588 yield from self
.n2vc
.logout()
589 debug("teardown_class(): Logging out of N2VC...done.")
591 debug("Running teardown_class...done.")
592 except Exception as ex
:
593 debug("Exception in teardown_class: {}".format(ex
))
596 def all_charms_active(self
):
597 """Determine if the all deployed charms are active."""
600 for application
in self
.state
:
601 if 'status' in self
.state
[application
]:
602 debug("status of {} is '{}'".format(
604 self
.state
[application
]['status'],
606 if self
.state
[application
]['status'] == 'active':
609 debug("Active charms: {}/{}".format(
614 if active
== len(self
.charms
):
620 def are_tests_finished(self
):
621 appcount
= len(self
.state
)
623 # If we don't have state yet, keep running.
625 debug("No applications")
629 debug("_stopping is True")
633 for application
in self
.state
:
634 if self
.state
[application
]['done']:
637 debug("{}/{} charms tested".format(appdone
, appcount
))
639 if appcount
== appdone
:
645 async def running(self
, timeout
=600):
646 """Returns if the test is still running.
648 @param timeout The time, in seconds, to wait for the test to complete.
650 if self
.are_tests_finished():
654 await asyncio
.sleep(30)
659 def get_charm(self
, charm
):
660 """Build and return the path to the test charm.
662 Builds one of the charms in tests/charms/layers and returns the path
663 to the compiled charm. The charm will automatically be removed when
664 when the test is complete.
666 Returns: The path to the built charm or None if `charm build` failed.
668 # Make sure the charm snap is installed
671 subprocess
.check_call(['which', 'charm'])
672 charm_cmd
= "charm build"
673 except subprocess
.CalledProcessError
:
674 # charm_cmd = "charm-build"
675 # debug("Using legacy charm-build")
676 raise Exception("charm snap not installed.")
678 if charm
not in self
.artifacts
:
680 # Note: This builds the charm under N2VC/tests/charms/builds/
681 # Currently, the snap-installed command only has write access
682 # to the $HOME (changing in an upcoming release) so writing to
683 # /tmp isn't possible at the moment.
685 builds
= get_charm_path()
686 if not os
.path
.exists("{}/builds/{}".format(builds
, charm
)):
687 cmd
= "{} --no-local-layers {}/{} -o {}/".format(
695 env
= os
.environ
.copy()
696 env
["CHARM_BUILD_DIR"] = builds
698 subprocess
.check_call(shlex
.split(cmd
), env
=env
)
700 except subprocess
.CalledProcessError
as e
:
701 # charm build will return error code 100 if the charm fails
702 # the auto-run of charm proof, which we can safely ignore for
704 if e
.returncode
!= 100:
705 raise Exception("charm build failed: {}.".format(e
))
707 self
.artifacts
[charm
] = {
709 'charm': "{}/builds/{}".format(builds
, charm
),
712 return self
.artifacts
[charm
]['charm']
715 async def deploy(self
, vnf_index
, charm
, params
, loop
):
716 """An inner function to do the deployment of a charm from
721 self
.n2vc
= get_n2vc(loop
=loop
)
723 debug("Creating model for Network Service {}".format(self
.ns_name
))
724 await self
.n2vc
.CreateNetworkService(self
.ns_name
)
726 application
= self
.n2vc
.FormatApplicationName(
732 # Initialize the state of the application
733 self
.state
[application
] = {
734 'status': None, # Juju status
735 'container': None, # lxd container, for proxy charms
736 'actions': {}, # Actions we've executed
737 'done': False, # Are we done testing this charm?
738 'phase': "deploy", # What phase is this application in?
741 debug("Deploying charm at {}".format(self
.artifacts
[charm
]))
743 # If this is a native charm, we need to provision the underlying
744 # machine ala an LXC container.
747 if not self
.isproxy(application
):
748 debug("Creating container for native charm")
749 # args = ("default", application, None, None)
750 self
.state
[application
]['container'] = create_lxd_container(
751 name
=os
.path
.basename(__file__
)
754 hostname
= self
.get_container_ip(
755 self
.state
[application
]['container'],
759 'hostname': hostname
,
760 'username': 'ubuntu',
763 await self
.n2vc
.DeployCharms(
767 self
.get_charm(charm
),
774 def parse_vnf_descriptor(self
):
775 """Parse the VNF descriptor to make running tests easier.
777 Parse the charm information in the descriptor to make it easy to write
778 tests to run again it.
780 Each charm becomes a dictionary in a list:
783 'vnf-member-index': 1,
786 'initial-config-primitive': {},
787 'config-primitive': {}
790 - is this a proxy charm?
791 - what are the initial-config-primitives (day 1)?
792 - what are the config primitives (day 2)?
797 # You'd think this would be explicit, but it's just an incremental
798 # value that should be consistent.
801 """Get all vdu and/or vdu config in a descriptor."""
802 config
= self
.get_config()
806 # Get the name to be used for the deployed application
807 application_name
= n2vc
.vnf
.N2VC().FormatApplicationName(
810 str(vnf_member_index
),
814 'application-name': application_name
,
816 'vnf-member-index': vnf_member_index
,
817 'vnf-name': self
.vnf_name
,
819 'initial-config-primitive': {},
820 'config-primitive': {},
824 charm
['name'] = juju
['charm']
827 charm
['proxy'] = juju
['proxy']
829 if 'initial-config-primitive' in cfg
:
830 charm
['initial-config-primitive'] = \
831 cfg
['initial-config-primitive']
833 if 'config-primitive' in cfg
:
834 charm
['config-primitive'] = cfg
['config-primitive']
836 charms
[application_name
] = charm
838 # Increment the vnf-member-index
839 vnf_member_index
+= 1
844 def isproxy(self
, application_name
):
846 assert application_name
in self
.charms
847 assert 'proxy' in self
.charms
[application_name
]
848 assert type(self
.charms
[application_name
]['proxy']) is bool
850 # debug(self.charms[application_name])
851 return self
.charms
[application_name
]['proxy']
854 def get_config(self
):
855 """Return an iterable list of config items (vdu and vnf).
857 As far as N2VC is concerned, the config section for vdu and vnf are
858 identical. This joins them together so tests only need to iterate
863 """Get all vdu and/or vdu config in a descriptor."""
864 vnf_config
= self
.vnfd
.get("vnf-configuration")
866 juju
= vnf_config
['juju']
868 configs
.append(vnf_config
)
870 for vdu
in self
.vnfd
['vdu']:
871 vdu_config
= vdu
.get('vdu-configuration')
873 juju
= vdu_config
['juju']
875 configs
.append(vdu_config
)
880 def get_charm_names(self
):
881 """Return a list of charms used by the test descriptor."""
885 # Check if the VDUs in this VNF have a charm
886 for config
in self
.get_config():
887 juju
= config
['juju']
890 if name
not in charms
:
896 def get_phase(self
, application
):
897 return self
.state
[application
]['phase']
900 def set_phase(self
, application
, phase
):
901 self
.state
[application
]['phase'] = phase
904 async def configure_proxy_charm(self
, *args
):
905 """Configure a container for use via ssh."""
906 (model
, application
, _
, _
) = args
909 if self
.get_phase(application
) == "deploy":
910 self
.set_phase(application
, "configure")
912 debug("Start CreateContainer for {}".format(application
))
913 self
.state
[application
]['container'] = \
914 await self
.CreateContainer(*args
)
915 debug("Done CreateContainer for {}".format(application
))
917 if self
.state
[application
]['container']:
918 debug("Configure {} for container".format(application
))
919 if await self
.configure_ssh_proxy(application
):
920 await asyncio
.sleep(0.1)
923 debug("Failed to configure container for {}".format(application
))
925 debug("skipping CreateContainer for {}: {}".format(
927 self
.get_phase(application
),
930 except Exception as ex
:
931 debug("configure_proxy_charm exception: {}".format(ex
))
933 await asyncio
.sleep(0.1)
938 async def execute_charm_tests(self
, *args
):
939 (model
, application
, _
, _
) = args
941 debug("Executing charm test(s) for {}".format(application
))
943 if self
.state
[application
]['done']:
944 debug("Trying to execute tests against finished charm...aborting")
948 phase
= self
.get_phase(application
)
949 # We enter the test phase when after deploy (for native charms) or
950 # configure, for proxy charms.
951 if phase
in ["deploy", "configure"]:
952 self
.set_phase(application
, "test")
953 if self
.are_tests_finished():
954 raise Exception("Trying to execute init-config on finished test")
956 if await self
.execute_initial_config_primitives(application
):
958 await self
.check_metrics(application
)
960 debug("Done testing {}".format(application
))
961 self
.state
[application
]['done'] = True
963 except Exception as ex
:
964 debug("Exception in execute_charm_tests: {}".format(ex
))
966 await asyncio
.sleep(0.1)
971 async def CreateContainer(self
, *args
):
972 """Create a LXD container for use with a proxy charm.abs
974 1. Get the public key from the charm via `get-ssh-public-key` action
975 2. Create container with said key injected for the ubuntu user
977 Returns a Container object
979 # Create and configure a LXD container for use with a proxy charm.
980 (model
, application
, _
, _
) = args
982 debug("[CreateContainer] {}".format(args
))
986 # Execute 'get-ssh-public-key' primitive and get returned value
987 uuid
= await self
.n2vc
.ExecutePrimitive(
990 "get-ssh-public-key",
994 result
= await self
.n2vc
.GetPrimitiveOutput(model
, uuid
)
995 pubkey
= result
['pubkey']
997 container
= create_lxd_container(
999 name
=os
.path
.basename(__file__
)
1003 except Exception as ex
:
1004 debug("Error creating container: {}".format(ex
))
1010 async def stop(self
):
1014 - Stop and delete containers
1017 TODO: Clean up duplicate code between teardown_class() and stop()
1019 debug("stop() called")
1021 if self
.n2vc
and self
._running
and not self
._stopping
:
1022 self
._running
= False
1023 self
._stopping
= True
1025 # Destroy the network service
1027 await self
.n2vc
.DestroyNetworkService(self
.ns_name
)
1028 except Exception as e
:
1030 "Error Destroying Network Service \"{}\": {}".format(
1036 # Wait for the applications to be removed and delete the containers
1037 for application
in self
.charms
:
1041 # Wait for the application to be removed
1042 await asyncio
.sleep(10)
1043 if not await self
.n2vc
.HasApplication(
1049 # Need to wait for the charm to finish, because native charms
1050 if self
.state
[application
]['container']:
1051 debug("Deleting LXD container...")
1052 destroy_lxd_container(
1053 self
.state
[application
]['container']
1055 self
.state
[application
]['container'] = None
1056 debug("Deleting LXD container...done.")
1058 debug("No container found for {}".format(application
))
1059 except Exception as e
:
1060 debug("Error while deleting container: {}".format(e
))
1064 debug("stop(): Logging out of N2VC...")
1065 await self
.n2vc
.logout()
1067 debug("stop(): Logging out of N2VC...Done.")
1068 except Exception as ex
:
1071 # Let the test know we're finished.
1072 debug("Marking test as finished.")
1073 # self._running = False
1075 debug("Skipping stop()")
1078 def get_container_ip(self
, container
):
1079 """Return the IPv4 address of container's eth0 interface."""
1082 addresses
= container
.state().network
['eth0']['addresses']
1083 # The interface may have more than one address, but we only need
1084 # the first one for testing purposes.
1085 ipaddr
= addresses
[0]['address']
1090 async def configure_ssh_proxy(self
, application
, task
=None):
1091 """Configure the proxy charm to use the lxd container.
1093 Configure the charm to use a LXD container as it's VNF.
1095 debug("Configuring ssh proxy for {}".format(application
))
1097 mgmtaddr
= self
.get_container_ip(
1098 self
.state
[application
]['container'],
1102 "Setting ssh-hostname for {} to {}".format(
1108 await self
.n2vc
.ExecutePrimitive(
1114 'ssh-hostname': mgmtaddr
,
1115 'ssh-username': 'ubuntu',
1122 async def execute_initial_config_primitives(self
, application
, task
=None):
1123 debug("Executing initial_config_primitives for {}".format(application
))
1125 init_config
= self
.charms
[application
]
1128 The initial-config-primitive is run during deploy but may fail
1129 on some steps because proxy charm access isn't configured.
1131 Re-run those actions so we can inspect the status.
1133 uuids
= await self
.n2vc
.ExecuteInitialPrimitives(
1140 ExecutePrimitives will return a list of uuids. We need to check the
1141 status of each. The test continues if all Actions succeed, and
1142 fails if any of them fail.
1144 await self
.wait_for_uuids(application
, uuids
)
1145 debug("Primitives for {} finished.".format(application
))
1148 except Exception as ex
:
1149 debug("execute_initial_config_primitives exception: {}".format(ex
))
1155 async def check_metrics(self
, application
, task
=None):
1156 """Check and run metrics, if present.
1158 Checks to see if metrics are specified by the charm. If so, collects
1161 If no metrics, then mark the test as finished.
1163 if has_metrics(self
.charms
[application
]['name']):
1164 debug("Collecting metrics for {}".format(application
))
1166 metrics
= await self
.n2vc
.GetMetrics(
1171 return await self
.verify_metrics(application
, metrics
)
1174 async def verify_metrics(self
, application
, metrics
):
1175 """Verify the charm's metrics.
1177 Verify that the charm has sent metrics successfully.
1179 Stops the test when finished.
1181 debug("Verifying metrics for {}: {}".format(application
, metrics
))
1187 # TODO: Ran into a case where it took 9 attempts before metrics
1188 # were available; the controller is slow sometimes.
1189 await asyncio
.sleep(30)
1190 return await self
.check_metrics(application
)
1193 async def wait_for_uuids(self
, application
, uuids
):
1194 """Wait for primitives to execute.
1196 The task will provide a list of uuids representing primitives that are
1199 debug("Waiting for uuids for {}: {}".format(application
, uuids
))
1200 waitfor
= len(uuids
)
1203 while waitfor
> finished
:
1205 await asyncio
.sleep(10)
1207 if uuid
not in self
.state
[application
]['actions']:
1208 self
.state
[application
]['actions'][uid
] = "pending"
1210 status
= self
.state
[application
]['actions'][uid
]
1212 # Have we already marked this as done?
1213 if status
in ["pending", "running"]:
1215 debug("Getting status of {} ({})...".format(uid
, status
))
1216 status
= await self
.n2vc
.GetPrimitiveStatus(
1220 debug("...state of {} is {}".format(uid
, status
))
1221 self
.state
[application
]['actions'][uid
] = status
1223 if status
in ['completed', 'failed']:
1226 debug("{}/{} actions complete".format(finished
, waitfor
))
1228 # Wait for the primitive to finish and try again
1229 if waitfor
> finished
:
1230 debug("Waiting 10s for action to finish...")
1231 await asyncio
.sleep(10)
1234 def n2vc_callback(self
, *args
, **kwargs
):
1235 (model
, application
, status
, message
) = args
1236 # debug("callback: {}".format(args))
1238 if application
not in self
.state
:
1239 # Initialize the state of the application
1240 self
.state
[application
] = {
1241 'status': None, # Juju status
1242 'container': None, # lxd container, for proxy charms
1243 'actions': {}, # Actions we've executed
1244 'done': False, # Are we done testing this charm?
1245 'phase': "deploy", # What phase is this application in?
1248 self
.state
[application
]['status'] = status
1250 if status
in ['waiting', 'maintenance', 'unknown']:
1251 # Nothing to do for these
1254 debug("callback: {}".format(args
))
1256 if self
.state
[application
]['done']:
1257 debug("{} is done".format(application
))
1260 if status
in ['error']:
1261 # To test broken charms, if a charm enters an error state we should
1263 debug("{} is in an error state, stop the test.".format(application
))
1264 # asyncio.ensure_future(self.stop())
1265 self
.state
[application
]['done'] = True
1268 if status
in ["blocked"] and self
.isproxy(application
):
1269 if self
.state
[application
]['phase'] == "deploy":
1270 debug("Configuring proxy charm for {}".format(application
))
1271 asyncio
.ensure_future(self
.configure_proxy_charm(*args
))
1273 elif status
in ["active"]:
1274 """When a charm is active, we can assume that it has been properly
1275 configured (not blocked), regardless of if it's a proxy or not.
1277 All primitives should be complete by init_config_primitive
1279 asyncio
.ensure_future(self
.execute_charm_tests(*args
))