4 # Copyright 2016 RIFT.IO Inc
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
32 gi.require_version('RwDtsToyTaskletYang', '1.0')
33 gi.require_version('RwManifestYang', '1.0')
34 gi.require_version('RwVcsYang', '1.0')
36 import gi.repository.RwManifestYang as rwmanifest
37 import gi.repository.RwVcsYang as rwvcs
38 import gi.repository.RwDtsToyTaskletYang as toyyang
39 import gi.repository.RwYang as RwYang
40 import rift.auto.session
46 if sys.version_info < (3, 4, 4):
47 asyncio.ensure_future = asyncio.async
49 class LaunchPad(rift.test.dts.AbstractDTSTest):
51 DTS GI interface unittests
53 Note: Each tests uses a list of asyncio.Events for staging through the
54 test. These are required here because we are bring up each coroutine
55 ("tasklet") at the same time and are not implementing any re-try
56 mechanisms. For instance, this is used in numerous tests to make sure that
57 a publisher is up and ready before the subscriber sends queries. Such
58 event lists should not be used in production software.
62 1. Creates an asyncio loop
63 2. Triggers the hook configure_test
65 def scheduler_tick(self, *args):
66 self.call_soon(self.stop)
69 # Init params: loop & timers
70 self.loop = asyncio.new_event_loop()
72 self.loop.scheduler_tick = types.MethodType(scheduler_tick, self.loop)
74 self.asyncio_timer = None
75 self.stop_timer = None
76 self.__class__.id_cnt += 1
77 self.configure_test(self.loop, self.__class__.id_cnt)
80 def configure_schema(cls):
81 schema = RwYang.Model.load_and_merge_schema(rwvcs.get_schema(), 'librwcal_yang_gen.so', 'Rwcal')
82 cls.model = RwYang.Model.create_libyang()
83 cls.model.load_schema_ypbc(schema)
84 xml = cls.manifest.to_xml_v2(cls.model, 1)
85 xml = re.sub('rw-manifest:', '', xml)
86 xml = re.sub('<manifest xmlns:rw-manifest="http://riftio.com/ns/riftware-1.0/rw-manifest">', '<?xml version="1.0" ?>\n<manifest xmlns="http://riftio.com/ns/riftware-1.0/rw-manifest" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://riftio.com/ns/riftware-1.0/rw-manifest ./rw-manifest.xsd">', xml)
87 xml = '\n'.join(xml.split('\n')[1:])
88 with open('lptestmanifest.xml', 'w') as f:
95 def configure_manifest(cls):
96 manifest = rwmanifest.Manifest()
97 manifest.bootstrap_phase = rwmanifest.BootstrapPhase.from_dict({
99 "northbound_listing": [ "platform_schema_listing.txt", "platform_mgmt_schema_listing.txt", "cli_launchpad_schema_listing.txt" ]
102 "plugin_name": "rwinit-c"
111 "bootstrap_time": 30,
112 "console_severity": 4
116 "ip_addr": "127.0.0.1",
120 "master_ip": "127.0.0.1",
121 "unique_ports": False,
130 "component_name": "msgbroker",
134 "component_name": "dtsrouter",
141 # "cert": "/net/mahi/localdisk/kelayath/ws/coreha/etc/ssl/current.cert",
142 # "key": "/net/mahi/localdisk/kelayath/ws/coreha/etc/ssl/current.key"
145 manifest.init_phase = rwmanifest.InitPhase.from_dict({
148 "vm_ip_address = '127.0.0.1'",
149 "rw_component_name = 'vm-launchpad'",
151 "component_type = 'rwvm'",
153 "component_name": "$python(rw_component_name)",
154 "instance_id": "$python(instance_id)",
155 "component_type": "$python(rw_component_type)"
169 "collapse_each_rwvm": False,
170 "collapse_each_rwprocess": False
174 manifest.inventory = rwmanifest.Inventory.from_dict({
177 "component_name": "master",
178 "component_type": "RWCOLLECTION",
180 "collection_type": "rwcolony",
185 "name": "Start vm-launchpad for master",
187 "python_variable": ["vm_ip_address = '127.0.0.1'"],
188 "component_name": "vm-launchpad",
198 "component_name": "vm-launchpad",
199 "component_type": "RWVM",
207 "name": "Start the master",
209 "component_name": "master",
210 "recovery_action": "RESTART",
215 "name": "Start the RW.Proc_1.Restconf",
217 "component_name": "RW.Proc_1.Restconf",
218 "recovery_action": "RESTART",
223 "name": "Start the RW.Proc_3.CalProxy",
225 "component_name": "RW.Proc_3.CalProxy",
226 "recovery_action": "RESTART",
231 "name": "Start the RW.Proc_4.nfvi-metrics-monitor",
233 "component_name": "RW.Proc_4.nfvi-metrics-monitor",
234 "recovery_action": "RESTART",
239 "name": "Start the RW.Proc_5.network-services-manager",
241 "component_name": "RW.Proc_5.network-services-manager",
242 "recovery_action": "RESTART",
247 "name": "Start the RW.Proc_6.virtual-network-function-manager",
249 "component_name": "RW.Proc_6.virtual-network-function-manager",
250 "recovery_action": "RESTART",
255 "name": "Start the RW.Proc_7.virtual-network-service",
257 "component_name": "RW.Proc_7.virtual-network-service",
258 "recovery_action": "RESTART",
263 "name": "Start the RW.Proc_8.nfvi-metrics-monitor",
265 "component_name": "RW.Proc_8.nfvi-metrics-monitor",
266 "recovery_action": "RESTART",
271 "name": "Start the RW.MC.UI",
273 "component_name": "RW.MC.UI",
274 "recovery_action": "RESTART",
279 # "name": "Start the RW.COMPOSER.UI",
281 # "component_name": "RW.COMPOSER.UI",
282 # "config_ready": True
286 "name": "Start the RW.Proc_10.launchpad",
288 "component_name": "RW.Proc_10.launchpad",
289 "recovery_action": "RESTART",
294 "name": "Start the RW.Proc_11.Resource-Manager",
296 "component_name": "RW.Proc_11.Resource-Manager",
297 "recovery_action": "RESTART",
302 "name": "Start the RW.uAgent",
304 "python_variable": ["cmdargs_str = '--confd-proto AF_INET --confd-ip 127.0.0.1'"],
305 "component_name": "RW.uAgent",
306 "recovery_action": "RESTART",
311 "name": "Start the logd",
313 "component_name": "logd",
314 "recovery_action": "RESTART",
324 # "component_name": "RW.CLI",
325 # "component_type": "PROC",
327 # "exe_path": "./usr/bin/rwcli",
328 # "args": "--netconf_host 127.0.0.1 --netconf_port 2022 --schema_listing cli_launchpad_schema_listing.txt",
332 "component_name": "RW.Proc_1.Restconf",
333 "component_type": "RWPROC",
336 "name": "Start RW.Restconf for RW.Proc_1.Restconf",
337 "component_name": "RW.Restconf",
338 "recovery_action": "RESTART",
344 "component_name": "RW.Restconf",
345 "component_type": "RWTASKLET",
347 "plugin_directory": "./usr/lib/rift/plugins/restconf",
348 "plugin_name": "restconf"
352 "component_name": "RW.Proc_3.CalProxy",
353 "component_type": "RWPROC",
356 "name": "Start RW.CalProxy for RW.Proc_3.CalProxy",
357 "component_name": "RW.CalProxy",
358 "recovery_action": "RESTART",
364 "component_name": "RW.CalProxy",
365 "component_type": "RWTASKLET",
367 "plugin_directory": "./usr/lib/rift/plugins/rwcalproxytasklet",
368 "plugin_name": "rwcalproxytasklet"
372 "component_name": "RW.Proc_4.nfvi-metrics-monitor",
373 "component_type": "RWPROC",
376 "name": "Start nfvi-metrics-monitor for RW.Proc_4.nfvi-metrics-monitor",
377 "component_name": "nfvi-metrics-monitor",
378 "recovery_action": "RESTART",
384 "component_name": "nfvi-metrics-monitor",
385 "component_type": "RWTASKLET",
387 "plugin_directory": "./usr/lib/rift/plugins/rwmonitor",
388 "plugin_name": "rwmonitor"
392 "component_name": "RW.Proc_5.network-services-manager",
393 "component_type": "RWPROC",
396 "name": "Start network-services-manager for RW.Proc_5.network-services-manager",
397 "component_name": "network-services-manager",
398 "recovery_action": "RESTART",
404 "component_name": "network-services-manager",
405 "component_type": "RWTASKLET",
407 "plugin_directory": "./usr/lib/rift/plugins/rwnsmtasklet",
408 "plugin_name": "rwnsmtasklet"
412 "component_name": "RW.Proc_6.virtual-network-function-manager",
413 "component_type": "RWPROC",
416 "name": "Start virtual-network-function-manager for RW.Proc_6.virtual-network-function-manager",
417 "component_name": "virtual-network-function-manager",
418 "recovery_action": "RESTART",
424 "component_name": "virtual-network-function-manager",
425 "component_type": "RWTASKLET",
427 "plugin_directory": "./usr/lib/rift/plugins/rwvnfmtasklet",
428 "plugin_name": "rwvnfmtasklet"
432 "component_name": "RW.Proc_7.virtual-network-service",
433 "component_type": "RWPROC",
436 "name": "Start virtual-network-service for RW.Proc_7.virtual-network-service",
437 "component_name": "virtual-network-service",
438 "recovery_action": "RESTART",
444 "component_name": "virtual-network-service",
445 "component_type": "RWTASKLET",
447 "plugin_directory": "./usr/lib/rift/plugins/rwvnstasklet",
448 "plugin_name": "rwvnstasklet"
452 "component_name": "RW.Proc_8.nfvi-metrics-monitor",
453 "component_type": "RWPROC",
456 "name": "Start nfvi-metrics-monitor for RW.Proc_8.nfvi-metrics-monitor",
457 "component_name": "nfvi-metrics-monitor",
458 "recovery_action": "RESTART",
464 "component_name": "RW.MC.UI",
465 "component_type": "PROC",
467 "exe_path": "./usr/share/rw.ui/skyquake/scripts/launch_ui.sh",
471 "component_name": "RW.COMPOSER.UI",
472 "component_type": "PROC",
474 "exe_path": "./usr/share/composer/scripts/launch_composer.sh",
478 "component_name": "RW.Proc_9.Configuration-Manager",
479 "component_type": "RWPROC",
482 "name": "Start Configuration-Manager for RW.Proc_9.Configuration-Manager",
483 "component_name": "Configuration-Manager",
484 "recovery_action": "RESTART",
490 "component_name": "Configuration-Manager",
491 "component_type": "RWTASKLET",
493 "plugin_directory": "./usr/lib/rift/plugins/rwconmantasklet",
494 "plugin_name": "rwconmantasklet"
498 "component_name": "RW.Proc_10.launchpad",
499 "component_type": "RWPROC",
502 "name": "Start launchpad for RW.Proc_10.launchpad",
503 "component_name": "launchpad",
504 "recovery_action": "RESTART",
510 "component_name": "launchpad",
511 "component_type": "RWTASKLET",
513 "plugin_directory": "./usr/lib/rift/plugins/rwlaunchpad",
514 "plugin_name": "rwlaunchpad"
518 "component_name": "RW.Proc_11.Resource-Manager",
519 "component_type": "RWPROC",
522 "name": "Start Resource-Manager for RW.Proc_11.Resource-Manager",
523 "component_name": "Resource-Manager",
524 "recovery_action": "RESTART",
530 "component_name": "Resource-Manager",
531 "component_type": "RWTASKLET",
533 "plugin_directory": "./usr/lib/rift/plugins/rwresmgrtasklet",
534 "plugin_name": "rwresmgrtasklet"
538 "component_name": "RW.uAgent",
539 "component_type": "RWTASKLET",
541 "plugin_directory": "./usr/lib/rift/plugins/rwuagent-c",
542 "plugin_name": "rwuagent-c"
546 "component_name": "logd",
547 "component_type": "RWTASKLET",
549 "plugin_directory": "./usr/lib/rift/plugins/rwlogd-c",
550 "plugin_name": "rwlogd-c"
554 "component_name": "msgbroker",
555 "component_type": "RWTASKLET",
557 "plugin_directory": "./usr/lib/rift/plugins/rwmsgbroker-c",
558 "plugin_name": "rwmsgbroker-c"
562 "component_name": "dtsrouter",
563 "component_type": "RWTASKLET",
565 "plugin_directory": "./usr/lib/rift/plugins/rwdtsrouter-c",
566 "plugin_name": "rwdtsrouter-c"
574 tasklist = { 'reaperd',
577 for proc in psutil.process_iter():
578 if proc.name() in tasklist:
579 print("killing", proc.name())
583 print(proc.name(), "no longer exists")
588 class LaunchPadTest(LaunchPad):
590 DTS GI interface unittests
592 Note: Each tests uses a list of asyncio.Events for staging through the
593 test. These are required here because we are bring up each coroutine
594 ("tasklet") at the same time and are not implementing any re-try
595 mechanisms. For instance, this is used in numerous tests to make sure that
596 a publisher is up and ready before the subscriber sends queries. Such
597 event lists should not be used in production software.
601 res_iter = yield from self.dts_mgmt.query_read('/rw-base:vcs/rw-base:info', flags=0)
603 info_result = yield from i
604 components = info_result.result.components.component_info
606 for component in components:
607 recvd_list[component.component_name] = (component.instance_id,
608 component.rwcomponent_parent,
609 component.component_type,
614 def issue_vcrash(self, component_type):
615 # critical_components = {'msgbroker', 'dtsrouter'}
616 critical_components = {'msgbroker', 'dtsrouter', 'RW.uAgent'}
617 comp_inventory = yield from self.inventory()
618 for component in comp_inventory:
619 if ((comp_inventory[component])[2] == component_type):
620 inst = (comp_inventory[component])[0]
621 if (component in critical_components):
622 print(component, 'Marked as CRITICAL - Not restarting')
624 print('Crashing ', component_type,component)
625 vcrash_input = rwvcs.VCrashInput(instance_name=component+'-'+str(inst))
626 query_iter = yield from self.dts_mgmt.query_rpc( xpath="/rw-vcs:vcrash",
627 flags=0, msg=vcrash_input)
628 yield from asyncio.sleep(1, loop=self.loop)
629 restarted_inventory = yield from self.inventory()
630 self.assertTrue(restarted_inventory[component][3] != 'TO_RECOVER')
632 def test_launch_pad(self):
634 Verify the launchpad setup functions
635 The test will progress through stages defined by the events list:
636 0: mission_control setup is brought up
637 2: Tasklet/PROC/VM restarts tested to confirm recovery is proper
640 print("{{{{{{{{{{{{{{{{{{{{STARTING - mano recovery test")
641 # confd_host="127.0.0.1"
643 events = [asyncio.Event(loop=self.loop) for _ in range(2)]
648 tinfo = self.new_tinfo('sub')
649 self.dts_mgmt = rift.tasklets.DTS(tinfo, self.schema, self.loop)
651 # Sleep for DTS registrations to complete
652 print('.........................................................')
653 print('........SLEEPING 80 seconds for system to come up........')
654 yield from asyncio.sleep(80, loop=self.loop)
655 print('........RESUMING........')
658 def issue_vstop(component,inst,flag=0):
659 vstop_input = rwvcs.VStopInput(instance_name=component+'-'+(str(inst)))
660 query_iter = yield from self.dts_mgmt.query_rpc( xpath="/rw-vcs:vstop",
661 flags=flag, msg=vstop_input)
662 yield from asyncio.sleep(1, loop=self.loop)
667 def issue_vstart(component, parent, recover=False):
668 vstart_input = rwvcs.VStartInput()
669 vstart_input.component_name = component
670 vstart_input.parent_instance = parent
671 vstart_input.recover = recover
672 query_iter = yield from self.dts_mgmt.query_rpc( xpath="/rw-vcs:vstart",
673 flags=0, msg=vstart_input)
674 yield from asyncio.sleep(1, loop=self.loop)
677 def issue_start_stop(comp_inventory, component_type):
678 # critical_components = {'msgbroker', 'dtsrouter'}
679 critical_components = {'msgbroker', 'dtsrouter', 'RW.uAgent'}
680 for component in comp_inventory:
681 if ((comp_inventory[component])[2] == component_type):
682 inst = (comp_inventory[component])[0]
683 parent = (comp_inventory[component])[1]
684 if (component in critical_components):
685 print(component, 'Marked as CRITICAL - Not restarting')
687 print('Stopping ', component_type,component)
688 yield from issue_vstop(component,inst)
689 restarted_inventory = yield from self.inventory()
690 # self.assertEqual(restarted_inventory[component][3],'TO_RECOVER')
691 print('Starting ',component_type,component)
692 yield from issue_vstart(component, parent, recover=True)
693 restarted_inventory = yield from self.inventory()
694 self.assertTrue(restarted_inventory[component][3] != 'TO_RECOVER')
696 yield from asyncio.sleep(20, loop=self.loop)
697 comp_inventory = yield from self.inventory()
698 yield from issue_start_stop(comp_inventory, 'RWTASKLET')
699 # yield from issue_start_stop(comp_inventory, 'RWPROC')
700 # yield from self.issue_vcrash('RWTASKLET')
702 yield from asyncio.sleep(20, loop=self.loop)
703 restarted_inventory = yield from self.inventory()
704 # critical_components = {'msgbroker', 'dtsrouter', 'RW.uAgent'}
705 for comp in comp_inventory:
706 self.assertEqual(str(comp_inventory[comp]), str(restarted_inventory[comp]))
707 # if (comp not in critical_components):
708 # inst = (comp_inventory[comp])[0]
709 # yield from issue_vstop(comp,inst)
713 asyncio.ensure_future(sub(), loop=self.loop)
714 self.run_until(events[1].is_set, timeout=260)
718 plugin_dir = os.path.join(os.environ["RIFT_INSTALL"], "usr/lib/rift/plugins")
719 if 'DTS_TEST_PUB_DIR' not in os.environ:
720 os.environ['DTS_TEST_PUB_DIR'] = os.path.join(plugin_dir, 'dtstestpub')
722 if 'RIFT_NO_SUDO_REAPER' not in os.environ:
723 os.environ['RIFT_NO_SUDO_REAPER'] = '1'
725 if 'MESSAGE_BROKER_DIR' not in os.environ:
726 os.environ['MESSAGE_BROKER_DIR'] = os.path.join(plugin_dir, 'rwmsgbroker-c')
728 if 'ROUTER_DIR' not in os.environ:
729 os.environ['ROUTER_DIR'] = os.path.join(plugin_dir, 'rwdtsrouter-c')
731 if 'RW_VAR_RIFT' not in os.environ:
732 os.environ['RW_VAR_RIFT'] = '1'
734 if 'INSTALLDIR' in os.environ:
735 os.chdir(os.environ.get('INSTALLDIR'))
737 # if 'RWMSG_BROKER_SHUNT' not in os.environ:
738 # os.environ['RWMSG_BROKER_SHUNT'] = '1'
740 if 'TEST_ENVIRON' not in os.environ:
741 os.environ['TEST_ENVIRON'] = '1'
743 if 'RW_MANIFEST' not in os.environ:
744 os.environ['RW_MANIFEST'] = os.path.join(install_dir, 'lptestmanifest.xml')
746 parser = argparse.ArgumentParser()
747 parser.add_argument('-v', '--verbose', action='store_true')
748 args, _ = parser.parse_known_args()
751 runner = xmlrunner.XMLTestRunner(output=os.environ["RIFT_MODULE_TEST"])
752 unittest.main(testRunner=runner)
754 if __name__ == '__main__':