RIFT OSM R1 Initial Submission
[osm/SO.git] / rwlaunchpad / test / launchpad_recovery
diff --git a/rwlaunchpad/test/launchpad_recovery b/rwlaunchpad/test/launchpad_recovery
new file mode 100755 (executable)
index 0000000..eea5d4a
--- /dev/null
@@ -0,0 +1,793 @@
+#!/usr/bin/env python3
+
+# 
+#   Copyright 2016 RIFT.IO Inc
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+import argparse
+import asyncio
+import logging
+import os
+import sys
+import unittest
+import re
+import psutil
+import types
+
+import xmlrunner
+
+import gi
+gi.require_version('RwDtsToyTaskletYang', '1.0')
+gi.require_version('RwManifestYang', '1.0')
+gi.require_version('RwVcsYang', '1.0')
+
+import gi.repository.RwManifestYang as rwmanifest
+import gi.repository.RwVcsYang as rwvcs
+import gi.repository.RwDtsToyTaskletYang as toyyang
+import gi.repository.RwYang as RwYang
+import rift.auto.session
+import rift.vcs.vcs
+
+import rift.tasklets
+import rift.test.dts
+
+if sys.version_info < (3, 4, 4):
+    asyncio.ensure_future = asyncio.async
+
+class LaunchPad(rift.test.dts.AbstractDTSTest):
+    """
+    DTS GI interface unittests
+
+    Note:  Each tests uses a list of asyncio.Events for staging through the
+    test.  These are required here because we are bring up each coroutine
+    ("tasklet") at the same time and are not implementing any re-try
+    mechanisms.  For instance, this is used in numerous tests to make sure that
+    a publisher is up and ready before the subscriber sends queries.  Such
+    event lists should not be used in production software.
+    """
+    def setUp(self):
+        """
+        1. Creates an asyncio loop
+        2. Triggers the hook configure_test
+        """
+        def scheduler_tick(self, *args):
+            self.call_soon(self.stop)
+            self.run_forever()
+
+        # Init params: loop & timers
+        self.loop = asyncio.new_event_loop()
+
+        self.loop.scheduler_tick = types.MethodType(scheduler_tick, self.loop)
+
+        self.asyncio_timer = None
+        self.stop_timer = None
+        self.__class__.id_cnt += 1
+        self.configure_test(self.loop, self.__class__.id_cnt)
+
+    @classmethod
+    def configure_schema(cls):
+        schema =  RwYang.Model.load_and_merge_schema(rwvcs.get_schema(), 'librwcal_yang_gen.so', 'Rwcal')
+        cls.model = RwYang.Model.create_libncx()
+        cls.model.load_schema_ypbc(schema)
+        xml = cls.manifest.to_xml_v2(cls.model, 1)
+        xml = re.sub('rw-manifest:', '', xml)
+        xml = re.sub('<manifest xmlns:rw-manifest="http://riftio.com/ns/riftware-1.0/rw-manifest">', '<?xml version="1.0" ?>\n<manifest xmlns="http://riftio.com/ns/riftware-1.0/rw-manifest" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://riftio.com/ns/riftware-1.0/rw-manifest ./rw-manifest.xsd">', xml)
+        xml = '\n'.join(xml.split('\n')[1:]) 
+        with open('lptestmanifest.xml', 'w') as f:
+           f.write(str(xml))
+        f.close()
+        return schema
+
+
+    @classmethod
+    def configure_manifest(cls):
+        manifest = rwmanifest.Manifest()
+        manifest.bootstrap_phase = rwmanifest.BootstrapPhase.from_dict({
+            "rwmgmt": {
+                "northbound_listing": [ "cli_launchpad_schema_listing.txt" ]
+            }, 
+            "rwtasklet": {
+                "plugin_name": "rwinit-c"
+            }, 
+            "rwtrace": {
+                "enable": True, 
+                "level": 5, 
+            }, 
+            "log": {
+                "enable": True, 
+                "severity": 4, 
+                "bootstrap_time": 30, 
+                "console_severity": 4
+            }, 
+            "ip_addrs_list": [
+                {
+                    "ip_addr": "127.0.0.1", 
+                }
+            ], 
+            "zookeeper": {
+                "master_ip": "127.0.0.1", 
+                "unique_ports": False, 
+                "zake": False
+            }, 
+            "serf": {
+                "start": True
+            }, 
+            "rwvm": {
+                "instances": [
+                    {
+                        "component_name": "msgbroker", 
+                        "config_ready": True
+                    }, 
+                    {
+                        "component_name": "dtsrouter", 
+                        "config_ready": True
+                    }
+                ]
+            }, 
+#           "rwsecurity": {
+#               "use_ssl": True, 
+#               "cert": "/net/mahi/localdisk/kelayath/ws/coreha/etc/ssl/current.cert", 
+#               "key": "/net/mahi/localdisk/kelayath/ws/coreha/etc/ssl/current.key"
+#           }
+        }) 
+        manifest.init_phase = rwmanifest.InitPhase.from_dict({
+            "environment": {
+                "python_variable": [
+                    "vm_ip_address = '127.0.0.1'",
+                    "rw_component_name = 'vm-launchpad'",
+                    "instance_id = 1",
+                    "component_type = 'rwvm'",
+                ], 
+                "component_name": "$python(rw_component_name)", 
+                "instance_id": "$python(instance_id)", 
+                "component_type": "$python(rw_component_type)"
+            }, 
+            "settings": {
+                "rwmsg": {
+                    "multi_broker": {
+                        "enable": False
+                    }
+                }, 
+                "rwdtsrouter": {
+                    "multi_dtsrouter": {
+                        "enable": True
+                    }
+                }, 
+                "rwvcs": {
+                    "collapse_each_rwvm": False, 
+                    "collapse_each_rwprocess": False
+                }
+            }
+        }) 
+        manifest.inventory = rwmanifest.Inventory.from_dict({
+            "component": [
+                {
+                    "component_name": "master", 
+                    "component_type": "RWCOLLECTION", 
+                    "rwcollection": {
+                        "collection_type": "rwcolony", 
+                        "event_list": {
+                            "event": [{
+                                "name": "onentry", 
+                                "action": [{
+                                    "name": "Start vm-launchpad for master", 
+                                    "start": {
+                                        "python_variable": ["vm_ip_address = '127.0.0.1'"], 
+                                        "component_name": "vm-launchpad", 
+                                        "instance_id": "1", 
+                                        "config_ready": True
+                                    }
+                                }]
+                            }]
+                        }
+                    }
+                }, 
+                {
+                    "component_name": "vm-launchpad", 
+                    "component_type": "RWVM", 
+                    "rwvm": {
+                        "leader": True, 
+                        "event_list": {
+                            "event": [{
+                                "name": "onentry", 
+                                "action": [
+                                    {
+                                        "name": "Start the master", 
+                                        "start": {
+                                            "component_name": "master", 
+                                            "recovery_action": "RESTART",
+                                            "config_ready": True
+                                        }
+                                    }, 
+#                                   {
+#                                       "name": "Start the RW.CLI", 
+#                                       "start": {
+#                                           "component_name": "RW.CLI", 
+#                                           "recovery_action": "RESTART",
+#                                           "config_ready": True
+#                                       }
+#                                   }, 
+                                    {
+                                        "name": "Start the RW.Proc_1.Restconf", 
+                                        "start": {
+                                            "component_name": "RW.Proc_1.Restconf", 
+                                            "recovery_action": "RESTART",
+                                            "config_ready": True
+                                        }
+                                    }, 
+#                                   {
+#                                       "name": "Start the RW.Proc_2.RestPortForward", 
+#                                       "start": {
+#                                           "component_name": "RW.Proc_2.RestPortForward", 
+#                                           "recovery_action": "RESTART",
+#                                           "config_ready": True
+#                                       }
+#                                   }, 
+                                    {
+                                        "name": "Start the RW.Proc_3.CalProxy", 
+                                        "start": {
+                                            "component_name": "RW.Proc_3.CalProxy", 
+                                            "recovery_action": "RESTART",
+                                            "config_ready": True
+                                        }
+                                    }, 
+                                    {
+                                        "name": "Start the RW.Proc_4.nfvi-metrics-monitor", 
+                                        "start": {
+                                            "component_name": "RW.Proc_4.nfvi-metrics-monitor", 
+                                            "recovery_action": "RESTART",
+                                            "config_ready": True
+                                        }
+                                    }, 
+                                    {
+                                        "name": "Start the RW.Proc_5.network-services-manager", 
+                                        "start": {
+                                            "component_name": "RW.Proc_5.network-services-manager", 
+                                            "recovery_action": "RESTART",
+                                            "config_ready": True
+                                        }
+                                    }, 
+                                    {
+                                        "name": "Start the RW.Proc_6.virtual-network-function-manager", 
+                                        "start": {
+                                            "component_name": "RW.Proc_6.virtual-network-function-manager", 
+                                            "recovery_action": "RESTART",
+                                            "config_ready": True
+                                        }
+                                    }, 
+                                    {
+                                        "name": "Start the RW.Proc_7.virtual-network-service", 
+                                        "start": {
+                                            "component_name": "RW.Proc_7.virtual-network-service", 
+                                            "recovery_action": "RESTART",
+                                            "config_ready": True
+                                        }
+                                    }, 
+                                    {
+                                        "name": "Start the RW.Proc_8.nfvi-metrics-monitor", 
+                                        "start": {
+                                            "component_name": "RW.Proc_8.nfvi-metrics-monitor", 
+                                            "recovery_action": "RESTART",
+                                            "config_ready": True
+                                        }
+                                    }, 
+                                    {
+                                        "name": "Start the RW.MC.UI", 
+                                        "start": {
+                                            "component_name": "RW.MC.UI", 
+                                            "recovery_action": "RESTART",
+                                            "config_ready": True
+                                        }
+                                    }, 
+#                                   {
+#                                       "name": "Start the RW.COMPOSER.UI", 
+#                                       "start": {
+#                                           "component_name": "RW.COMPOSER.UI", 
+#                                           "config_ready": True
+#                                       }
+#                                   }, 
+                                    {
+                                        "name": "Start the RW.Proc_10.launchpad", 
+                                        "start": {
+                                            "component_name": "RW.Proc_10.launchpad", 
+                                            "recovery_action": "RESTART",
+                                            "config_ready": True
+                                        }
+                                    }, 
+                                    {
+                                        "name": "Start the RW.Proc_11.Resource-Manager", 
+                                        "start": {
+                                            "component_name": "RW.Proc_11.Resource-Manager", 
+                                            "recovery_action": "RESTART",
+                                            "config_ready": True
+                                        }
+                                    }, 
+                                    {
+                                        "name": "Start the RW.uAgent", 
+                                        "start": {
+                                            "python_variable": ["cmdargs_str = '--confd-proto AF_INET --confd-ip 127.0.0.1'"], 
+                                            "component_name": "RW.uAgent", 
+                                            "recovery_action": "RESTART",
+                                            "config_ready": True
+                                        }
+                                    }, 
+                                    {
+                                        "name": "Start the logd", 
+                                        "start": {
+                                            "component_name": "logd", 
+                                            "recovery_action": "RESTART",
+                                            "config_ready": True
+                                        }
+                                    }
+                                ]
+                            }]
+                        }
+                    }
+                }, 
+#               {
+#                   "component_name": "RW.CLI", 
+#                   "component_type": "PROC", 
+#                   "native_proc": {
+#                       "exe_path": "./usr/bin/rwcli", 
+#                       "args": "--netconf_host 127.0.0.1 --netconf_port 2022 --schema_listing cli_launchpad_schema_listing.txt", 
+#                   }
+#               }, 
+                {
+                    "component_name": "RW.Proc_1.Restconf", 
+                    "component_type": "RWPROC", 
+                    "rwproc": {
+                        "tasklet": [{
+                            "name": "Start RW.Restconf for RW.Proc_1.Restconf", 
+                            "component_name": "RW.Restconf", 
+                            "recovery_action": "RESTART",
+                            "config_ready": True
+                        }]
+                    }
+                }, 
+                {
+                    "component_name": "RW.Restconf", 
+                    "component_type": "RWTASKLET", 
+                    "rwtasklet": {
+                        "plugin_directory": "./usr/lib/rift/plugins/restconf", 
+                        "plugin_name": "restconf"
+                    }
+                }, 
+#               {
+#                   "component_name": "RW.Proc_2.RestPortForward", 
+#                   "component_type": "RWPROC", 
+#                   "rwproc": {
+#                       "tasklet": [{
+#                           "name": "Start RW.RestPortForward for RW.Proc_2.RestPortForward", 
+#                           "component_name": "RW.RestPortForward", 
+#                           "recovery_action": "RESTART",
+#                           "config_ready": True
+#                       }]
+#                   }
+#               }, 
+#               {
+#                   "component_name": "RW.RestPortForward", 
+#                   "component_type": "RWTASKLET", 
+#                   "rwtasklet": {
+#                       "plugin_directory": "./usr/lib/rift/plugins/restportforward", 
+#                       "plugin_name": "restportforward"
+#                   }
+#               }, 
+                {
+                    "component_name": "RW.Proc_3.CalProxy", 
+                    "component_type": "RWPROC", 
+                    "rwproc": {
+                        "tasklet": [{
+                            "name": "Start RW.CalProxy for RW.Proc_3.CalProxy", 
+                            "component_name": "RW.CalProxy", 
+                            "recovery_action": "RESTART",
+                            "config_ready": True
+                        }]
+                    }
+                }, 
+                {
+                    "component_name": "RW.CalProxy", 
+                    "component_type": "RWTASKLET", 
+                    "rwtasklet": {
+                        "plugin_directory": "./usr/lib/rift/plugins/rwcalproxytasklet", 
+                        "plugin_name": "rwcalproxytasklet"
+                    }
+                }, 
+                {
+                    "component_name": "RW.Proc_4.nfvi-metrics-monitor", 
+                    "component_type": "RWPROC", 
+                    "rwproc": {
+                        "tasklet": [{
+                            "name": "Start nfvi-metrics-monitor for RW.Proc_4.nfvi-metrics-monitor", 
+                            "component_name": "nfvi-metrics-monitor", 
+                            "recovery_action": "RESTART",
+                            "config_ready": True
+                        }]
+                    }
+                }, 
+                {
+                    "component_name": "nfvi-metrics-monitor", 
+                    "component_type": "RWTASKLET", 
+                    "rwtasklet": {
+                        "plugin_directory": "./usr/lib/rift/plugins/rwmonitor", 
+                        "plugin_name": "rwmonitor"
+                    }
+                }, 
+                {
+                    "component_name": "RW.Proc_5.network-services-manager", 
+                    "component_type": "RWPROC", 
+                    "rwproc": {
+                        "tasklet": [{
+                            "name": "Start network-services-manager for RW.Proc_5.network-services-manager", 
+                            "component_name": "network-services-manager", 
+                            "recovery_action": "RESTART",
+                            "config_ready": True
+                        }]
+                    }
+                }, 
+                {
+                    "component_name": "network-services-manager", 
+                    "component_type": "RWTASKLET", 
+                    "rwtasklet": {
+                        "plugin_directory": "./usr/lib/rift/plugins/rwnsmtasklet", 
+                        "plugin_name": "rwnsmtasklet"
+                    }
+                }, 
+                {
+                    "component_name": "RW.Proc_6.virtual-network-function-manager", 
+                    "component_type": "RWPROC", 
+                    "rwproc": {
+                        "tasklet": [{
+                            "name": "Start virtual-network-function-manager for RW.Proc_6.virtual-network-function-manager", 
+                            "component_name": "virtual-network-function-manager", 
+                            "recovery_action": "RESTART",
+                            "config_ready": True
+                        }]
+                    }
+                }, 
+                {
+                    "component_name": "virtual-network-function-manager", 
+                    "component_type": "RWTASKLET", 
+                    "rwtasklet": {
+                        "plugin_directory": "./usr/lib/rift/plugins/rwvnfmtasklet", 
+                        "plugin_name": "rwvnfmtasklet"
+                    }
+                }, 
+                {
+                    "component_name": "RW.Proc_7.virtual-network-service", 
+                    "component_type": "RWPROC", 
+                    "rwproc": {
+                        "tasklet": [{
+                            "name": "Start virtual-network-service for RW.Proc_7.virtual-network-service", 
+                            "component_name": "virtual-network-service", 
+                            "recovery_action": "RESTART",
+                            "config_ready": True
+                        }]
+                    }
+                }, 
+                {
+                    "component_name": "virtual-network-service", 
+                    "component_type": "RWTASKLET", 
+                    "rwtasklet": {
+                        "plugin_directory": "./usr/lib/rift/plugins/rwvnstasklet", 
+                        "plugin_name": "rwvnstasklet"
+                    }
+                }, 
+                {
+                    "component_name": "RW.Proc_8.nfvi-metrics-monitor", 
+                    "component_type": "RWPROC", 
+                    "rwproc": {
+                        "tasklet": [{
+                            "name": "Start nfvi-metrics-monitor for RW.Proc_8.nfvi-metrics-monitor", 
+                            "component_name": "nfvi-metrics-monitor", 
+                            "recovery_action": "RESTART",
+                            "config_ready": True
+                        }]
+                    }
+                }, 
+                {
+                    "component_name": "RW.MC.UI", 
+                    "component_type": "PROC", 
+                    "native_proc": {
+                        "exe_path": "./usr/share/rw.ui/skyquake/scripts/launch_ui.sh", 
+                    }
+                },
+                {
+                    "component_name": "RW.COMPOSER.UI",
+                    "component_type": "PROC", 
+                    "native_proc": {
+                        "exe_path": "./usr/share/composer/scripts/launch_composer.sh",
+                    }
+                },
+                {
+                    "component_name": "RW.Proc_9.Configuration-Manager", 
+                    "component_type": "RWPROC", 
+                    "rwproc": {
+                        "tasklet": [{
+                            "name": "Start Configuration-Manager for RW.Proc_9.Configuration-Manager", 
+                            "component_name": "Configuration-Manager", 
+                            "recovery_action": "RESTART",
+                            "config_ready": True
+                        }]
+                    }
+                }, 
+                {
+                    "component_name": "Configuration-Manager", 
+                    "component_type": "RWTASKLET", 
+                    "rwtasklet": {
+                        "plugin_directory": "./usr/lib/rift/plugins/rwconmantasklet", 
+                        "plugin_name": "rwconmantasklet"
+                    }
+                }, 
+                {
+                    "component_name": "RW.Proc_10.launchpad", 
+                    "component_type": "RWPROC", 
+                    "rwproc": {
+                        "tasklet": [{
+                            "name": "Start launchpad for RW.Proc_10.launchpad", 
+                            "component_name": "launchpad", 
+                            "recovery_action": "RESTART",
+                            "config_ready": True
+                        }]
+                    }
+                }, 
+                {
+                    "component_name": "launchpad", 
+                    "component_type": "RWTASKLET", 
+                    "rwtasklet": {
+                        "plugin_directory": "./usr/lib/rift/plugins/rwlaunchpad", 
+                        "plugin_name": "rwlaunchpad"
+                    }
+                }, 
+                {
+                    "component_name": "RW.Proc_11.Resource-Manager", 
+                    "component_type": "RWPROC", 
+                    "rwproc": {
+                        "tasklet": [{
+                            "name": "Start Resource-Manager for RW.Proc_11.Resource-Manager", 
+                            "component_name": "Resource-Manager", 
+                            "recovery_action": "RESTART",
+                            "config_ready": True
+                        }]
+                    }
+                }, 
+                {
+                    "component_name": "Resource-Manager", 
+                    "component_type": "RWTASKLET", 
+                    "rwtasklet": {
+                        "plugin_directory": "./usr/lib/rift/plugins/rwresmgrtasklet", 
+                        "plugin_name": "rwresmgrtasklet"
+                    }
+                }, 
+                {
+                    "component_name": "RW.uAgent", 
+                    "component_type": "RWTASKLET", 
+                    "rwtasklet": {
+                        "plugin_directory": "./usr/lib/rift/plugins/rwuagent-c", 
+                        "plugin_name": "rwuagent-c"
+                    }
+                }, 
+                {
+                    "component_name": "logd", 
+                    "component_type": "RWTASKLET", 
+                    "rwtasklet": {
+                        "plugin_directory": "./usr/lib/rift/plugins/rwlogd-c", 
+                        "plugin_name": "rwlogd-c"
+                    }
+                }, 
+                {
+                    "component_name": "msgbroker", 
+                    "component_type": "RWTASKLET", 
+                    "rwtasklet": {
+                        "plugin_directory": "./usr/lib/rift/plugins/rwmsgbroker-c", 
+                        "plugin_name": "rwmsgbroker-c"
+                    }
+                }, 
+                {
+                    "component_name": "dtsrouter", 
+                    "component_type": "RWTASKLET", 
+                    "rwtasklet": {
+                        "plugin_directory": "./usr/lib/rift/plugins/rwdtsrouter-c", 
+                        "plugin_name": "rwdtsrouter-c"
+                    }
+                }
+            ]
+        })
+        return manifest
+
+    def tearDown(self):
+        tasklist = { 'reaperd', 
+                     'rwlogd-report-c', 
+                     'launch_ui.sh' }
+        for proc in psutil.process_iter():
+          if proc.name() in tasklist:
+             print("killing", proc.name())
+             try:
+               proc.kill()
+             except:
+               print(proc.name(), "no longer exists")
+        self.loop.stop()
+        self.loop.close()
+
+
+class LaunchPadTest(LaunchPad):
+    """
+    DTS GI interface unittests
+
+    Note:  Each tests uses a list of asyncio.Events for staging through the
+    test.  These are required here because we are bring up each coroutine
+    ("tasklet") at the same time and are not implementing any re-try
+    mechanisms.  For instance, this is used in numerous tests to make sure that
+    a publisher is up and ready before the subscriber sends queries.  Such
+    event lists should not be used in production software.
+    """
+    @asyncio.coroutine
+    def inventory(self):
+        res_iter = yield from self.dts_mgmt.query_read('/rw-base:vcs/rw-base:info', flags=0)
+        for i in res_iter:
+           info_result = yield from i
+        components = info_result.result.components.component_info
+        recvd_list = {}
+        for component in components:
+            recvd_list[component.component_name] = (component.instance_id, 
+                         component.rwcomponent_parent, 
+                         component.component_type,
+                         component.state)
+        return recvd_list
+
+    @asyncio.coroutine
+    def issue_vcrash(self, component_type):
+#       critical_components = {'msgbroker', 'dtsrouter'}
+        critical_components = {'msgbroker', 'dtsrouter', 'RW.uAgent'}
+        comp_inventory = yield from self.inventory()
+        for component in comp_inventory:
+          if ((comp_inventory[component])[2] == component_type):
+              inst = (comp_inventory[component])[0]
+              if (component in critical_components):
+                  print(component, 'Marked as CRITICAL - Not restarting')
+              else:
+                  print('Crashing ', component_type,component)
+                  vcrash_input = rwvcs.VCrashInput(instance_name=component+'-'+str(inst))
+                  query_iter = yield from self.dts_mgmt.query_rpc( xpath="/rw-vcs:vcrash",
+                                                      flags=0, msg=vcrash_input)
+                  yield from asyncio.sleep(1, loop=self.loop)
+                  restarted_inventory = yield from self.inventory()
+                  self.assertTrue(restarted_inventory[component][3] != 'TO_RECOVER')
+
+    def test_launch_pad(self):
+        """
+        Verify the launchpad setup functions
+        The test will progress through stages defined by the events list:
+            0:  mission_control setup is brought up
+            2:  Tasklet/PROC/VM restarts tested to confirm recovery is proper
+        """
+
+        print("{{{{{{{{{{{{{{{{{{{{STARTING - mano recovery test")
+#       confd_host="127.0.0.1"
+
+        events = [asyncio.Event(loop=self.loop) for _ in range(2)]
+
+        @asyncio.coroutine
+        def sub():
+
+            tinfo = self.new_tinfo('sub')
+            self.dts_mgmt = rift.tasklets.DTS(tinfo, self.schema, self.loop)
+
+            # Sleep for DTS registrations to complete
+            print('.........................................................')
+            print('........SLEEPING 80 seconds for system to come up........')
+            yield from asyncio.sleep(80, loop=self.loop)
+            print('........RESUMING........')
+
+            @asyncio.coroutine
+            def issue_vstop(component,inst,flag=0):
+                vstop_input = rwvcs.VStopInput(instance_name=component+'-'+(str(inst))) 
+                query_iter = yield from self.dts_mgmt.query_rpc( xpath="/rw-vcs:vstop",
+                                    flags=flag, msg=vstop_input)
+                yield from asyncio.sleep(1, loop=self.loop)
+
+
+
+            @asyncio.coroutine
+            def issue_vstart(component, parent, recover=False):
+                vstart_input = rwvcs.VStartInput()
+                vstart_input.component_name = component
+                vstart_input.parent_instance = parent
+                vstart_input.recover = recover
+                query_iter = yield from self.dts_mgmt.query_rpc( xpath="/rw-vcs:vstart",
+                                                      flags=0, msg=vstart_input)
+                yield from asyncio.sleep(1, loop=self.loop)
+
+            @asyncio.coroutine
+            def issue_start_stop(comp_inventory, component_type):
+#               critical_components = {'msgbroker', 'dtsrouter'}
+                critical_components = {'msgbroker', 'dtsrouter', 'RW.uAgent'}
+                for component in comp_inventory:
+                  if ((comp_inventory[component])[2] == component_type):
+                      inst = (comp_inventory[component])[0]
+                      parent = (comp_inventory[component])[1]
+                      if (component in critical_components):
+                          print(component, 'Marked as CRITICAL - Not restarting')
+                      else:
+                          print('Stopping ', component_type,component)
+                          yield from issue_vstop(component,inst)
+                          restarted_inventory = yield from self.inventory()
+#                         self.assertEqual(restarted_inventory[component][3],'TO_RECOVER')
+                          print('Starting ',component_type,component)
+                          yield from issue_vstart(component, parent, recover=True)
+                          restarted_inventory = yield from self.inventory()
+                          self.assertTrue(restarted_inventory[component][3] != 'TO_RECOVER')
+
+            yield from asyncio.sleep(20, loop=self.loop)
+            comp_inventory = yield from self.inventory()
+            yield from issue_start_stop(comp_inventory, 'RWTASKLET')
+#           yield from issue_start_stop(comp_inventory, 'RWPROC')
+#           yield from self.issue_vcrash('RWTASKLET')
+
+            yield from asyncio.sleep(20, loop=self.loop)
+            restarted_inventory = yield from self.inventory()
+#           critical_components = {'msgbroker', 'dtsrouter', 'RW.uAgent'}
+            for comp in comp_inventory:
+                self.assertEqual(str(comp_inventory[comp]), str(restarted_inventory[comp])) 
+#               if (comp not in critical_components):
+#                   inst = (comp_inventory[comp])[0]
+#                   yield from issue_vstop(comp,inst)
+
+            events[1].set()
+
+        asyncio.ensure_future(sub(), loop=self.loop)
+        self.run_until(events[1].is_set, timeout=260)
+
+
+def main():
+    plugin_dir = os.path.join(os.environ["RIFT_INSTALL"], "usr/lib/rift/plugins")
+    if 'DTS_TEST_PUB_DIR' not in os.environ:
+        os.environ['DTS_TEST_PUB_DIR'] = os.path.join(plugin_dir, 'dtstestpub')
+
+    if 'RIFT_NO_SUDO_REAPER' not in os.environ:
+        os.environ['RIFT_NO_SUDO_REAPER'] = '1'
+
+    if 'MESSAGE_BROKER_DIR' not in os.environ:
+        os.environ['MESSAGE_BROKER_DIR'] = os.path.join(plugin_dir, 'rwmsgbroker-c')
+
+    if 'ROUTER_DIR' not in os.environ:
+        os.environ['ROUTER_DIR'] = os.path.join(plugin_dir, 'rwdtsrouter-c')
+
+    if 'RW_VAR_RIFT' not in os.environ:
+        os.environ['RW_VAR_RIFT'] = '1'
+    
+    if 'INSTALLDIR' in os.environ:
+        os.chdir(os.environ.get('INSTALLDIR')) 
+
+#   if 'RWMSG_BROKER_SHUNT' not in os.environ:
+#       os.environ['RWMSG_BROKER_SHUNT'] = '1'
+
+    if 'TEST_ENVIRON' not in os.environ:
+        os.environ['TEST_ENVIRON'] = '1'
+
+    if 'RW_MANIFEST' not in os.environ:
+        os.environ['RW_MANIFEST'] = os.path.join(install_dir, 'lptestmanifest.xml')
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-v', '--verbose', action='store_true')
+    args, _ = parser.parse_known_args()
+
+
+    runner = xmlrunner.XMLTestRunner(output=os.environ["RIFT_MODULE_TEST"])
+    unittest.main(testRunner=runner)
+
+if __name__ == '__main__':
+    main()
+
+# vim: sw=4