rwlaunchpad/test/launchpad_recovery

   1 #!/usr/bin/env python3
   2
   3 #
   4 #   Copyright 2016 RIFT.IO Inc
   5 #
   6 #   Licensed under the Apache License, Version 2.0 (the "License");
   7 #   you may not use this file except in compliance with the License.
   8 #   You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #   Unless required by applicable law or agreed to in writing, software
  13 #   distributed under the License is distributed on an "AS IS" BASIS,
  14 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #   See the License for the specific language governing permissions and
  16 #   limitations under the License.
  17 #
  18
  19 import argparse
  20 import asyncio
  21 import logging
  22 import os
  23 import sys
  24 import unittest
  25 import re
  26 import psutil
  27 import types
  28
  29 import xmlrunner
  30
  31 import gi
  32 gi.require_version('RwDtsToyTaskletYang', '1.0')
  33 gi.require_version('RwManifestYang', '1.0')
  34 gi.require_version('RwVcsYang', '1.0')
  35
  36 import gi.repository.RwManifestYang as rwmanifest
  37 import gi.repository.RwVcsYang as rwvcs
  38 import gi.repository.RwDtsToyTaskletYang as toyyang
  39 import gi.repository.RwYang as RwYang
  40 import rift.auto.session
  41 import rift.vcs.vcs
  42
  43 import rift.tasklets
  44 import rift.test.dts
  45
  46 if sys.version_info < (3, 4, 4):
  47     asyncio.ensure_future = asyncio.async
  48
  49 class LaunchPad(rift.test.dts.AbstractDTSTest):
  50     """
  51     DTS GI interface unittests
  52
  53     Note:  Each tests uses a list of asyncio.Events for staging through the
  54     test.  These are required here because we are bring up each coroutine
  55     ("tasklet") at the same time and are not implementing any re-try
  56     mechanisms.  For instance, this is used in numerous tests to make sure that
  57     a publisher is up and ready before the subscriber sends queries.  Such
  58     event lists should not be used in production software.
  59     """
  60     def setUp(self):
  61         """
  62         1. Creates an asyncio loop
  63         2. Triggers the hook configure_test
  64         """
  65         def scheduler_tick(self, *args):
  66             self.call_soon(self.stop)
  67             self.run_forever()
  68
  69         # Init params: loop & timers
  70         self.loop = asyncio.new_event_loop()
  71
  72         self.loop.scheduler_tick = types.MethodType(scheduler_tick, self.loop)
  73
  74         self.asyncio_timer = None
  75         self.stop_timer = None
  76         self.__class__.id_cnt += 1
  77         self.configure_test(self.loop, self.__class__.id_cnt)
  78
  79     @classmethod
  80     def configure_schema(cls):
  81         schema =  RwYang.Model.load_and_merge_schema(rwvcs.get_schema(), 'librwcal_yang_gen.so', 'Rwcal')
  82         cls.model = RwYang.Model.create_libyang()
  83         cls.model.load_schema_ypbc(schema)
  84         xml = cls.manifest.to_xml_v2(cls.model, 1)
  85         xml = re.sub('rw-manifest:', '', xml)
  86         xml = re.sub('<manifest xmlns:rw-manifest="http://riftio.com/ns/riftware-1.0/rw-manifest">', '<?xml version="1.0" ?>\n<manifest xmlns="http://riftio.com/ns/riftware-1.0/rw-manifest" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://riftio.com/ns/riftware-1.0/rw-manifest ./rw-manifest.xsd">', xml)
  87         xml = '\n'.join(xml.split('\n')[1:])
  88         with open('lptestmanifest.xml', 'w') as f:
  89            f.write(str(xml))
  90         f.close()
  91         return schema
  92
  93
  94     @classmethod
  95     def configure_manifest(cls):
  96         manifest = rwmanifest.Manifest()
  97         manifest.bootstrap_phase = rwmanifest.BootstrapPhase.from_dict({
  98             "rwmgmt": {
  99                 "northbound_listing": [ "platform_schema_listing.txt", "platform_mgmt_schema_listing.txt", "cli_launchpad_schema_listing.txt" ]
 100             },
 101             "rwtasklet": {
 102                 "plugin_name": "rwinit-c"
 103             },
 104             "rwtrace": {
 105                 "enable": True,
 106                 "level": 5,
 107             },
 108             "log": {
 109                 "enable": True,
 110                 "severity": 4,
 111                 "bootstrap_time": 30,
 112                 "console_severity": 4
 113             },
 114             "ip_addrs_list": [
 115                 {
 116                     "ip_addr": "127.0.0.1",
 117                 }
 118             ],
 119             "zookeeper": {
 120                 "master_ip": "127.0.0.1",
 121                 "unique_ports": False,
 122                 "zake": False
 123             },
 124             "serf": {
 125                 "start": True
 126             },
 127             "rwvm": {
 128                 "instances": [
 129                     {
 130                         "component_name": "msgbroker",
 131                         "config_ready": True
 132                     },
 133                     {
 134                         "component_name": "dtsrouter",
 135                         "config_ready": True
 136                     }
 137                 ]
 138             },
 139 #           "rwsecurity": {
 140 #               "use_ssl": True,
 141 #               "cert": "/net/mahi/localdisk/kelayath/ws/coreha/etc/ssl/current.cert",
 142 #               "key": "/net/mahi/localdisk/kelayath/ws/coreha/etc/ssl/current.key"
 143 #           }
 144         })
 145         manifest.init_phase = rwmanifest.InitPhase.from_dict({
 146             "environment": {
 147                 "python_variable": [
 148                     "vm_ip_address = '127.0.0.1'",
 149                     "rw_component_name = 'vm-launchpad'",
 150                     "instance_id = 1",
 151                     "component_type = 'rwvm'",
 152                 ],
 153                 "component_name": "$python(rw_component_name)",
 154                 "instance_id": "$python(instance_id)",
 155                 "component_type": "$python(rw_component_type)"
 156             },
 157             "settings": {
 158                 "rwmsg": {
 159                     "multi_broker": {
 160                         "enable": False
 161                     }
 162                 },
 163                 "rwdtsrouter": {
 164                     "multi_dtsrouter": {
 165                         "enable": True
 166                     }
 167                 },
 168                 "rwvcs": {
 169                     "collapse_each_rwvm": False,
 170                     "collapse_each_rwprocess": False
 171                 }
 172             }
 173         })
 174         manifest.inventory = rwmanifest.Inventory.from_dict({
 175             "component": [
 176                 {
 177                     "component_name": "master",
 178                     "component_type": "RWCOLLECTION",
 179                     "rwcollection": {
 180                         "collection_type": "rwcolony",
 181                         "event_list": {
 182                             "event": [{
 183                                 "name": "onentry",
 184                                 "action": [{
 185                                     "name": "Start vm-launchpad for master",
 186                                     "start": {
 187                                         "python_variable": ["vm_ip_address = '127.0.0.1'"],
 188                                         "component_name": "vm-launchpad",
 189                                         "instance_id": "1",
 190                                         "config_ready": True
 191                                     }
 192                                 }]
 193                             }]
 194                         }
 195                     }
 196                 },
 197                 {
 198                     "component_name": "vm-launchpad",
 199                     "component_type": "RWVM",
 200                     "rwvm": {
 201                         "leader": True,
 202                         "event_list": {
 203                             "event": [{
 204                                 "name": "onentry",
 205                                 "action": [
 206                                     {
 207                                         "name": "Start the master",
 208                                         "start": {
 209                                             "component_name": "master",
 210                                             "recovery_action": "RESTART",
 211                                             "config_ready": True
 212                                         }
 213                                     },
 214                                     {
 215                                         "name": "Start the RW.Proc_1.Restconf",
 216                                         "start": {
 217                                             "component_name": "RW.Proc_1.Restconf",
 218                                             "recovery_action": "RESTART",
 219                                             "config_ready": True
 220                                         }
 221                                     },
 222                                     {
 223                                         "name": "Start the RW.Proc_3.CalProxy",
 224                                         "start": {
 225                                             "component_name": "RW.Proc_3.CalProxy",
 226                                             "recovery_action": "RESTART",
 227                                             "config_ready": True
 228                                         }
 229                                     },
 230                                     {
 231                                         "name": "Start the RW.Proc_4.nfvi-metrics-monitor",
 232                                         "start": {
 233                                             "component_name": "RW.Proc_4.nfvi-metrics-monitor",
 234                                             "recovery_action": "RESTART",
 235                                             "config_ready": True
 236                                         }
 237                                     },
 238                                     {
 239                                         "name": "Start the RW.Proc_5.network-services-manager",
 240                                         "start": {
 241                                             "component_name": "RW.Proc_5.network-services-manager",
 242                                             "recovery_action": "RESTART",
 243                                             "config_ready": True
 244                                         }
 245                                     },
 246                                     {
 247                                         "name": "Start the RW.Proc_6.virtual-network-function-manager",
 248                                         "start": {
 249                                             "component_name": "RW.Proc_6.virtual-network-function-manager",
 250                                             "recovery_action": "RESTART",
 251                                             "config_ready": True
 252                                         }
 253                                     },
 254                                     {
 255                                         "name": "Start the RW.Proc_7.virtual-network-service",
 256                                         "start": {
 257                                             "component_name": "RW.Proc_7.virtual-network-service",
 258                                             "recovery_action": "RESTART",
 259                                             "config_ready": True
 260                                         }
 261                                     },
 262                                     {
 263                                         "name": "Start the RW.Proc_8.nfvi-metrics-monitor",
 264                                         "start": {
 265                                             "component_name": "RW.Proc_8.nfvi-metrics-monitor",
 266                                             "recovery_action": "RESTART",
 267                                             "config_ready": True
 268                                         }
 269                                     },
 270                                     {
 271                                         "name": "Start the RW.MC.UI",
 272                                         "start": {
 273                                             "component_name": "RW.MC.UI",
 274                                             "recovery_action": "RESTART",
 275                                             "config_ready": True
 276                                         }
 277                                     },
 278 #                                   {
 279 #                                       "name": "Start the RW.COMPOSER.UI",
 280 #                                       "start": {
 281 #                                           "component_name": "RW.COMPOSER.UI",
 282 #                                           "config_ready": True
 283 #                                       }
 284 #                                   },
 285                                     {
 286                                         "name": "Start the RW.Proc_10.launchpad",
 287                                         "start": {
 288                                             "component_name": "RW.Proc_10.launchpad",
 289                                             "recovery_action": "RESTART",
 290                                             "config_ready": True
 291                                         }
 292                                     },
 293                                     {
 294                                         "name": "Start the RW.Proc_11.Resource-Manager",
 295                                         "start": {
 296                                             "component_name": "RW.Proc_11.Resource-Manager",
 297                                             "recovery_action": "RESTART",
 298                                             "config_ready": True
 299                                         }
 300                                     },
 301                                     {
 302                                         "name": "Start the RW.uAgent",
 303                                         "start": {
 304                                             "python_variable": ["cmdargs_str = '--confd-proto AF_INET --confd-ip 127.0.0.1'"],
 305                                             "component_name": "RW.uAgent",
 306                                             "recovery_action": "RESTART",
 307                                             "config_ready": True
 308                                         }
 309                                     },
 310                                     {
 311                                         "name": "Start the logd",
 312                                         "start": {
 313                                             "component_name": "logd",
 314                                             "recovery_action": "RESTART",
 315                                             "config_ready": True
 316                                         }
 317                                     }
 318                                 ]
 319                             }]
 320                         }
 321                     }
 322                 },
 323 #               {
 324 #                   "component_name": "RW.CLI",
 325 #                   "component_type": "PROC",
 326 #                   "native_proc": {
 327 #                       "exe_path": "./usr/bin/rwcli",
 328 #                       "args": "--netconf_host 127.0.0.1 --netconf_port 2022 --schema_listing cli_launchpad_schema_listing.txt",
 329 #                   }
 330 #               },
 331                 {
 332                     "component_name": "RW.Proc_1.Restconf",
 333                     "component_type": "RWPROC",
 334                     "rwproc": {
 335                         "tasklet": [{
 336                             "name": "Start RW.Restconf for RW.Proc_1.Restconf",
 337                             "component_name": "RW.Restconf",
 338                             "recovery_action": "RESTART",
 339                             "config_ready": True
 340                         }]
 341                     }
 342                 },
 343                 {
 344                     "component_name": "RW.Restconf",
 345                     "component_type": "RWTASKLET",
 346                     "rwtasklet": {
 347                         "plugin_directory": "./usr/lib/rift/plugins/restconf",
 348                         "plugin_name": "restconf"
 349                     }
 350                 },
 351                 {
 352                     "component_name": "RW.Proc_3.CalProxy",
 353                     "component_type": "RWPROC",
 354                     "rwproc": {
 355                         "tasklet": [{
 356                             "name": "Start RW.CalProxy for RW.Proc_3.CalProxy",
 357                             "component_name": "RW.CalProxy",
 358                             "recovery_action": "RESTART",
 359                             "config_ready": True
 360                         }]
 361                     }
 362                 },
 363                 {
 364                     "component_name": "RW.CalProxy",
 365                     "component_type": "RWTASKLET",
 366                     "rwtasklet": {
 367                         "plugin_directory": "./usr/lib/rift/plugins/rwcalproxytasklet",
 368                         "plugin_name": "rwcalproxytasklet"
 369                     }
 370                 },
 371                 {
 372                     "component_name": "RW.Proc_4.nfvi-metrics-monitor",
 373                     "component_type": "RWPROC",
 374                     "rwproc": {
 375                         "tasklet": [{
 376                             "name": "Start nfvi-metrics-monitor for RW.Proc_4.nfvi-metrics-monitor",
 377                             "component_name": "nfvi-metrics-monitor",
 378                             "recovery_action": "RESTART",
 379                             "config_ready": True
 380                         }]
 381                     }
 382                 },
 383                 {
 384                     "component_name": "nfvi-metrics-monitor",
 385                     "component_type": "RWTASKLET",
 386                     "rwtasklet": {
 387                         "plugin_directory": "./usr/lib/rift/plugins/rwmonitor",
 388                         "plugin_name": "rwmonitor"
 389                     }
 390                 },
 391                 {
 392                     "component_name": "RW.Proc_5.network-services-manager",
 393                     "component_type": "RWPROC",
 394                     "rwproc": {
 395                         "tasklet": [{
 396                             "name": "Start network-services-manager for RW.Proc_5.network-services-manager",
 397                             "component_name": "network-services-manager",
 398                             "recovery_action": "RESTART",
 399                             "config_ready": True
 400                         }]
 401                     }
 402                 },
 403                 {
 404                     "component_name": "network-services-manager",
 405                     "component_type": "RWTASKLET",
 406                     "rwtasklet": {
 407                         "plugin_directory": "./usr/lib/rift/plugins/rwnsmtasklet",
 408                         "plugin_name": "rwnsmtasklet"
 409                     }
 410                 },
 411                 {
 412                     "component_name": "RW.Proc_6.virtual-network-function-manager",
 413                     "component_type": "RWPROC",
 414                     "rwproc": {
 415                         "tasklet": [{
 416                             "name": "Start virtual-network-function-manager for RW.Proc_6.virtual-network-function-manager",
 417                             "component_name": "virtual-network-function-manager",
 418                             "recovery_action": "RESTART",
 419                             "config_ready": True
 420                         }]
 421                     }
 422                 },
 423                 {
 424                     "component_name": "virtual-network-function-manager",
 425                     "component_type": "RWTASKLET",
 426                     "rwtasklet": {
 427                         "plugin_directory": "./usr/lib/rift/plugins/rwvnfmtasklet",
 428                         "plugin_name": "rwvnfmtasklet"
 429                     }
 430                 },
 431                 {
 432                     "component_name": "RW.Proc_7.virtual-network-service",
 433                     "component_type": "RWPROC",
 434                     "rwproc": {
 435                         "tasklet": [{
 436                             "name": "Start virtual-network-service for RW.Proc_7.virtual-network-service",
 437                             "component_name": "virtual-network-service",
 438                             "recovery_action": "RESTART",
 439                             "config_ready": True
 440                         }]
 441                     }
 442                 },
 443                 {
 444                     "component_name": "virtual-network-service",
 445                     "component_type": "RWTASKLET",
 446                     "rwtasklet": {
 447                         "plugin_directory": "./usr/lib/rift/plugins/rwvnstasklet",
 448                         "plugin_name": "rwvnstasklet"
 449                     }
 450                 },
 451                 {
 452                     "component_name": "RW.Proc_8.nfvi-metrics-monitor",
 453                     "component_type": "RWPROC",
 454                     "rwproc": {
 455                         "tasklet": [{
 456                             "name": "Start nfvi-metrics-monitor for RW.Proc_8.nfvi-metrics-monitor",
 457                             "component_name": "nfvi-metrics-monitor",
 458                             "recovery_action": "RESTART",
 459                             "config_ready": True
 460                         }]
 461                     }
 462                 },
 463                 {
 464                     "component_name": "RW.MC.UI",
 465                     "component_type": "PROC",
 466                     "native_proc": {
 467                         "exe_path": "./usr/share/rw.ui/skyquake/scripts/launch_ui.sh",
 468                     }
 469                 },
 470                 {
 471                     "component_name": "RW.COMPOSER.UI",
 472                     "component_type": "PROC",
 473                     "native_proc": {
 474                         "exe_path": "./usr/share/composer/scripts/launch_composer.sh",
 475                     }
 476                 },
 477                 {
 478                     "component_name": "RW.Proc_9.Configuration-Manager",
 479                     "component_type": "RWPROC",
 480                     "rwproc": {
 481                         "tasklet": [{
 482                             "name": "Start Configuration-Manager for RW.Proc_9.Configuration-Manager",
 483                             "component_name": "Configuration-Manager",
 484                             "recovery_action": "RESTART",
 485                             "config_ready": True
 486                         }]
 487                     }
 488                 },
 489                 {
 490                     "component_name": "Configuration-Manager",
 491                     "component_type": "RWTASKLET",
 492                     "rwtasklet": {
 493                         "plugin_directory": "./usr/lib/rift/plugins/rwconmantasklet",
 494                         "plugin_name": "rwconmantasklet"
 495                     }
 496                 },
 497                 {
 498                     "component_name": "RW.Proc_10.launchpad",
 499                     "component_type": "RWPROC",
 500                     "rwproc": {
 501                         "tasklet": [{
 502                             "name": "Start launchpad for RW.Proc_10.launchpad",
 503                             "component_name": "launchpad",
 504                             "recovery_action": "RESTART",
 505                             "config_ready": True
 506                         }]
 507                     }
 508                 },
 509                 {
 510                     "component_name": "launchpad",
 511                     "component_type": "RWTASKLET",
 512                     "rwtasklet": {
 513                         "plugin_directory": "./usr/lib/rift/plugins/rwlaunchpad",
 514                         "plugin_name": "rwlaunchpad"
 515                     }
 516                 },
 517                 {
 518                     "component_name": "RW.Proc_11.Resource-Manager",
 519                     "component_type": "RWPROC",
 520                     "rwproc": {
 521                         "tasklet": [{
 522                             "name": "Start Resource-Manager for RW.Proc_11.Resource-Manager",
 523                             "component_name": "Resource-Manager",
 524                             "recovery_action": "RESTART",
 525                             "config_ready": True
 526                         }]
 527                     }
 528                 },
 529                 {
 530                     "component_name": "Resource-Manager",
 531                     "component_type": "RWTASKLET",
 532                     "rwtasklet": {
 533                         "plugin_directory": "./usr/lib/rift/plugins/rwresmgrtasklet",
 534                         "plugin_name": "rwresmgrtasklet"
 535                     }
 536                 },
 537                 {
 538                     "component_name": "RW.uAgent",
 539                     "component_type": "RWTASKLET",
 540                     "rwtasklet": {
 541                         "plugin_directory": "./usr/lib/rift/plugins/rwuagent-c",
 542                         "plugin_name": "rwuagent-c"
 543                     }
 544                 },
 545                 {
 546                     "component_name": "logd",
 547                     "component_type": "RWTASKLET",
 548                     "rwtasklet": {
 549                         "plugin_directory": "./usr/lib/rift/plugins/rwlogd-c",
 550                         "plugin_name": "rwlogd-c"
 551                     }
 552                 },
 553                 {
 554                     "component_name": "msgbroker",
 555                     "component_type": "RWTASKLET",
 556                     "rwtasklet": {
 557                         "plugin_directory": "./usr/lib/rift/plugins/rwmsgbroker-c",
 558                         "plugin_name": "rwmsgbroker-c"
 559                     }
 560                 },
 561                 {
 562                     "component_name": "dtsrouter",
 563                     "component_type": "RWTASKLET",
 564                     "rwtasklet": {
 565                         "plugin_directory": "./usr/lib/rift/plugins/rwdtsrouter-c",
 566                         "plugin_name": "rwdtsrouter-c"
 567                     }
 568                 }
 569             ]
 570         })
 571         return manifest
 572
 573     def tearDown(self):
 574         tasklist = { 'reaperd',
 575                      'rwlogd-report-c',
 576                      'launch_ui.sh' }
 577         for proc in psutil.process_iter():
 578           if proc.name() in tasklist:
 579              print("killing", proc.name())
 580              try:
 581                proc.kill()
 582              except:
 583                print(proc.name(), "no longer exists")
 584         self.loop.stop()
 585         self.loop.close()
 586
 587
 588 class LaunchPadTest(LaunchPad):
 589     """
 590     DTS GI interface unittests
 591
 592     Note:  Each tests uses a list of asyncio.Events for staging through the
 593     test.  These are required here because we are bring up each coroutine
 594     ("tasklet") at the same time and are not implementing any re-try
 595     mechanisms.  For instance, this is used in numerous tests to make sure that
 596     a publisher is up and ready before the subscriber sends queries.  Such
 597     event lists should not be used in production software.
 598     """
 599     @asyncio.coroutine
 600     def inventory(self):
 601         res_iter = yield from self.dts_mgmt.query_read('/rw-base:vcs/rw-base:info', flags=0)
 602         for i in res_iter:
 603            info_result = yield from i
 604         components = info_result.result.components.component_info
 605         recvd_list = {}
 606         for component in components:
 607             recvd_list[component.component_name] = (component.instance_id,
 608                          component.rwcomponent_parent,
 609                          component.component_type,
 610                          component.state)
 611         return recvd_list
 612
 613     @asyncio.coroutine
 614     def issue_vcrash(self, component_type):
 615 #       critical_components = {'msgbroker', 'dtsrouter'}
 616         critical_components = {'msgbroker', 'dtsrouter', 'RW.uAgent'}
 617         comp_inventory = yield from self.inventory()
 618         for component in comp_inventory:
 619           if ((comp_inventory[component])[2] == component_type):
 620               inst = (comp_inventory[component])[0]
 621               if (component in critical_components):
 622                   print(component, 'Marked as CRITICAL - Not restarting')
 623               else:
 624                   print('Crashing ', component_type,component)
 625                   vcrash_input = rwvcs.VCrashInput(instance_name=component+'-'+str(inst))
 626                   query_iter = yield from self.dts_mgmt.query_rpc( xpath="/rw-vcs:vcrash",
 627                                                       flags=0, msg=vcrash_input)
 628                   yield from asyncio.sleep(1, loop=self.loop)
 629                   restarted_inventory = yield from self.inventory()
 630                   self.assertTrue(restarted_inventory[component][3] != 'TO_RECOVER')
 631
 632     def test_launch_pad(self):
 633         """
 634         Verify the launchpad setup functions
 635         The test will progress through stages defined by the events list:
 636             0:  mission_control setup is brought up
 637             2:  Tasklet/PROC/VM restarts tested to confirm recovery is proper
 638         """
 639
 640         print("{{{{{{{{{{{{{{{{{{{{STARTING - mano recovery test")
 641 #       confd_host="127.0.0.1"
 642
 643         events = [asyncio.Event(loop=self.loop) for _ in range(2)]
 644
 645         @asyncio.coroutine
 646         def sub():
 647
 648             tinfo = self.new_tinfo('sub')
 649             self.dts_mgmt = rift.tasklets.DTS(tinfo, self.schema, self.loop)
 650
 651             # Sleep for DTS registrations to complete
 652             print('.........................................................')
 653             print('........SLEEPING 80 seconds for system to come up........')
 654             yield from asyncio.sleep(80, loop=self.loop)
 655             print('........RESUMING........')
 656
 657             @asyncio.coroutine
 658             def issue_vstop(component,inst,flag=0):
 659                 vstop_input = rwvcs.VStopInput(instance_name=component+'-'+(str(inst)))
 660                 query_iter = yield from self.dts_mgmt.query_rpc( xpath="/rw-vcs:vstop",
 661                                     flags=flag, msg=vstop_input)
 662                 yield from asyncio.sleep(1, loop=self.loop)
 663
 664
 665
 666             @asyncio.coroutine
 667             def issue_vstart(component, parent, recover=False):
 668                 vstart_input = rwvcs.VStartInput()
 669                 vstart_input.component_name = component
 670                 vstart_input.parent_instance = parent
 671                 vstart_input.recover = recover
 672                 query_iter = yield from self.dts_mgmt.query_rpc( xpath="/rw-vcs:vstart",
 673                                                       flags=0, msg=vstart_input)
 674                 yield from asyncio.sleep(1, loop=self.loop)
 675
 676             @asyncio.coroutine
 677             def issue_start_stop(comp_inventory, component_type):
 678 #               critical_components = {'msgbroker', 'dtsrouter'}
 679                 critical_components = {'msgbroker', 'dtsrouter', 'RW.uAgent'}
 680                 for component in comp_inventory:
 681                   if ((comp_inventory[component])[2] == component_type):
 682                       inst = (comp_inventory[component])[0]
 683                       parent = (comp_inventory[component])[1]
 684                       if (component in critical_components):
 685                           print(component, 'Marked as CRITICAL - Not restarting')
 686                       else:
 687                           print('Stopping ', component_type,component)
 688                           yield from issue_vstop(component,inst)
 689                           restarted_inventory = yield from self.inventory()
 690 #                         self.assertEqual(restarted_inventory[component][3],'TO_RECOVER')
 691                           print('Starting ',component_type,component)
 692                           yield from issue_vstart(component, parent, recover=True)
 693                           restarted_inventory = yield from self.inventory()
 694                           self.assertTrue(restarted_inventory[component][3] != 'TO_RECOVER')
 695
 696             yield from asyncio.sleep(20, loop=self.loop)
 697             comp_inventory = yield from self.inventory()
 698             yield from issue_start_stop(comp_inventory, 'RWTASKLET')
 699 #           yield from issue_start_stop(comp_inventory, 'RWPROC')
 700 #           yield from self.issue_vcrash('RWTASKLET')
 701
 702             yield from asyncio.sleep(20, loop=self.loop)
 703             restarted_inventory = yield from self.inventory()
 704 #           critical_components = {'msgbroker', 'dtsrouter', 'RW.uAgent'}
 705             for comp in comp_inventory:
 706                 self.assertEqual(str(comp_inventory[comp]), str(restarted_inventory[comp]))
 707 #               if (comp not in critical_components):
 708 #                   inst = (comp_inventory[comp])[0]
 709 #                   yield from issue_vstop(comp,inst)
 710
 711             events[1].set()
 712
 713         asyncio.ensure_future(sub(), loop=self.loop)
 714         self.run_until(events[1].is_set, timeout=260)
 715
 716
 717 def main():
 718     plugin_dir = os.path.join(os.environ["RIFT_INSTALL"], "usr/lib/rift/plugins")
 719     if 'DTS_TEST_PUB_DIR' not in os.environ:
 720         os.environ['DTS_TEST_PUB_DIR'] = os.path.join(plugin_dir, 'dtstestpub')
 721
 722     if 'RIFT_NO_SUDO_REAPER' not in os.environ:
 723         os.environ['RIFT_NO_SUDO_REAPER'] = '1'
 724
 725     if 'MESSAGE_BROKER_DIR' not in os.environ:
 726         os.environ['MESSAGE_BROKER_DIR'] = os.path.join(plugin_dir, 'rwmsgbroker-c')
 727
 728     if 'ROUTER_DIR' not in os.environ:
 729         os.environ['ROUTER_DIR'] = os.path.join(plugin_dir, 'rwdtsrouter-c')
 730
 731     if 'RW_VAR_RIFT' not in os.environ:
 732         os.environ['RW_VAR_RIFT'] = '1'
 733
 734     if 'INSTALLDIR' in os.environ:
 735         os.chdir(os.environ.get('INSTALLDIR'))
 736
 737 #   if 'RWMSG_BROKER_SHUNT' not in os.environ:
 738 #       os.environ['RWMSG_BROKER_SHUNT'] = '1'
 739
 740     if 'TEST_ENVIRON' not in os.environ:
 741         os.environ['TEST_ENVIRON'] = '1'
 742
 743     if 'RW_MANIFEST' not in os.environ:
 744         os.environ['RW_MANIFEST'] = os.path.join(install_dir, 'lptestmanifest.xml')
 745
 746     parser = argparse.ArgumentParser()
 747     parser.add_argument('-v', '--verbose', action='store_true')
 748     args, _ = parser.parse_known_args()
 749
 750
 751     runner = xmlrunner.XMLTestRunner(output=os.environ["RIFT_MODULE_TEST"])
 752     unittest.main(testRunner=runner)
 753
 754 if __name__ == '__main__':
 755     main()
 756
 757 # vim: sw=4