rwlaunchpad/test/launchpad_recovery

   1 #!/usr/bin/env python3
   2
   3 #
   4 #   Copyright 2016 RIFT.IO Inc
   5 #
   6 #   Licensed under the Apache License, Version 2.0 (the "License");
   7 #   you may not use this file except in compliance with the License.
   8 #   You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #   Unless required by applicable law or agreed to in writing, software
  13 #   distributed under the License is distributed on an "AS IS" BASIS,
  14 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #   See the License for the specific language governing permissions and
  16 #   limitations under the License.
  17 #
  18
  19 import argparse
  20 import asyncio
  21 import logging
  22 import os
  23 import sys
  24 import unittest
  25 import re
  26 import psutil
  27 import types
  28
  29 import xmlrunner
  30
  31 import gi
  32 gi.require_version('RwDtsToyTaskletYang', '1.0')
  33 gi.require_version('RwManifestYang', '1.0')
  34 gi.require_version('RwVcsYang', '1.0')
  35
  36 import gi.repository.RwManifestYang as rwmanifest
  37 import gi.repository.RwVcsYang as rwvcs
  38 import gi.repository.RwDtsToyTaskletYang as toyyang
  39 import gi.repository.RwYang as RwYang
  40 import rift.auto.session
  41 import rift.vcs.vcs
  42
  43 import rift.tasklets
  44 import rift.test.dts
  45
  46 if sys.version_info < (3, 4, 4):
  47     asyncio.ensure_future = asyncio.async
  48
  49 class LaunchPad(rift.test.dts.AbstractDTSTest):
  50     """
  51     DTS GI interface unittests
  52
  53     Note:  Each tests uses a list of asyncio.Events for staging through the
  54     test.  These are required here because we are bring up each coroutine
  55     ("tasklet") at the same time and are not implementing any re-try
  56     mechanisms.  For instance, this is used in numerous tests to make sure that
  57     a publisher is up and ready before the subscriber sends queries.  Such
  58     event lists should not be used in production software.
  59     """
  60     def setUp(self):
  61         """
  62         1. Creates an asyncio loop
  63         2. Triggers the hook configure_test
  64         """
  65         def scheduler_tick(self, *args):
  66             self.call_soon(self.stop)
  67             self.run_forever()
  68
  69         # Init params: loop & timers
  70         self.loop = asyncio.new_event_loop()
  71
  72         self.loop.scheduler_tick = types.MethodType(scheduler_tick, self.loop)
  73
  74         self.asyncio_timer = None
  75         self.stop_timer = None
  76         self.__class__.id_cnt += 1
  77         self.configure_test(self.loop, self.__class__.id_cnt)
  78
  79     @classmethod
  80     def configure_schema(cls):
  81         schema =  RwYang.Model.load_and_merge_schema(rwvcs.get_schema(), 'librwcal_yang_gen.so', 'Rwcal')
  82         cls.model = RwYang.Model.create_libncx()
  83         cls.model.load_schema_ypbc(schema)
  84         xml = cls.manifest.to_xml_v2(cls.model, 1)
  85         xml = re.sub('rw-manifest:', '', xml)
  86         xml = re.sub('<manifest xmlns:rw-manifest="http://riftio.com/ns/riftware-1.0/rw-manifest">', '<?xml version="1.0" ?>\n<manifest xmlns="http://riftio.com/ns/riftware-1.0/rw-manifest" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://riftio.com/ns/riftware-1.0/rw-manifest ./rw-manifest.xsd">', xml)
  87         xml = '\n'.join(xml.split('\n')[1:])
  88         with open('lptestmanifest.xml', 'w') as f:
  89            f.write(str(xml))
  90         f.close()
  91         return schema
  92
  93
  94     @classmethod
  95     def configure_manifest(cls):
  96         manifest = rwmanifest.Manifest()
  97         manifest.bootstrap_phase = rwmanifest.BootstrapPhase.from_dict({
  98             "rwmgmt": {
  99                 "northbound_listing": [ "cli_launchpad_schema_listing.txt" ]
 100             },
 101             "rwtasklet": {
 102                 "plugin_name": "rwinit-c"
 103             },
 104             "rwtrace": {
 105                 "enable": True,
 106                 "level": 5,
 107             },
 108             "log": {
 109                 "enable": True,
 110                 "severity": 4,
 111                 "bootstrap_time": 30,
 112                 "console_severity": 4
 113             },
 114             "ip_addrs_list": [
 115                 {
 116                     "ip_addr": "127.0.0.1",
 117                 }
 118             ],
 119             "zookeeper": {
 120                 "master_ip": "127.0.0.1",
 121                 "unique_ports": False,
 122                 "zake": False
 123             },
 124             "serf": {
 125                 "start": True
 126             },
 127             "rwvm": {
 128                 "instances": [
 129                     {
 130                         "component_name": "msgbroker",
 131                         "config_ready": True
 132                     },
 133                     {
 134                         "component_name": "dtsrouter",
 135                         "config_ready": True
 136                     }
 137                 ]
 138             },
 139 #           "rwsecurity": {
 140 #               "use_ssl": True,
 141 #               "cert": "/net/mahi/localdisk/kelayath/ws/coreha/etc/ssl/current.cert",
 142 #               "key": "/net/mahi/localdisk/kelayath/ws/coreha/etc/ssl/current.key"
 143 #           }
 144         })
 145         manifest.init_phase = rwmanifest.InitPhase.from_dict({
 146             "environment": {
 147                 "python_variable": [
 148                     "vm_ip_address = '127.0.0.1'",
 149                     "rw_component_name = 'vm-launchpad'",
 150                     "instance_id = 1",
 151                     "component_type = 'rwvm'",
 152                 ],
 153                 "component_name": "$python(rw_component_name)",
 154                 "instance_id": "$python(instance_id)",
 155                 "component_type": "$python(rw_component_type)"
 156             },
 157             "settings": {
 158                 "rwmsg": {
 159                     "multi_broker": {
 160                         "enable": False
 161                     }
 162                 },
 163                 "rwdtsrouter": {
 164                     "multi_dtsrouter": {
 165                         "enable": True
 166                     }
 167                 },
 168                 "rwvcs": {
 169                     "collapse_each_rwvm": False,
 170                     "collapse_each_rwprocess": False
 171                 }
 172             }
 173         })
 174         manifest.inventory = rwmanifest.Inventory.from_dict({
 175             "component": [
 176                 {
 177                     "component_name": "master",
 178                     "component_type": "RWCOLLECTION",
 179                     "rwcollection": {
 180                         "collection_type": "rwcolony",
 181                         "event_list": {
 182                             "event": [{
 183                                 "name": "onentry",
 184                                 "action": [{
 185                                     "name": "Start vm-launchpad for master",
 186                                     "start": {
 187                                         "python_variable": ["vm_ip_address = '127.0.0.1'"],
 188                                         "component_name": "vm-launchpad",
 189                                         "instance_id": "1",
 190                                         "config_ready": True
 191                                     }
 192                                 }]
 193                             }]
 194                         }
 195                     }
 196                 },
 197                 {
 198                     "component_name": "vm-launchpad",
 199                     "component_type": "RWVM",
 200                     "rwvm": {
 201                         "leader": True,
 202                         "event_list": {
 203                             "event": [{
 204                                 "name": "onentry",
 205                                 "action": [
 206                                     {
 207                                         "name": "Start the master",
 208                                         "start": {
 209                                             "component_name": "master",
 210                                             "recovery_action": "RESTART",
 211                                             "config_ready": True
 212                                         }
 213                                     },
 214 #                                   {
 215 #                                       "name": "Start the RW.CLI",
 216 #                                       "start": {
 217 #                                           "component_name": "RW.CLI",
 218 #                                           "recovery_action": "RESTART",
 219 #                                           "config_ready": True
 220 #                                       }
 221 #                                   },
 222                                     {
 223                                         "name": "Start the RW.Proc_1.Restconf",
 224                                         "start": {
 225                                             "component_name": "RW.Proc_1.Restconf",
 226                                             "recovery_action": "RESTART",
 227                                             "config_ready": True
 228                                         }
 229                                     },
 230 #                                   {
 231 #                                       "name": "Start the RW.Proc_2.RestPortForward",
 232 #                                       "start": {
 233 #                                           "component_name": "RW.Proc_2.RestPortForward",
 234 #                                           "recovery_action": "RESTART",
 235 #                                           "config_ready": True
 236 #                                       }
 237 #                                   },
 238                                     {
 239                                         "name": "Start the RW.Proc_3.CalProxy",
 240                                         "start": {
 241                                             "component_name": "RW.Proc_3.CalProxy",
 242                                             "recovery_action": "RESTART",
 243                                             "config_ready": True
 244                                         }
 245                                     },
 246                                     {
 247                                         "name": "Start the RW.Proc_4.nfvi-metrics-monitor",
 248                                         "start": {
 249                                             "component_name": "RW.Proc_4.nfvi-metrics-monitor",
 250                                             "recovery_action": "RESTART",
 251                                             "config_ready": True
 252                                         }
 253                                     },
 254                                     {
 255                                         "name": "Start the RW.Proc_5.network-services-manager",
 256                                         "start": {
 257                                             "component_name": "RW.Proc_5.network-services-manager",
 258                                             "recovery_action": "RESTART",
 259                                             "config_ready": True
 260                                         }
 261                                     },
 262                                     {
 263                                         "name": "Start the RW.Proc_6.virtual-network-function-manager",
 264                                         "start": {
 265                                             "component_name": "RW.Proc_6.virtual-network-function-manager",
 266                                             "recovery_action": "RESTART",
 267                                             "config_ready": True
 268                                         }
 269                                     },
 270                                     {
 271                                         "name": "Start the RW.Proc_7.virtual-network-service",
 272                                         "start": {
 273                                             "component_name": "RW.Proc_7.virtual-network-service",
 274                                             "recovery_action": "RESTART",
 275                                             "config_ready": True
 276                                         }
 277                                     },
 278                                     {
 279                                         "name": "Start the RW.Proc_8.nfvi-metrics-monitor",
 280                                         "start": {
 281                                             "component_name": "RW.Proc_8.nfvi-metrics-monitor",
 282                                             "recovery_action": "RESTART",
 283                                             "config_ready": True
 284                                         }
 285                                     },
 286                                     {
 287                                         "name": "Start the RW.MC.UI",
 288                                         "start": {
 289                                             "component_name": "RW.MC.UI",
 290                                             "recovery_action": "RESTART",
 291                                             "config_ready": True
 292                                         }
 293                                     },
 294 #                                   {
 295 #                                       "name": "Start the RW.COMPOSER.UI",
 296 #                                       "start": {
 297 #                                           "component_name": "RW.COMPOSER.UI",
 298 #                                           "config_ready": True
 299 #                                       }
 300 #                                   },
 301                                     {
 302                                         "name": "Start the RW.Proc_10.launchpad",
 303                                         "start": {
 304                                             "component_name": "RW.Proc_10.launchpad",
 305                                             "recovery_action": "RESTART",
 306                                             "config_ready": True
 307                                         }
 308                                     },
 309                                     {
 310                                         "name": "Start the RW.Proc_11.Resource-Manager",
 311                                         "start": {
 312                                             "component_name": "RW.Proc_11.Resource-Manager",
 313                                             "recovery_action": "RESTART",
 314                                             "config_ready": True
 315                                         }
 316                                     },
 317                                     {
 318                                         "name": "Start the RW.uAgent",
 319                                         "start": {
 320                                             "python_variable": ["cmdargs_str = '--confd-proto AF_INET --confd-ip 127.0.0.1'"],
 321                                             "component_name": "RW.uAgent",
 322                                             "recovery_action": "RESTART",
 323                                             "config_ready": True
 324                                         }
 325                                     },
 326                                     {
 327                                         "name": "Start the logd",
 328                                         "start": {
 329                                             "component_name": "logd",
 330                                             "recovery_action": "RESTART",
 331                                             "config_ready": True
 332                                         }
 333                                     }
 334                                 ]
 335                             }]
 336                         }
 337                     }
 338                 },
 339 #               {
 340 #                   "component_name": "RW.CLI",
 341 #                   "component_type": "PROC",
 342 #                   "native_proc": {
 343 #                       "exe_path": "./usr/bin/rwcli",
 344 #                       "args": "--netconf_host 127.0.0.1 --netconf_port 2022 --schema_listing cli_launchpad_schema_listing.txt",
 345 #                   }
 346 #               },
 347                 {
 348                     "component_name": "RW.Proc_1.Restconf",
 349                     "component_type": "RWPROC",
 350                     "rwproc": {
 351                         "tasklet": [{
 352                             "name": "Start RW.Restconf for RW.Proc_1.Restconf",
 353                             "component_name": "RW.Restconf",
 354                             "recovery_action": "RESTART",
 355                             "config_ready": True
 356                         }]
 357                     }
 358                 },
 359                 {
 360                     "component_name": "RW.Restconf",
 361                     "component_type": "RWTASKLET",
 362                     "rwtasklet": {
 363                         "plugin_directory": "./usr/lib/rift/plugins/restconf",
 364                         "plugin_name": "restconf"
 365                     }
 366                 },
 367 #               {
 368 #                   "component_name": "RW.Proc_2.RestPortForward",
 369 #                   "component_type": "RWPROC",
 370 #                   "rwproc": {
 371 #                       "tasklet": [{
 372 #                           "name": "Start RW.RestPortForward for RW.Proc_2.RestPortForward",
 373 #                           "component_name": "RW.RestPortForward",
 374 #                           "recovery_action": "RESTART",
 375 #                           "config_ready": True
 376 #                       }]
 377 #                   }
 378 #               },
 379 #               {
 380 #                   "component_name": "RW.RestPortForward",
 381 #                   "component_type": "RWTASKLET",
 382 #                   "rwtasklet": {
 383 #                       "plugin_directory": "./usr/lib/rift/plugins/restportforward",
 384 #                       "plugin_name": "restportforward"
 385 #                   }
 386 #               },
 387                 {
 388                     "component_name": "RW.Proc_3.CalProxy",
 389                     "component_type": "RWPROC",
 390                     "rwproc": {
 391                         "tasklet": [{
 392                             "name": "Start RW.CalProxy for RW.Proc_3.CalProxy",
 393                             "component_name": "RW.CalProxy",
 394                             "recovery_action": "RESTART",
 395                             "config_ready": True
 396                         }]
 397                     }
 398                 },
 399                 {
 400                     "component_name": "RW.CalProxy",
 401                     "component_type": "RWTASKLET",
 402                     "rwtasklet": {
 403                         "plugin_directory": "./usr/lib/rift/plugins/rwcalproxytasklet",
 404                         "plugin_name": "rwcalproxytasklet"
 405                     }
 406                 },
 407                 {
 408                     "component_name": "RW.Proc_4.nfvi-metrics-monitor",
 409                     "component_type": "RWPROC",
 410                     "rwproc": {
 411                         "tasklet": [{
 412                             "name": "Start nfvi-metrics-monitor for RW.Proc_4.nfvi-metrics-monitor",
 413                             "component_name": "nfvi-metrics-monitor",
 414                             "recovery_action": "RESTART",
 415                             "config_ready": True
 416                         }]
 417                     }
 418                 },
 419                 {
 420                     "component_name": "nfvi-metrics-monitor",
 421                     "component_type": "RWTASKLET",
 422                     "rwtasklet": {
 423                         "plugin_directory": "./usr/lib/rift/plugins/rwmonitor",
 424                         "plugin_name": "rwmonitor"
 425                     }
 426                 },
 427                 {
 428                     "component_name": "RW.Proc_5.network-services-manager",
 429                     "component_type": "RWPROC",
 430                     "rwproc": {
 431                         "tasklet": [{
 432                             "name": "Start network-services-manager for RW.Proc_5.network-services-manager",
 433                             "component_name": "network-services-manager",
 434                             "recovery_action": "RESTART",
 435                             "config_ready": True
 436                         }]
 437                     }
 438                 },
 439                 {
 440                     "component_name": "network-services-manager",
 441                     "component_type": "RWTASKLET",
 442                     "rwtasklet": {
 443                         "plugin_directory": "./usr/lib/rift/plugins/rwnsmtasklet",
 444                         "plugin_name": "rwnsmtasklet"
 445                     }
 446                 },
 447                 {
 448                     "component_name": "RW.Proc_6.virtual-network-function-manager",
 449                     "component_type": "RWPROC",
 450                     "rwproc": {
 451                         "tasklet": [{
 452                             "name": "Start virtual-network-function-manager for RW.Proc_6.virtual-network-function-manager",
 453                             "component_name": "virtual-network-function-manager",
 454                             "recovery_action": "RESTART",
 455                             "config_ready": True
 456                         }]
 457                     }
 458                 },
 459                 {
 460                     "component_name": "virtual-network-function-manager",
 461                     "component_type": "RWTASKLET",
 462                     "rwtasklet": {
 463                         "plugin_directory": "./usr/lib/rift/plugins/rwvnfmtasklet",
 464                         "plugin_name": "rwvnfmtasklet"
 465                     }
 466                 },
 467                 {
 468                     "component_name": "RW.Proc_7.virtual-network-service",
 469                     "component_type": "RWPROC",
 470                     "rwproc": {
 471                         "tasklet": [{
 472                             "name": "Start virtual-network-service for RW.Proc_7.virtual-network-service",
 473                             "component_name": "virtual-network-service",
 474                             "recovery_action": "RESTART",
 475                             "config_ready": True
 476                         }]
 477                     }
 478                 },
 479                 {
 480                     "component_name": "virtual-network-service",
 481                     "component_type": "RWTASKLET",
 482                     "rwtasklet": {
 483                         "plugin_directory": "./usr/lib/rift/plugins/rwvnstasklet",
 484                         "plugin_name": "rwvnstasklet"
 485                     }
 486                 },
 487                 {
 488                     "component_name": "RW.Proc_8.nfvi-metrics-monitor",
 489                     "component_type": "RWPROC",
 490                     "rwproc": {
 491                         "tasklet": [{
 492                             "name": "Start nfvi-metrics-monitor for RW.Proc_8.nfvi-metrics-monitor",
 493                             "component_name": "nfvi-metrics-monitor",
 494                             "recovery_action": "RESTART",
 495                             "config_ready": True
 496                         }]
 497                     }
 498                 },
 499                 {
 500                     "component_name": "RW.MC.UI",
 501                     "component_type": "PROC",
 502                     "native_proc": {
 503                         "exe_path": "./usr/share/rw.ui/skyquake/scripts/launch_ui.sh",
 504                     }
 505                 },
 506                 {
 507                     "component_name": "RW.COMPOSER.UI",
 508                     "component_type": "PROC",
 509                     "native_proc": {
 510                         "exe_path": "./usr/share/composer/scripts/launch_composer.sh",
 511                     }
 512                 },
 513                 {
 514                     "component_name": "RW.Proc_9.Configuration-Manager",
 515                     "component_type": "RWPROC",
 516                     "rwproc": {
 517                         "tasklet": [{
 518                             "name": "Start Configuration-Manager for RW.Proc_9.Configuration-Manager",
 519                             "component_name": "Configuration-Manager",
 520                             "recovery_action": "RESTART",
 521                             "config_ready": True
 522                         }]
 523                     }
 524                 },
 525                 {
 526                     "component_name": "Configuration-Manager",
 527                     "component_type": "RWTASKLET",
 528                     "rwtasklet": {
 529                         "plugin_directory": "./usr/lib/rift/plugins/rwconmantasklet",
 530                         "plugin_name": "rwconmantasklet"
 531                     }
 532                 },
 533                 {
 534                     "component_name": "RW.Proc_10.launchpad",
 535                     "component_type": "RWPROC",
 536                     "rwproc": {
 537                         "tasklet": [{
 538                             "name": "Start launchpad for RW.Proc_10.launchpad",
 539                             "component_name": "launchpad",
 540                             "recovery_action": "RESTART",
 541                             "config_ready": True
 542                         }]
 543                     }
 544                 },
 545                 {
 546                     "component_name": "launchpad",
 547                     "component_type": "RWTASKLET",
 548                     "rwtasklet": {
 549                         "plugin_directory": "./usr/lib/rift/plugins/rwlaunchpad",
 550                         "plugin_name": "rwlaunchpad"
 551                     }
 552                 },
 553                 {
 554                     "component_name": "RW.Proc_11.Resource-Manager",
 555                     "component_type": "RWPROC",
 556                     "rwproc": {
 557                         "tasklet": [{
 558                             "name": "Start Resource-Manager for RW.Proc_11.Resource-Manager",
 559                             "component_name": "Resource-Manager",
 560                             "recovery_action": "RESTART",
 561                             "config_ready": True
 562                         }]
 563                     }
 564                 },
 565                 {
 566                     "component_name": "Resource-Manager",
 567                     "component_type": "RWTASKLET",
 568                     "rwtasklet": {
 569                         "plugin_directory": "./usr/lib/rift/plugins/rwresmgrtasklet",
 570                         "plugin_name": "rwresmgrtasklet"
 571                     }
 572                 },
 573                 {
 574                     "component_name": "RW.uAgent",
 575                     "component_type": "RWTASKLET",
 576                     "rwtasklet": {
 577                         "plugin_directory": "./usr/lib/rift/plugins/rwuagent-c",
 578                         "plugin_name": "rwuagent-c"
 579                     }
 580                 },
 581                 {
 582                     "component_name": "logd",
 583                     "component_type": "RWTASKLET",
 584                     "rwtasklet": {
 585                         "plugin_directory": "./usr/lib/rift/plugins/rwlogd-c",
 586                         "plugin_name": "rwlogd-c"
 587                     }
 588                 },
 589                 {
 590                     "component_name": "msgbroker",
 591                     "component_type": "RWTASKLET",
 592                     "rwtasklet": {
 593                         "plugin_directory": "./usr/lib/rift/plugins/rwmsgbroker-c",
 594                         "plugin_name": "rwmsgbroker-c"
 595                     }
 596                 },
 597                 {
 598                     "component_name": "dtsrouter",
 599                     "component_type": "RWTASKLET",
 600                     "rwtasklet": {
 601                         "plugin_directory": "./usr/lib/rift/plugins/rwdtsrouter-c",
 602                         "plugin_name": "rwdtsrouter-c"
 603                     }
 604                 }
 605             ]
 606         })
 607         return manifest
 608
 609     def tearDown(self):
 610         tasklist = { 'reaperd',
 611                      'rwlogd-report-c',
 612                      'launch_ui.sh' }
 613         for proc in psutil.process_iter():
 614           if proc.name() in tasklist:
 615              print("killing", proc.name())
 616              try:
 617                proc.kill()
 618              except:
 619                print(proc.name(), "no longer exists")
 620         self.loop.stop()
 621         self.loop.close()
 622
 623
 624 class LaunchPadTest(LaunchPad):
 625     """
 626     DTS GI interface unittests
 627
 628     Note:  Each tests uses a list of asyncio.Events for staging through the
 629     test.  These are required here because we are bring up each coroutine
 630     ("tasklet") at the same time and are not implementing any re-try
 631     mechanisms.  For instance, this is used in numerous tests to make sure that
 632     a publisher is up and ready before the subscriber sends queries.  Such
 633     event lists should not be used in production software.
 634     """
 635     @asyncio.coroutine
 636     def inventory(self):
 637         res_iter = yield from self.dts_mgmt.query_read('/rw-base:vcs/rw-base:info', flags=0)
 638         for i in res_iter:
 639            info_result = yield from i
 640         components = info_result.result.components.component_info
 641         recvd_list = {}
 642         for component in components:
 643             recvd_list[component.component_name] = (component.instance_id,
 644                          component.rwcomponent_parent,
 645                          component.component_type,
 646                          component.state)
 647         return recvd_list
 648
 649     @asyncio.coroutine
 650     def issue_vcrash(self, component_type):
 651 #       critical_components = {'msgbroker', 'dtsrouter'}
 652         critical_components = {'msgbroker', 'dtsrouter', 'RW.uAgent'}
 653         comp_inventory = yield from self.inventory()
 654         for component in comp_inventory:
 655           if ((comp_inventory[component])[2] == component_type):
 656               inst = (comp_inventory[component])[0]
 657               if (component in critical_components):
 658                   print(component, 'Marked as CRITICAL - Not restarting')
 659               else:
 660                   print('Crashing ', component_type,component)
 661                   vcrash_input = rwvcs.VCrashInput(instance_name=component+'-'+str(inst))
 662                   query_iter = yield from self.dts_mgmt.query_rpc( xpath="/rw-vcs:vcrash",
 663                                                       flags=0, msg=vcrash_input)
 664                   yield from asyncio.sleep(1, loop=self.loop)
 665                   restarted_inventory = yield from self.inventory()
 666                   self.assertTrue(restarted_inventory[component][3] != 'TO_RECOVER')
 667
 668     def test_launch_pad(self):
 669         """
 670         Verify the launchpad setup functions
 671         The test will progress through stages defined by the events list:
 672             0:  mission_control setup is brought up
 673             2:  Tasklet/PROC/VM restarts tested to confirm recovery is proper
 674         """
 675
 676         print("{{{{{{{{{{{{{{{{{{{{STARTING - mano recovery test")
 677 #       confd_host="127.0.0.1"
 678
 679         events = [asyncio.Event(loop=self.loop) for _ in range(2)]
 680
 681         @asyncio.coroutine
 682         def sub():
 683
 684             tinfo = self.new_tinfo('sub')
 685             self.dts_mgmt = rift.tasklets.DTS(tinfo, self.schema, self.loop)
 686
 687             # Sleep for DTS registrations to complete
 688             print('.........................................................')
 689             print('........SLEEPING 80 seconds for system to come up........')
 690             yield from asyncio.sleep(80, loop=self.loop)
 691             print('........RESUMING........')
 692
 693             @asyncio.coroutine
 694             def issue_vstop(component,inst,flag=0):
 695                 vstop_input = rwvcs.VStopInput(instance_name=component+'-'+(str(inst)))
 696                 query_iter = yield from self.dts_mgmt.query_rpc( xpath="/rw-vcs:vstop",
 697                                     flags=flag, msg=vstop_input)
 698                 yield from asyncio.sleep(1, loop=self.loop)
 699
 700
 701
 702             @asyncio.coroutine
 703             def issue_vstart(component, parent, recover=False):
 704                 vstart_input = rwvcs.VStartInput()
 705                 vstart_input.component_name = component
 706                 vstart_input.parent_instance = parent
 707                 vstart_input.recover = recover
 708                 query_iter = yield from self.dts_mgmt.query_rpc( xpath="/rw-vcs:vstart",
 709                                                       flags=0, msg=vstart_input)
 710                 yield from asyncio.sleep(1, loop=self.loop)
 711
 712             @asyncio.coroutine
 713             def issue_start_stop(comp_inventory, component_type):
 714 #               critical_components = {'msgbroker', 'dtsrouter'}
 715                 critical_components = {'msgbroker', 'dtsrouter', 'RW.uAgent'}
 716                 for component in comp_inventory:
 717                   if ((comp_inventory[component])[2] == component_type):
 718                       inst = (comp_inventory[component])[0]
 719                       parent = (comp_inventory[component])[1]
 720                       if (component in critical_components):
 721                           print(component, 'Marked as CRITICAL - Not restarting')
 722                       else:
 723                           print('Stopping ', component_type,component)
 724                           yield from issue_vstop(component,inst)
 725                           restarted_inventory = yield from self.inventory()
 726 #                         self.assertEqual(restarted_inventory[component][3],'TO_RECOVER')
 727                           print('Starting ',component_type,component)
 728                           yield from issue_vstart(component, parent, recover=True)
 729                           restarted_inventory = yield from self.inventory()
 730                           self.assertTrue(restarted_inventory[component][3] != 'TO_RECOVER')
 731
 732             yield from asyncio.sleep(20, loop=self.loop)
 733             comp_inventory = yield from self.inventory()
 734             yield from issue_start_stop(comp_inventory, 'RWTASKLET')
 735 #           yield from issue_start_stop(comp_inventory, 'RWPROC')
 736 #           yield from self.issue_vcrash('RWTASKLET')
 737
 738             yield from asyncio.sleep(20, loop=self.loop)
 739             restarted_inventory = yield from self.inventory()
 740 #           critical_components = {'msgbroker', 'dtsrouter', 'RW.uAgent'}
 741             for comp in comp_inventory:
 742                 self.assertEqual(str(comp_inventory[comp]), str(restarted_inventory[comp]))
 743 #               if (comp not in critical_components):
 744 #                   inst = (comp_inventory[comp])[0]
 745 #                   yield from issue_vstop(comp,inst)
 746
 747             events[1].set()
 748
 749         asyncio.ensure_future(sub(), loop=self.loop)
 750         self.run_until(events[1].is_set, timeout=260)
 751
 752
 753 def main():
 754     plugin_dir = os.path.join(os.environ["RIFT_INSTALL"], "usr/lib/rift/plugins")
 755     if 'DTS_TEST_PUB_DIR' not in os.environ:
 756         os.environ['DTS_TEST_PUB_DIR'] = os.path.join(plugin_dir, 'dtstestpub')
 757
 758     if 'RIFT_NO_SUDO_REAPER' not in os.environ:
 759         os.environ['RIFT_NO_SUDO_REAPER'] = '1'
 760
 761     if 'MESSAGE_BROKER_DIR' not in os.environ:
 762         os.environ['MESSAGE_BROKER_DIR'] = os.path.join(plugin_dir, 'rwmsgbroker-c')
 763
 764     if 'ROUTER_DIR' not in os.environ:
 765         os.environ['ROUTER_DIR'] = os.path.join(plugin_dir, 'rwdtsrouter-c')
 766
 767     if 'RW_VAR_RIFT' not in os.environ:
 768         os.environ['RW_VAR_RIFT'] = '1'
 769
 770     if 'INSTALLDIR' in os.environ:
 771         os.chdir(os.environ.get('INSTALLDIR'))
 772
 773 #   if 'RWMSG_BROKER_SHUNT' not in os.environ:
 774 #       os.environ['RWMSG_BROKER_SHUNT'] = '1'
 775
 776     if 'TEST_ENVIRON' not in os.environ:
 777         os.environ['TEST_ENVIRON'] = '1'
 778
 779     if 'RW_MANIFEST' not in os.environ:
 780         os.environ['RW_MANIFEST'] = os.path.join(install_dir, 'lptestmanifest.xml')
 781
 782     parser = argparse.ArgumentParser()
 783     parser.add_argument('-v', '--verbose', action='store_true')
 784     args, _ = parser.parse_known_args()
 785
 786
 787     runner = xmlrunner.XMLTestRunner(output=os.environ["RIFT_MODULE_TEST"])
 788     unittest.main(testRunner=runner)
 789
 790 if __name__ == '__main__':
 791     main()
 792
 793 # vim: sw=4