rwcal/test/openstack_resources.py

   1 #!/usr/bin/env python3
   2
   3 #
   4 #   Copyright 2016 RIFT.IO Inc
   5 #
   6 #   Licensed under the Apache License, Version 2.0 (the "License");
   7 #   you may not use this file except in compliance with the License.
   8 #   You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #   Unless required by applicable law or agreed to in writing, software
  13 #   distributed under the License is distributed on an "AS IS" BASIS,
  14 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #   See the License for the specific language governing permissions and
  16 #   limitations under the License.
  17 #
  18
  19 from gi import require_version
  20 require_version('RwCal', '1.0')
  21
  22 from gi.repository import RwcalYang
  23 from gi.repository.RwTypes import RwStatus
  24 import logging
  25 import rw_peas
  26 import rwlogger
  27 import time
  28 import argparse
  29 import os
  30 import sys
  31 import uuid
  32 from os.path import basename
  33
  34 FLAVOR_NAME = 'm1.medium'
  35 DEFAULT_IMAGE='/net/sharedfiles/home1/common/vm/rift-root-latest.qcow2'
  36
  37 persistent_resources = {
  38     'vms'      : ['mission_control','launchpad',],
  39     'networks' : ['public', 'private', 'multisite'],
  40     'flavors'  : ['m1.tiny', 'm1.small', 'm1.medium', 'm1.large', 'm1.xlarge'],
  41     'images'   : ['rwimage','rift-root-latest.qcow2','rift-root-latest-trafgen.qcow2', 'rift-root-latest-trafgen-f.qcow2']
  42 }
  43
  44 #
  45 # Important information about openstack installation. This needs to be manually verified
  46 #
  47 openstack_info = {
  48     'username'           : 'pluto',
  49     'password'           : 'mypasswd',
  50     'project_name'       : 'demo',
  51     'mgmt_network'       : 'private',
  52     'physical_network'   : 'physnet1',
  53     'network_type'       : 'VLAN',
  54     'segmentation_id'    : 42, ### What else?
  55     'subnets'            : ["11.0.0.0/24", "12.0.0.0/24", "13.0.0.0/24", "14.0.0.0/24"],
  56     'subnet_index'       : 0,
  57     }
  58
  59
  60 logging.basicConfig(level=logging.INFO)
  61
  62 USERDATA_FILENAME = os.path.join(os.environ['RIFT_INSTALL'],
  63                                  'etc/userdata-template')
  64
  65
  66 RIFT_BASE_USERDATA = '''
  67 #cloud-config
  68 runcmd:
  69  - sleep 5
  70  - /usr/rift/scripts/cloud/enable_lab
  71  - /usr/rift/etc/fix_this_vm
  72 '''
  73
  74 try:
  75     fd = open(USERDATA_FILENAME, 'r')
  76 except Exception as e:
  77     #logger.error("Received exception during opening of userdata (%s) file. Exception: %s" %(USERDATA_FILENAME, str(e)))
  78     sys.exit(-1)
  79 else:
  80     LP_USERDATA_FILE = fd.read()
  81     # Run the enable lab script when the openstack vm comes up
  82     LP_USERDATA_FILE += "runcmd:\n"
  83     LP_USERDATA_FILE += " - /usr/rift/scripts/cloud/enable_lab\n"
  84     LP_USERDATA_FILE += " - /usr/rift/etc/fix_this_vm\n"
  85
  86
  87
  88 def get_cal_plugin():
  89     """
  90     Loads rw.cal plugin via libpeas
  91     """
  92     plugin = rw_peas.PeasPlugin('rwcal_openstack', 'RwCal-1.0')
  93     engine, info, extension = plugin()
  94     cal = plugin.get_interface("Cloud")
  95     # Get the RwLogger context
  96     rwloggerctx = rwlogger.RwLog.Ctx.new("Cal-Log")
  97     try:
  98         rc = cal.init(rwloggerctx)
  99         assert rc == RwStatus.SUCCESS
 100     except:
 101         logger.error("ERROR:Cal plugin instantiation failed. Aborting tests")
 102     else:
 103         logger.info("Openstack Cal plugin successfully instantiated")
 104         return cal
 105
 106 def get_cal_account(auth_url):
 107     """
 108     Returns cal account
 109     """
 110     account                        = RwcalYang.YangData_RwProject_Project_CloudAccounts_CloudAccountList()
 111     account.account_type           = "openstack"
 112     account.openstack.key          = openstack_info['username']
 113     account.openstack.secret       = openstack_info['password']
 114     account.openstack.auth_url     = auth_url
 115     account.openstack.tenant       = openstack_info['project_name']
 116     account.openstack.mgmt_network = openstack_info['mgmt_network']
 117     return account
 118
 119
 120 logger = logging.getLogger('rift.cal.openstackresources')
 121
 122 class OpenstackResources(object):
 123     """
 124     A stupid class to manage bunch of openstack resources
 125     """
 126     def __init__(self, controller):
 127         self._cal      = get_cal_plugin()
 128         self._acct     = get_cal_account('http://'+controller+':5000/v3/')
 129         self._id       = 0
 130         self._image_id = None
 131         self._flavor_id = None
 132
 133     def _destroy_vms(self):
 134         """
 135         Destroy VMs
 136         """
 137         logger.info("Initiating VM cleanup")
 138         rc, rsp = self._cal.get_vdu_list(self._acct)
 139         vdu_list = [vm for vm in rsp.vdu_info_list if vm.name not in persistent_resources['vms']]
 140         logger.info("Deleting VMs : %s" %([x.name for x in vdu_list]))
 141
 142         for vdu in vdu_list:
 143             self._cal.delete_vdu(self._acct, vdu.vdu_id)
 144
 145         logger.info("VM cleanup complete")
 146
 147     def _destroy_networks(self):
 148         """
 149         Destroy Networks
 150         """
 151         logger.info("Initiating Network cleanup")
 152         rc, rsp = self._cal.get_virtual_link_list(self._acct)
 153         vlink_list = [vlink for vlink in rsp.virtual_link_info_list if vlink.name not in persistent_resources['networks']]
 154
 155         logger.info("Deleting Networks : %s" %([x.name for x in vlink_list]))
 156         for vlink in vlink_list:
 157             self._cal.delete_virtual_link(self._acct, vlink.virtual_link_id)
 158         logger.info("Network cleanup complete")
 159
 160     def _destroy_flavors(self):
 161         """
 162         Destroy Flavors
 163         """
 164         logger.info("Initiating flavor cleanup")
 165         rc, rsp = self._cal.get_flavor_list(self._acct)
 166         flavor_list = [flavor for flavor in rsp.flavorinfo_list if flavor.name not in persistent_resources['flavors']]
 167
 168         logger.info("Deleting flavors : %s" %([x.name for x in flavor_list]))
 169
 170         for flavor in flavor_list:
 171             self._cal.delete_flavor(self._acct, flavor.id)
 172
 173         logger.info("Flavor cleanup complete")
 174
 175     def _destroy_images(self):
 176         logger.info("Initiating image cleanup")
 177         rc, rsp = self._cal.get_image_list(self._acct)
 178         image_list = [image for image in rsp.imageinfo_list if image.name not in persistent_resources['images']]
 179
 180         logger.info("Deleting images : %s" %([x.name for x in image_list]))
 181
 182         for image in image_list:
 183             self._cal.delete_image(self._acct, image.id)
 184
 185         logger.info("Image cleanup complete")
 186
 187     def destroy_resource(self):
 188         """
 189         Destroy resources
 190         """
 191         logger.info("Cleaning up openstack resources")
 192         self._destroy_vms()
 193         self._destroy_networks()
 194         self._destroy_flavors()
 195         self._destroy_images()
 196         logger.info("Cleaning up openstack resources.......[Done]")
 197
 198     def create_mission_control(self):
 199         vm_id = self.create_vm('mission_control',
 200                                userdata = RIFT_BASE_USERDATA)
 201         return vm_id
 202
 203
 204     def create_launchpad_vm(self, salt_master=None):
 205         node_id = str(uuid.uuid4())
 206         if salt_master is not None:
 207            userdata = LP_USERDATA_FILE.format(master_ip = salt_master,
 208                                            lxcname = node_id)
 209         else:
 210            userdata = RIFT_BASE_USERDATA
 211
 212         vm_id = self.create_vm('launchpad',
 213                               userdata = userdata,
 214                               node_id = node_id)
 215 #        vm_id = self.create_vm('launchpad2',
 216 #                               userdata = userdata,
 217 #                               node_id = node_id)
 218         return vm_id
 219
 220     def create_vm(self, name, userdata, node_id = None):
 221         """
 222         Creates a VM. The VM name is derived from username
 223
 224         """
 225         vm = RwcalYang.YangData_RwProject_Project_VduInitParams()
 226         vm.name = name
 227         vm.flavor_id = self._flavor_id
 228         vm.image_id  = self._image_id
 229         if node_id is not None:
 230             vm.node_id = node_id
 231         vm.vdu_init.userdata = userdata
 232         vm.allocate_public_address = True
 233         logger.info("Starting a VM with parameter: %s" %(vm))
 234
 235         rc, vm_id = self._cal.create_vdu(self._acct, vm)
 236         assert rc == RwStatus.SUCCESS
 237         logger.info('Created vm: %s with id: %s', name, vm_id)
 238         return vm_id
 239
 240     def create_network(self, name):
 241         logger.info("Creating network with name: %s" %name)
 242         network                = RwcalYang.YangData_RwProject_Project_VimResources_NetworkinfoList()
 243         network.network_name   = name
 244         network.subnet         = openstack_info['subnets'][openstack_info['subnet_index']]
 245
 246         if openstack_info['subnet_index'] == len(openstack_info['subnets']):
 247             openstack_info['subnet_index'] = 0
 248         else:
 249             openstack_info['subnet_index'] += 1
 250
 251         if openstack_info['physical_network']:
 252             network.provider_network.physical_network = openstack_info['physical_network']
 253         if openstack_info['network_type']:
 254             network.provider_network.overlay_type     = openstack_info['network_type']
 255         if openstack_info['segmentation_id']:
 256             network.provider_network.segmentation_id  = openstack_info['segmentation_id']
 257             openstack_info['segmentation_id'] += 1
 258
 259         rc, net_id = self._cal.create_network(self._acct, network)
 260         assert rc == RwStatus.SUCCESS
 261
 262         logger.info("Successfully created network with id: %s" %net_id)
 263         return net_id
 264
 265
 266
 267     def create_image(self, location):
 268         img = RwcalYang.YangData_RwProject_Project_VimResources_ImageinfoList()
 269         img.name = basename(location)
 270         img.location = location
 271         img.disk_format = "qcow2"
 272         img.container_format = "bare"
 273
 274         logger.info("Uploading image : %s" %img.name)
 275         rc, img_id = self._cal.create_image(self._acct, img)
 276         assert rc == RwStatus.SUCCESS
 277
 278         rs = None
 279         rc = None
 280         image = None
 281         for i in range(100):
 282             rc, rs = self._cal.get_image(self._acct, img_id)
 283             assert rc == RwStatus.SUCCESS
 284             logger.info("Image (image_id: %s) reached status : %s" %(img_id, rs.state))
 285             if rs.state == 'active':
 286                 image = rs
 287                 break
 288             else:
 289                 time.sleep(2) # Sleep for a second
 290
 291         if image is None:
 292             logger.error("Failed to upload openstack image: %s", img)
 293             sys.exit(1)
 294
 295         self._image_id = img_id
 296         logger.info("Uploading image.......[Done]")
 297
 298     def create_flavor(self):
 299         """
 300         Create Flavor suitable for rift_ping_pong VNF
 301         """
 302         flavor = RwcalYang.YangData_RwProject_Project_VimResources_FlavorinfoList()
 303         flavor.name = FLAVOR_NAME
 304         flavor.vm_flavor.memory_mb   = 16384 # 16GB
 305         flavor.vm_flavor.vcpu_count  = 4
 306         flavor.vm_flavor.storage_gb  = 20 # 20 GB
 307
 308         logger.info("Creating new flavor. Flavor Info: %s" %str(flavor.vm_flavor))
 309
 310         rc, flavor_id = self._cal.create_flavor(self._acct, flavor)
 311         assert rc == RwStatus.SUCCESS
 312         logger.info("Creating new flavor.......[Done]")
 313         return flavor_id
 314
 315     def find_image(self, name):
 316         logger.info("Searching for uploaded image: %s" %name)
 317         rc, rsp = self._cal.get_image_list(self._acct)
 318         image_list = [image for image in rsp.imageinfo_list if image.name ==  name]
 319
 320         if not image_list:
 321             logger.error("Image %s not found" %name)
 322             return None
 323
 324         self._image_id = image_list[0].id
 325         logger.info("Searching for uploaded image.......[Done]")
 326         return self._image_id
 327
 328     def find_flavor(self, name=FLAVOR_NAME):
 329         logger.info("Searching for required flavor: %s" %name)
 330         rc, rsp = self._cal.get_flavor_list(self._acct)
 331         flavor_list = [flavor for flavor in rsp.flavorinfo_list if flavor.name == name]
 332
 333         if not flavor_list:
 334             logger.error("Flavor %s not found" %name)
 335             self._flavor_id = self.create_flavor()
 336         else:
 337             self._flavor_id = flavor_list[0].id
 338
 339         logger.info("Searching for required flavor.......[Done]")
 340         return self._flavor_id
 341
 342
 343
 344
 345 def main():
 346     """
 347     Main routine
 348     """
 349     parser = argparse.ArgumentParser(description='Script to manage openstack resources')
 350
 351     parser.add_argument('--controller',
 352                         action = 'store',
 353                         dest = 'controller',
 354                         type = str,
 355                         help='IP Address of openstack controller. This is mandatory parameter')
 356
 357     parser.add_argument('--cleanup',
 358                         action = 'store',
 359                         dest = 'cleanup',
 360                         nargs = '+',
 361                         type = str,
 362                         help = 'Perform resource cleanup for openstack installation. \n Possible options are {all, flavors, vms, networks, images}')
 363
 364     parser.add_argument('--persist-vms',
 365                         action = 'store',
 366                         dest = 'persist_vms',
 367                         help = 'VM instance name to persist')
 368
 369     parser.add_argument('--salt-master',
 370                         action = 'store',
 371                         dest = 'salt_master',
 372                         type = str,
 373                         help='IP Address of salt controller. Required, if VMs are being created.')
 374
 375     parser.add_argument('--upload-image',
 376                         action = 'store',
 377                         dest = 'upload_image',
 378                         help='Openstack image location to upload and use when creating vms.x')
 379
 380     parser.add_argument('--use-image',
 381                         action = 'store',
 382                         dest = 'use_image',
 383                         help='Image name to be used for VM creation')
 384
 385     parser.add_argument('--use-flavor',
 386                         action = 'store',
 387                         dest = 'use_flavor',
 388                         help='Flavor name to be used for VM creation')
 389
 390     parser.add_argument('--mission-control',
 391                         action = 'store_true',
 392                         dest = 'mission_control',
 393                         help='Create Mission Control VM')
 394
 395
 396     parser.add_argument('--launchpad',
 397                         action = 'store_true',
 398                         dest = 'launchpad',
 399                         help='Create LaunchPad VM')
 400
 401     parser.add_argument('--use-project',
 402                         action = 'store',
 403                         dest = 'use_project',
 404                         help='Project name to be used for VM creation')
 405
 406     parser.add_argument('--clean-mclp',
 407                         action='store_true',
 408                         dest='clean_mclp',
 409                         help='Remove Mission Control and Launchpad VMs')
 410
 411     argument = parser.parse_args()
 412
 413     if argument.persist_vms is not None:
 414         global persistent_resources
 415         vm_name_list = argument.persist_vms.split(',')
 416         for single_vm in vm_name_list:
 417                 persistent_resources['vms'].append(single_vm)
 418         logger.info("persist-vms: %s" % persistent_resources['vms'])
 419
 420     if argument.clean_mclp:
 421         persistent_resources['vms'] = []
 422
 423     if argument.controller is None:
 424         logger.error('Need openstack controller IP address')
 425         sys.exit(-1)
 426
 427
 428     if argument.use_project is not None:
 429         openstack_info['project_name'] = argument.use_project
 430
 431     ### Start processing
 432     logger.info("Instantiating cloud-abstraction-layer")
 433     drv = OpenstackResources(argument.controller)
 434     logger.info("Instantiating cloud-abstraction-layer.......[Done]")
 435
 436
 437     if argument.cleanup is not None:
 438         for r_type in argument.cleanup:
 439             if r_type == 'all':
 440                 drv.destroy_resource()
 441                 break
 442             if r_type == 'images':
 443                 drv._destroy_images()
 444             if r_type == 'flavors':
 445                 drv._destroy_flavors()
 446             if r_type == 'vms':
 447                 drv._destroy_vms()
 448             if r_type == 'networks':
 449                 drv._destroy_networks()
 450
 451     if argument.upload_image is not None:
 452         image_name_list = argument.upload_image.split(',')
 453         logger.info("Will upload %d image(s): %s" % (len(image_name_list), image_name_list))
 454         for image_name in image_name_list:
 455             drv.create_image(image_name)
 456             #print("Uploaded :", image_name)
 457
 458     elif argument.use_image is not None:
 459         img = drv.find_image(argument.use_image)
 460         if img == None:
 461             logger.error("Image: %s not found" %(argument.use_image))
 462             sys.exit(-4)
 463     else:
 464         if argument.mission_control or argument.launchpad:
 465             img = drv.find_image(basename(DEFAULT_IMAGE))
 466             if img == None:
 467                 drv.create_image(DEFAULT_IMAGE)
 468
 469     if argument.use_flavor is not None:
 470         drv.find_flavor(argument.use_flavor)
 471     else:
 472         drv.find_flavor()
 473
 474     if argument.mission_control == True:
 475         drv.create_mission_control()
 476
 477     if argument.launchpad == True:
 478         drv.create_launchpad_vm(salt_master = argument.salt_master)
 479
 480
 481 if __name__ == '__main__':
 482     main()
 483