osm_ro/vimconn_openstack.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 osconnector implements all the methods to interact with openstack using the python-client.
  26 '''
  27 __author__="Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research"
  28 __date__ ="$22-jun-2014 11:19:29$"
  29
  30 import vimconn
  31 import json
  32 import yaml
  33 import logging
  34 import netaddr
  35 import time
  36 import yaml
  37 import random
  38
  39 from novaclient import client as nClient, exceptions as nvExceptions
  40 from keystoneauth1.identity import v2, v3
  41 from keystoneauth1 import session
  42 import keystoneclient.exceptions as ksExceptions
  43 import keystoneclient.v3.client as ksClient_v3
  44 import keystoneclient.v2_0.client as ksClient_v2
  45 from glanceclient import client as glClient
  46 import glanceclient.client as gl1Client
  47 import glanceclient.exc as gl1Exceptions
  48 from  cinderclient import client as cClient
  49 from httplib import HTTPException
  50 from neutronclient.neutron import client as neClient
  51 from neutronclient.common import exceptions as neExceptions
  52 from requests.exceptions import ConnectionError
  53
  54 '''contain the openstack virtual machine status to openmano status'''
  55 vmStatus2manoFormat={'ACTIVE':'ACTIVE',
  56                      'PAUSED':'PAUSED',
  57                      'SUSPENDED': 'SUSPENDED',
  58                      'SHUTOFF':'INACTIVE',
  59                      'BUILD':'BUILD',
  60                      'ERROR':'ERROR','DELETED':'DELETED'
  61                      }
  62 netStatus2manoFormat={'ACTIVE':'ACTIVE','PAUSED':'PAUSED','INACTIVE':'INACTIVE','BUILD':'BUILD','ERROR':'ERROR','DELETED':'DELETED'
  63                      }
  64
  65 #global var to have a timeout creating and deleting volumes
  66 volume_timeout = 60
  67 server_timeout = 300
  68
  69 class vimconnector(vimconn.vimconnector):
  70     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,
  71                  log_level=None, config={}, persistent_info={}):
  72         '''using common constructor parameters. In this case
  73         'url' is the keystone authorization url,
  74         'url_admin' is not use
  75         '''
  76         api_version = config.get('APIversion')
  77         if api_version and api_version not in ('v3.3', 'v2.0', '2', '3'):
  78             raise vimconn.vimconnException("Invalid value '{}' for config:APIversion. "
  79                                            "Allowed values are 'v3.3', 'v2.0', '2' or '3'".format(api_version))
  80         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
  81                                       config)
  82
  83         self.insecure = self.config.get("insecure", False)
  84         if not url:
  85             raise TypeError, 'url param can not be NoneType'
  86         self.persistent_info = persistent_info
  87         self.session = persistent_info.get('session', {'reload_client': True})
  88         self.nova = self.session.get('nova')
  89         self.neutron = self.session.get('neutron')
  90         self.cinder = self.session.get('cinder')
  91         self.glance = self.session.get('glance')
  92         self.glancev1 = self.session.get('glancev1')
  93         self.keystone = self.session.get('keystone')
  94         self.api_version3 = self.session.get('api_version3')
  95
  96         self.logger = logging.getLogger('openmano.vim.openstack')
  97         if log_level:
  98             self.logger.setLevel( getattr(logging, log_level) )
  99
 100     def __getitem__(self, index):
 101         """Get individuals parameters.
 102         Throw KeyError"""
 103         if index == 'project_domain_id':
 104             return self.config.get("project_domain_id")
 105         elif index == 'user_domain_id':
 106             return self.config.get("user_domain_id")
 107         else:
 108             return vimconn.vimconnector.__getitem__(self, index)
 109
 110     def __setitem__(self, index, value):
 111         """Set individuals parameters and it is marked as dirty so to force connection reload.
 112         Throw KeyError"""
 113         if index == 'project_domain_id':
 114             self.config["project_domain_id"] = value
 115         elif index == 'user_domain_id':
 116                 self.config["user_domain_id"] = value
 117         else:
 118             vimconn.vimconnector.__setitem__(self, index, value)
 119         self.session['reload_client'] = True
 120
 121     def _reload_connection(self):
 122         '''Called before any operation, it check if credentials has changed
 123         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
 124         '''
 125         #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
 126         if self.session['reload_client']:
 127             if self.config.get('APIversion'):
 128                 self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3'
 129             else:  # get from ending auth_url that end with v3 or with v2.0
 130                 self.api_version3 =  self.url.split("/")[-1] == "v3"
 131             self.session['api_version3'] = self.api_version3
 132             if self.api_version3:
 133                 auth = v3.Password(auth_url=self.url,
 134                                    username=self.user,
 135                                    password=self.passwd,
 136                                    project_name=self.tenant_name,
 137                                    project_id=self.tenant_id,
 138                                    project_domain_id=self.config.get('project_domain_id', 'default'),
 139                                    user_domain_id=self.config.get('user_domain_id', 'default'))
 140             else:
 141                 auth = v2.Password(auth_url=self.url,
 142                                    username=self.user,
 143                                    password=self.passwd,
 144                                    tenant_name=self.tenant_name,
 145                                    tenant_id=self.tenant_id)
 146             sess = session.Session(auth=auth, verify=not self.insecure)
 147             if self.api_version3:
 148                 self.keystone = ksClient_v3.Client(session=sess)
 149             else:
 150                 self.keystone = ksClient_v2.Client(session=sess)
 151             self.session['keystone'] = self.keystone
 152             self.nova = self.session['nova'] = nClient.Client("2.1", session=sess)
 153             self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess)
 154             self.cinder = self.session['cinder'] = cClient.Client(2, session=sess)
 155             self.glance = self.session['glance'] = glClient.Client(2, session=sess)
 156             self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess)
 157             self.session['reload_client'] = False
 158             self.persistent_info['session'] = self.session
 159
 160     def __net_os2mano(self, net_list_dict):
 161         '''Transform the net openstack format to mano format
 162         net_list_dict can be a list of dict or a single dict'''
 163         if type(net_list_dict) is dict:
 164             net_list_=(net_list_dict,)
 165         elif type(net_list_dict) is list:
 166             net_list_=net_list_dict
 167         else:
 168             raise TypeError("param net_list_dict must be a list or a dictionary")
 169         for net in net_list_:
 170             if net.get('provider:network_type') == "vlan":
 171                 net['type']='data'
 172             else:
 173                 net['type']='bridge'
 174
 175     def _format_exception(self, exception):
 176         '''Transform a keystone, nova, neutron  exception into a vimconn exception'''
 177         if isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
 178                                   ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed
 179                                   )):
 180             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 181         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
 182                                     neExceptions.NeutronException, nvExceptions.BadRequest)):
 183             raise vimconn.vimconnUnexpectedResponse(type(exception).__name__ + ": " + str(exception))
 184         elif isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound)):
 185             raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception))
 186         elif isinstance(exception, nvExceptions.Conflict):
 187             raise vimconn.vimconnConflictException(type(exception).__name__ + ": " + str(exception))
 188         elif isinstance(exception, vimconn.vimconnException):
 189             raise
 190         else:  # ()
 191             self.logger.error("General Exception " + str(exception), exc_info=True)
 192             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 193
 194     def get_tenant_list(self, filter_dict={}):
 195         '''Obtain tenants of VIM
 196         filter_dict can contain the following keys:
 197             name: filter by tenant name
 198             id: filter by tenant uuid/id
 199             <other VIM specific>
 200         Returns the tenant list of dictionaries: [{'name':'<name>, 'id':'<id>, ...}, ...]
 201         '''
 202         self.logger.debug("Getting tenants from VIM filter: '%s'", str(filter_dict))
 203         try:
 204             self._reload_connection()
 205             if self.api_version3:
 206                 project_class_list = self.keystone.projects.list(name=filter_dict.get("name"))
 207             else:
 208                 project_class_list = self.keystone.tenants.findall(**filter_dict)
 209             project_list=[]
 210             for project in project_class_list:
 211                 if filter_dict.get('id') and filter_dict["id"] != project.id:
 212                     continue
 213                 project_list.append(project.to_dict())
 214             return project_list
 215         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e:
 216             self._format_exception(e)
 217
 218     def new_tenant(self, tenant_name, tenant_description):
 219         '''Adds a new tenant to openstack VIM. Returns the tenant identifier'''
 220         self.logger.debug("Adding a new tenant name: %s", tenant_name)
 221         try:
 222             self._reload_connection()
 223             if self.api_version3:
 224                 project = self.keystone.projects.create(tenant_name, self.config.get("project_domain_id", "default"),
 225                                                         description=tenant_description, is_domain=False)
 226             else:
 227                 project = self.keystone.tenants.create(tenant_name, tenant_description)
 228             return project.id
 229         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 230             self._format_exception(e)
 231
 232     def delete_tenant(self, tenant_id):
 233         '''Delete a tenant from openstack VIM. Returns the old tenant identifier'''
 234         self.logger.debug("Deleting tenant %s from VIM", tenant_id)
 235         try:
 236             self._reload_connection()
 237             if self.api_version3:
 238                 self.keystone.projects.delete(tenant_id)
 239             else:
 240                 self.keystone.tenants.delete(tenant_id)
 241             return tenant_id
 242         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 243             self._format_exception(e)
 244
 245     def new_network(self,net_name, net_type, ip_profile=None, shared=False, vlan=None):
 246         '''Adds a tenant network to VIM. Returns the network identifier'''
 247         self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type)
 248         #self.logger.debug(">>>>>>>>>>>>>>>>>> IP profile %s", str(ip_profile))
 249         try:
 250             new_net = None
 251             self._reload_connection()
 252             network_dict = {'name': net_name, 'admin_state_up': True}
 253             if net_type=="data" or net_type=="ptp":
 254                 if self.config.get('dataplane_physical_net') == None:
 255                     raise vimconn.vimconnConflictException("You must provide a 'dataplane_physical_net' at config value before creating sriov network")
 256                 network_dict["provider:physical_network"] = self.config['dataplane_physical_net'] #"physnet_sriov" #TODO physical
 257                 network_dict["provider:network_type"]     = "vlan"
 258                 if vlan!=None:
 259                     network_dict["provider:network_type"] = vlan
 260             network_dict["shared"]=shared
 261             new_net=self.neutron.create_network({'network':network_dict})
 262             #print new_net
 263             #create subnetwork, even if there is no profile
 264             if not ip_profile:
 265                 ip_profile = {}
 266             if 'subnet_address' not in ip_profile:
 267                 #Fake subnet is required
 268                 subnet_rand = random.randint(0, 255)
 269                 ip_profile['subnet_address'] = "192.168.{}.0/24".format(subnet_rand)
 270             if 'ip_version' not in ip_profile:
 271                 ip_profile['ip_version'] = "IPv4"
 272             subnet = {"name":net_name+"-subnet",
 273                     "network_id": new_net["network"]["id"],
 274                     "ip_version": 4 if ip_profile['ip_version']=="IPv4" else 6,
 275                     "cidr": ip_profile['subnet_address']
 276                     }
 277             # Gateway should be set to None if not needed. Otherwise openstack assigns one by default
 278             subnet['gateway_ip'] = ip_profile.get('gateway_address')
 279             if ip_profile.get('dns_address'):
 280                 subnet['dns_nameservers'] = ip_profile['dns_address'].split(";")
 281             if 'dhcp_enabled' in ip_profile:
 282                 subnet['enable_dhcp'] = False if ip_profile['dhcp_enabled']=="false" else True
 283             if 'dhcp_start_address' in ip_profile:
 284                 subnet['allocation_pools'] = []
 285                 subnet['allocation_pools'].append(dict())
 286                 subnet['allocation_pools'][0]['start'] = ip_profile['dhcp_start_address']
 287             if 'dhcp_count' in ip_profile:
 288                 #parts = ip_profile['dhcp_start_address'].split('.')
 289                 #ip_int = (int(parts[0]) << 24) + (int(parts[1]) << 16) + (int(parts[2]) << 8) + int(parts[3])
 290                 ip_int = int(netaddr.IPAddress(ip_profile['dhcp_start_address']))
 291                 ip_int += ip_profile['dhcp_count'] - 1
 292                 ip_str = str(netaddr.IPAddress(ip_int))
 293                 subnet['allocation_pools'][0]['end'] = ip_str
 294             #self.logger.debug(">>>>>>>>>>>>>>>>>> Subnet: %s", str(subnet))
 295             self.neutron.create_subnet({"subnet": subnet} )
 296             return new_net["network"]["id"]
 297         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 298             if new_net:
 299                 self.neutron.delete_network(new_net['network']['id'])
 300             self._format_exception(e)
 301
 302     def get_network_list(self, filter_dict={}):
 303         '''Obtain tenant networks of VIM
 304         Filter_dict can be:
 305             name: network name
 306             id: network uuid
 307             shared: boolean
 308             tenant_id: tenant
 309             admin_state_up: boolean
 310             status: 'ACTIVE'
 311         Returns the network list of dictionaries
 312         '''
 313         self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict))
 314         try:
 315             self._reload_connection()
 316             if self.api_version3 and "tenant_id" in filter_dict:
 317                 filter_dict['project_id'] = filter_dict.pop('tenant_id') #TODO check
 318             net_dict=self.neutron.list_networks(**filter_dict)
 319             net_list=net_dict["networks"]
 320             self.__net_os2mano(net_list)
 321             return net_list
 322         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 323             self._format_exception(e)
 324
 325     def get_network(self, net_id):
 326         '''Obtain details of network from VIM
 327         Returns the network information from a network id'''
 328         self.logger.debug(" Getting tenant network %s from VIM", net_id)
 329         filter_dict={"id": net_id}
 330         net_list = self.get_network_list(filter_dict)
 331         if len(net_list)==0:
 332             raise vimconn.vimconnNotFoundException("Network '{}' not found".format(net_id))
 333         elif len(net_list)>1:
 334             raise vimconn.vimconnConflictException("Found more than one network with this criteria")
 335         net = net_list[0]
 336         subnets=[]
 337         for subnet_id in net.get("subnets", () ):
 338             try:
 339                 subnet = self.neutron.show_subnet(subnet_id)
 340             except Exception as e:
 341                 self.logger.error("osconnector.get_network(): Error getting subnet %s %s" % (net_id, str(e)))
 342                 subnet = {"id": subnet_id, "fault": str(e)}
 343             subnets.append(subnet)
 344         net["subnets"] = subnets
 345         net["encapsulation"] = net.get('provider:network_type')
 346         net["segmentation_id"] = net.get('provider:segmentation_id')
 347         return net
 348
 349     def delete_network(self, net_id):
 350         '''Deletes a tenant network from VIM. Returns the old network identifier'''
 351         self.logger.debug("Deleting network '%s' from VIM", net_id)
 352         try:
 353             self._reload_connection()
 354             #delete VM ports attached to this networks before the network
 355             ports = self.neutron.list_ports(network_id=net_id)
 356             for p in ports['ports']:
 357                 try:
 358                     self.neutron.delete_port(p["id"])
 359                 except Exception as e:
 360                     self.logger.error("Error deleting port %s: %s", p["id"], str(e))
 361             self.neutron.delete_network(net_id)
 362             return net_id
 363         except (neExceptions.ConnectionFailed, neExceptions.NetworkNotFoundClient, neExceptions.NeutronException,
 364                 ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 365             self._format_exception(e)
 366
 367     def refresh_nets_status(self, net_list):
 368         '''Get the status of the networks
 369            Params: the list of network identifiers
 370            Returns a dictionary with:
 371                 net_id:         #VIM id of this network
 372                     status:     #Mandatory. Text with one of:
 373                                 #  DELETED (not found at vim)
 374                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 375                                 #  OTHER (Vim reported other status not understood)
 376                                 #  ERROR (VIM indicates an ERROR status)
 377                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 378                                 #  BUILD (on building process)
 379                                 #
 380                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 381                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 382
 383         '''
 384         net_dict={}
 385         for net_id in net_list:
 386             net = {}
 387             try:
 388                 net_vim = self.get_network(net_id)
 389                 if net_vim['status'] in netStatus2manoFormat:
 390                     net["status"] = netStatus2manoFormat[ net_vim['status'] ]
 391                 else:
 392                     net["status"] = "OTHER"
 393                     net["error_msg"] = "VIM status reported " + net_vim['status']
 394
 395                 if net['status'] == "ACTIVE" and not net_vim['admin_state_up']:
 396                     net['status'] = 'DOWN'
 397                 try:
 398                     net['vim_info'] = yaml.safe_dump(net_vim, default_flow_style=True, width=256)
 399                 except yaml.representer.RepresenterError:
 400                     net['vim_info'] = str(net_vim)
 401                 if net_vim.get('fault'):  #TODO
 402                     net['error_msg'] = str(net_vim['fault'])
 403             except vimconn.vimconnNotFoundException as e:
 404                 self.logger.error("Exception getting net status: %s", str(e))
 405                 net['status'] = "DELETED"
 406                 net['error_msg'] = str(e)
 407             except vimconn.vimconnException as e:
 408                 self.logger.error("Exception getting net status: %s", str(e))
 409                 net['status'] = "VIM_ERROR"
 410                 net['error_msg'] = str(e)
 411             net_dict[net_id] = net
 412         return net_dict
 413
 414     def get_flavor(self, flavor_id):
 415         '''Obtain flavor details from the  VIM. Returns the flavor dict details'''
 416         self.logger.debug("Getting flavor '%s'", flavor_id)
 417         try:
 418             self._reload_connection()
 419             flavor = self.nova.flavors.find(id=flavor_id)
 420             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 421             return flavor.to_dict()
 422         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 423             self._format_exception(e)
 424
 425     def get_flavor_id_from_data(self, flavor_dict):
 426         """Obtain flavor id that match the flavor description
 427            Returns the flavor_id or raises a vimconnNotFoundException
 428            flavor_dict: contains the required ram, vcpus, disk
 429            If 'use_existing_flavors' is set to True at config, the closer flavor that provides same or more ram, vcpus
 430                 and disk is returned. Otherwise a flavor with exactly same ram, vcpus and disk is returned or a
 431                 vimconnNotFoundException is raised
 432         """
 433         exact_match = False if self.config.get('use_existing_flavors') else True
 434         try:
 435             self._reload_connection()
 436             flavor_candidate_id = None
 437             flavor_candidate_data = (10000, 10000, 10000)
 438             flavor_target = (flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"])
 439             # numa=None
 440             numas = flavor_dict.get("extended", {}).get("numas")
 441             if numas:
 442                 #TODO
 443                 raise vimconn.vimconnNotFoundException("Flavor with EPA still not implemted")
 444                 # if len(numas) > 1:
 445                 #     raise vimconn.vimconnNotFoundException("Cannot find any flavor with more than one numa")
 446                 # numa=numas[0]
 447                 # numas = extended.get("numas")
 448             for flavor in self.nova.flavors.list():
 449                 epa = flavor.get_keys()
 450                 if epa:
 451                     continue
 452                     # TODO
 453                 flavor_data = (flavor.ram, flavor.vcpus, flavor.disk)
 454                 if flavor_data == flavor_target:
 455                     return flavor.id
 456                 elif not exact_match and flavor_target < flavor_data < flavor_candidate_data:
 457                     flavor_candidate_id = flavor.id
 458                     flavor_candidate_data = flavor_data
 459             if not exact_match and flavor_candidate_id:
 460                 return flavor_candidate_id
 461             raise vimconn.vimconnNotFoundException("Cannot find any flavor matching '{}'".format(str(flavor_dict)))
 462         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 463             self._format_exception(e)
 464
 465
 466     def new_flavor(self, flavor_data, change_name_if_used=True):
 467         '''Adds a tenant flavor to openstack VIM
 468         if change_name_if_used is True, it will change name in case of conflict, because it is not supported name repetition
 469         Returns the flavor identifier
 470         '''
 471         self.logger.debug("Adding flavor '%s'", str(flavor_data))
 472         retry=0
 473         max_retries=3
 474         name_suffix = 0
 475         name=flavor_data['name']
 476         while retry<max_retries:
 477             retry+=1
 478             try:
 479                 self._reload_connection()
 480                 if change_name_if_used:
 481                     #get used names
 482                     fl_names=[]
 483                     fl=self.nova.flavors.list()
 484                     for f in fl:
 485                         fl_names.append(f.name)
 486                     while name in fl_names:
 487                         name_suffix += 1
 488                         name = flavor_data['name']+"-" + str(name_suffix)
 489
 490                 ram = flavor_data.get('ram',64)
 491                 vcpus = flavor_data.get('vcpus',1)
 492                 numa_properties=None
 493
 494                 extended = flavor_data.get("extended")
 495                 if extended:
 496                     numas=extended.get("numas")
 497                     if numas:
 498                         numa_nodes = len(numas)
 499                         if numa_nodes > 1:
 500                             return -1, "Can not add flavor with more than one numa"
 501                         numa_properties = {"hw:numa_nodes":str(numa_nodes)}
 502                         numa_properties["hw:mem_page_size"] = "large"
 503                         numa_properties["hw:cpu_policy"] = "dedicated"
 504                         numa_properties["hw:numa_mempolicy"] = "strict"
 505                         for numa in numas:
 506                             #overwrite ram and vcpus
 507                             ram = numa['memory']*1024
 508                             #See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/implemented/virt-driver-cpu-thread-pinning.html
 509                             if 'paired-threads' in numa:
 510                                 vcpus = numa['paired-threads']*2
 511                                 #cpu_thread_policy "require" implies that the compute node must have an STM architecture
 512                                 numa_properties["hw:cpu_thread_policy"] = "require"
 513                                 numa_properties["hw:cpu_policy"] = "dedicated"
 514                             elif 'cores' in numa:
 515                                 vcpus = numa['cores']
 516                                 # cpu_thread_policy "prefer" implies that the host must not have an SMT architecture, or a non-SMT architecture will be emulated
 517                                 numa_properties["hw:cpu_thread_policy"] = "isolate"
 518                                 numa_properties["hw:cpu_policy"] = "dedicated"
 519                             elif 'threads' in numa:
 520                                 vcpus = numa['threads']
 521                                 # cpu_thread_policy "prefer" implies that the host may or may not have an SMT architecture
 522                                 numa_properties["hw:cpu_thread_policy"] = "prefer"
 523                                 numa_properties["hw:cpu_policy"] = "dedicated"
 524                             # for interface in numa.get("interfaces",() ):
 525                             #     if interface["dedicated"]=="yes":
 526                             #         raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
 527                             #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"' when a way to connect it is available
 528
 529                 #create flavor
 530                 new_flavor=self.nova.flavors.create(name,
 531                                 ram,
 532                                 vcpus,
 533                                 flavor_data.get('disk',1),
 534                                 is_public=flavor_data.get('is_public', True)
 535                             )
 536                 #add metadata
 537                 if numa_properties:
 538                     new_flavor.set_keys(numa_properties)
 539                 return new_flavor.id
 540             except nvExceptions.Conflict as e:
 541                 if change_name_if_used and retry < max_retries:
 542                     continue
 543                 self._format_exception(e)
 544             #except nvExceptions.BadRequest as e:
 545             except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 546                 self._format_exception(e)
 547
 548     def delete_flavor(self,flavor_id):
 549         '''Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
 550         '''
 551         try:
 552             self._reload_connection()
 553             self.nova.flavors.delete(flavor_id)
 554             return flavor_id
 555         #except nvExceptions.BadRequest as e:
 556         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 557             self._format_exception(e)
 558
 559     def new_image(self,image_dict):
 560         '''
 561         Adds a tenant image to VIM. imge_dict is a dictionary with:
 562             name: name
 563             disk_format: qcow2, vhd, vmdk, raw (by default), ...
 564             location: path or URI
 565             public: "yes" or "no"
 566             metadata: metadata of the image
 567         Returns the image_id
 568         '''
 569         retry=0
 570         max_retries=3
 571         while retry<max_retries:
 572             retry+=1
 573             try:
 574                 self._reload_connection()
 575                 #determine format  http://docs.openstack.org/developer/glance/formats.html
 576                 if "disk_format" in image_dict:
 577                     disk_format=image_dict["disk_format"]
 578                 else: #autodiscover based on extension
 579                     if image_dict['location'][-6:]==".qcow2":
 580                         disk_format="qcow2"
 581                     elif image_dict['location'][-4:]==".vhd":
 582                         disk_format="vhd"
 583                     elif image_dict['location'][-5:]==".vmdk":
 584                         disk_format="vmdk"
 585                     elif image_dict['location'][-4:]==".vdi":
 586                         disk_format="vdi"
 587                     elif image_dict['location'][-4:]==".iso":
 588                         disk_format="iso"
 589                     elif image_dict['location'][-4:]==".aki":
 590                         disk_format="aki"
 591                     elif image_dict['location'][-4:]==".ari":
 592                         disk_format="ari"
 593                     elif image_dict['location'][-4:]==".ami":
 594                         disk_format="ami"
 595                     else:
 596                         disk_format="raw"
 597                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
 598                 if image_dict['location'][0:4]=="http":
 599                     new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 600                             container_format="bare", location=image_dict['location'], disk_format=disk_format)
 601                 else: #local path
 602                     with open(image_dict['location']) as fimage:
 603                         new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 604                             container_format="bare", data=fimage, disk_format=disk_format)
 605                 #insert metadata. We cannot use 'new_image.properties.setdefault'
 606                 #because nova and glance are "INDEPENDENT" and we are using nova for reading metadata
 607                 new_image_nova=self.nova.images.find(id=new_image.id)
 608                 new_image_nova.metadata.setdefault('location',image_dict['location'])
 609                 metadata_to_load = image_dict.get('metadata')
 610                 if metadata_to_load:
 611                     for k,v in yaml.load(metadata_to_load).iteritems():
 612                         new_image_nova.metadata.setdefault(k,v)
 613                 return new_image.id
 614             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
 615                 self._format_exception(e)
 616             except (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 617                 if retry==max_retries:
 618                     continue
 619                 self._format_exception(e)
 620             except IOError as e:  #can not open the file
 621                 raise vimconn.vimconnConnectionException(type(e).__name__ + ": " + str(e)+ " for " + image_dict['location'],
 622                                                          http_code=vimconn.HTTP_Bad_Request)
 623
 624     def delete_image(self, image_id):
 625         '''Deletes a tenant image from openstack VIM. Returns the old id
 626         '''
 627         try:
 628             self._reload_connection()
 629             self.nova.images.delete(image_id)
 630             return image_id
 631         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e: #TODO remove
 632             self._format_exception(e)
 633
 634     def get_image_id_from_path(self, path):
 635         '''Get the image id from image path in the VIM database. Returns the image_id'''
 636         try:
 637             self._reload_connection()
 638             images = self.nova.images.list()
 639             for image in images:
 640                 if image.metadata.get("location")==path:
 641                     return image.id
 642             raise vimconn.vimconnNotFoundException("image with location '{}' not found".format( path))
 643         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 644             self._format_exception(e)
 645
 646     def get_image_list(self, filter_dict={}):
 647         '''Obtain tenant images from VIM
 648         Filter_dict can be:
 649             id: image id
 650             name: image name
 651             checksum: image checksum
 652         Returns the image list of dictionaries:
 653             [{<the fields at Filter_dict plus some VIM specific>}, ...]
 654             List can be empty
 655         '''
 656         self.logger.debug("Getting image list from VIM filter: '%s'", str(filter_dict))
 657         try:
 658             self._reload_connection()
 659             filter_dict_os=filter_dict.copy()
 660             #First we filter by the available filter fields: name, id. The others are removed.
 661             filter_dict_os.pop('checksum',None)
 662             image_list=self.nova.images.findall(**filter_dict_os)
 663             if len(image_list)==0:
 664                 return []
 665             #Then we filter by the rest of filter fields: checksum
 666             filtered_list = []
 667             for image in image_list:
 668                 image_class=self.glance.images.get(image.id)
 669                 if 'checksum' not in filter_dict or image_class['checksum']==filter_dict.get('checksum'):
 670                     filtered_list.append(image_class.copy())
 671             return filtered_list
 672         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 673             self._format_exception(e)
 674
 675     def __wait_for_vm(self, vm_id, status):
 676         """wait until vm is in the desired status and return True.
 677         If the VM gets in ERROR status, return false.
 678         If the timeout is reached generate an exception"""
 679         elapsed_time = 0
 680         while elapsed_time < server_timeout:
 681             vm_status = self.nova.servers.get(vm_id).status
 682             if vm_status == status:
 683                 return True
 684             if vm_status == 'ERROR':
 685                 return False
 686             time.sleep(1)
 687             elapsed_time += 1
 688
 689         # if we exceeded the timeout rollback
 690         if elapsed_time >= server_timeout:
 691             raise vimconn.vimconnException('Timeout waiting for instance ' + vm_id + ' to get ' + status,
 692                                            http_code=vimconn.HTTP_Request_Timeout)
 693
 694     def new_vminstance(self,name,description,start,image_id,flavor_id,net_list,cloud_config=None,disk_list=None):
 695         '''Adds a VM instance to VIM
 696         Params:
 697             start: indicates if VM must start or boot in pause mode. Ignored
 698             image_id,flavor_id: iamge and flavor uuid
 699             net_list: list of interfaces, each one is a dictionary with:
 700                 name:
 701                 net_id: network uuid to connect
 702                 vpci: virtual vcpi to assign, ignored because openstack lack #TODO
 703                 model: interface model, ignored #TODO
 704                 mac_address: used for  SR-IOV ifaces #TODO for other types
 705                 use: 'data', 'bridge',  'mgmt'
 706                 type: 'virtual', 'PF', 'VF', 'VFnotShared'
 707                 vim_id: filled/added by this function
 708                 floating_ip: True/False (or it can be None)
 709                 #TODO ip, security groups
 710         Returns the instance identifier
 711         '''
 712         self.logger.debug("new_vminstance input: image='%s' flavor='%s' nics='%s'",image_id, flavor_id,str(net_list))
 713         try:
 714             server = None
 715             metadata={}
 716             net_list_vim=[]
 717             external_network=[]     # list of external networks to be connected to instance, later on used to create floating_ip
 718             no_secured_ports = []   # List of port-is with port-security disabled
 719             self._reload_connection()
 720             metadata_vpci={}   # For a specific neutron plugin
 721             block_device_mapping = None
 722             for net in net_list:
 723                 if not net.get("net_id"): #skip non connected iface
 724                     continue
 725
 726                 port_dict={
 727                     "network_id": net["net_id"],
 728                     "name": net.get("name"),
 729                     "admin_state_up": True
 730                 }
 731                 if net["type"]=="virtual":
 732                     if "vpci" in net:
 733                         metadata_vpci[ net["net_id"] ] = [[ net["vpci"], "" ]]
 734                 elif net["type"]=="VF": # for VF
 735                     if "vpci" in net:
 736                         if "VF" not in metadata_vpci:
 737                             metadata_vpci["VF"]=[]
 738                         metadata_vpci["VF"].append([ net["vpci"], "" ])
 739                     port_dict["binding:vnic_type"]="direct"
 740                 else:  # For PT
 741                     if "vpci" in net:
 742                         if "PF" not in metadata_vpci:
 743                             metadata_vpci["PF"]=[]
 744                         metadata_vpci["PF"].append([ net["vpci"], "" ])
 745                     port_dict["binding:vnic_type"]="direct-physical"
 746                 if not port_dict["name"]:
 747                     port_dict["name"]=name
 748                 if net.get("mac_address"):
 749                     port_dict["mac_address"]=net["mac_address"]
 750                 new_port = self.neutron.create_port({"port": port_dict })
 751                 net["mac_adress"] = new_port["port"]["mac_address"]
 752                 net["vim_id"] = new_port["port"]["id"]
 753                 # if try to use a network without subnetwork, it will return a emtpy list
 754                 fixed_ips = new_port["port"].get("fixed_ips")
 755                 if fixed_ips:
 756                     net["ip"] = fixed_ips[0].get("ip_address")
 757                 else:
 758                     net["ip"] = None
 759                 net_list_vim.append({"port-id": new_port["port"]["id"]})
 760
 761                 if net.get('floating_ip', False):
 762                     net['exit_on_floating_ip_error'] = True
 763                     external_network.append(net)
 764                 elif net['use'] == 'mgmt' and self.config.get('use_floating_ip'):
 765                     net['exit_on_floating_ip_error'] = False
 766                     external_network.append(net)
 767
 768                 # If port security is disabled when the port has not yet been attached to the VM, then all vm traffic is dropped.
 769                 # As a workaround we wait until the VM is active and then disable the port-security
 770                 if net.get("port_security") == False:
 771                     no_secured_ports.append(new_port["port"]["id"])
 772
 773             if metadata_vpci:
 774                 metadata = {"pci_assignement": json.dumps(metadata_vpci)}
 775                 if len(metadata["pci_assignement"]) >255:
 776                     #limit the metadata size
 777                     #metadata["pci_assignement"] = metadata["pci_assignement"][0:255]
 778                     self.logger.warn("Metadata deleted since it exceeds the expected length (255) ")
 779                     metadata = {}
 780
 781             self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s' metadata %s",
 782                               name, image_id, flavor_id, str(net_list_vim), description, str(metadata))
 783
 784             security_groups   = self.config.get('security_groups')
 785             if type(security_groups) is str:
 786                 security_groups = ( security_groups, )
 787             #cloud config
 788             userdata=None
 789             config_drive = None
 790             if isinstance(cloud_config, dict):
 791                 if cloud_config.get("user-data"):
 792                     userdata=cloud_config["user-data"]
 793                 if cloud_config.get("boot-data-drive") != None:
 794                     config_drive = cloud_config["boot-data-drive"]
 795                 if cloud_config.get("config-files") or cloud_config.get("users") or cloud_config.get("key-pairs"):
 796                     if userdata:
 797                         raise vimconn.vimconnConflictException("Cloud-config cannot contain both 'userdata' and 'config-files'/'users'/'key-pairs'")
 798                     userdata_dict={}
 799                     #default user
 800                     if cloud_config.get("key-pairs"):
 801                         userdata_dict["ssh-authorized-keys"] = cloud_config["key-pairs"]
 802                         userdata_dict["users"] = [{"default": None, "ssh-authorized-keys": cloud_config["key-pairs"] }]
 803                     if cloud_config.get("users"):
 804                         if "users" not in userdata_dict:
 805                             userdata_dict["users"] = [ "default" ]
 806                         for user in cloud_config["users"]:
 807                             user_info = {
 808                                 "name" : user["name"],
 809                                 "sudo": "ALL = (ALL)NOPASSWD:ALL"
 810                             }
 811                             if "user-info" in user:
 812                                 user_info["gecos"] = user["user-info"]
 813                             if user.get("key-pairs"):
 814                                 user_info["ssh-authorized-keys"] = user["key-pairs"]
 815                             userdata_dict["users"].append(user_info)
 816
 817                     if cloud_config.get("config-files"):
 818                         userdata_dict["write_files"] = []
 819                         for file in cloud_config["config-files"]:
 820                             file_info = {
 821                                 "path" : file["dest"],
 822                                 "content": file["content"]
 823                             }
 824                             if file.get("encoding"):
 825                                 file_info["encoding"] = file["encoding"]
 826                             if file.get("permissions"):
 827                                 file_info["permissions"] = file["permissions"]
 828                             if file.get("owner"):
 829                                 file_info["owner"] = file["owner"]
 830                             userdata_dict["write_files"].append(file_info)
 831                     userdata = "#cloud-config\n"
 832                     userdata += yaml.safe_dump(userdata_dict, indent=4, default_flow_style=False)
 833                 self.logger.debug("userdata: %s", userdata)
 834             elif isinstance(cloud_config, str):
 835                 userdata = cloud_config
 836
 837             #Create additional volumes in case these are present in disk_list
 838             base_disk_index = ord('b')
 839             if disk_list != None:
 840                 block_device_mapping = {}
 841                 for disk in disk_list:
 842                     if 'image_id' in disk:
 843                         volume = self.cinder.volumes.create(size = disk['size'],name = name + '_vd' +
 844                                     chr(base_disk_index), imageRef = disk['image_id'])
 845                     else:
 846                         volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
 847                                     chr(base_disk_index))
 848                     block_device_mapping['_vd' +  chr(base_disk_index)] = volume.id
 849                     base_disk_index += 1
 850
 851                 #wait until volumes are with status available
 852                 keep_waiting = True
 853                 elapsed_time = 0
 854                 while keep_waiting and elapsed_time < volume_timeout:
 855                     keep_waiting = False
 856                     for volume_id in block_device_mapping.itervalues():
 857                         if self.cinder.volumes.get(volume_id).status != 'available':
 858                             keep_waiting = True
 859                     if keep_waiting:
 860                         time.sleep(1)
 861                         elapsed_time += 1
 862
 863                 #if we exceeded the timeout rollback
 864                 if elapsed_time >= volume_timeout:
 865                     #delete the volumes we just created
 866                     for volume_id in block_device_mapping.itervalues():
 867                         self.cinder.volumes.delete(volume_id)
 868
 869                     #delete ports we just created
 870                     for net_item  in net_list_vim:
 871                         if 'port-id' in net_item:
 872                             self.neutron.delete_port(net_item['port-id'])
 873
 874                     raise vimconn.vimconnException('Timeout creating volumes for instance ' + name,
 875                                                    http_code=vimconn.HTTP_Request_Timeout)
 876
 877             self.logger.debug("nova.servers.create({}, {}, {}, nics={}, meta={}, security_groups={}," \
 878                                               "availability_zone={}, key_name={}, userdata={}, config_drive={}, " \
 879                                               "block_device_mapping={})".format(name, image_id, flavor_id, net_list_vim,
 880                                                 metadata, security_groups, self.config.get('availability_zone'),
 881                                                 self.config.get('keypair'), userdata, config_drive, block_device_mapping))
 882             server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim, meta=metadata,
 883                                               security_groups=security_groups,
 884                                               availability_zone=self.config.get('availability_zone'),
 885                                               key_name=self.config.get('keypair'),
 886                                               userdata=userdata,
 887                                               config_drive=config_drive,
 888                                               block_device_mapping=block_device_mapping
 889                                               )  # , description=description)
 890
 891             # Previously mentioned workaround to wait until the VM is active and then disable the port-security
 892             if no_secured_ports:
 893                 self.__wait_for_vm(server.id, 'ACTIVE')
 894
 895             for port_id in no_secured_ports:
 896                 try:
 897                     self.neutron.update_port(port_id, {"port": {"port_security_enabled": False, "security_groups": None} })
 898
 899                 except Exception as e:
 900                     self.logger.error("It was not possible to disable port security for port {}".format(port_id))
 901                     self.delete_vminstance(server.id)
 902                     raise
 903
 904             #print "DONE :-)", server
 905             pool_id = None
 906             floating_ips = self.neutron.list_floatingips().get("floatingips", ())
 907
 908             if external_network:
 909                 self.__wait_for_vm(server.id, 'ACTIVE')
 910
 911             for floating_network in external_network:
 912                 try:
 913                     assigned = False
 914                     while(assigned == False):
 915                         if floating_ips:
 916                             ip = floating_ips.pop(0)
 917                             if not ip.get("port_id", False) and ip.get('tenant_id') == server.tenant_id:
 918                                 free_floating_ip = ip.get("floating_ip_address")
 919                                 try:
 920                                     fix_ip = floating_network.get('ip')
 921                                     server.add_floating_ip(free_floating_ip, fix_ip)
 922                                     assigned = True
 923                                 except Exception as e:
 924                                     raise vimconn.vimconnException(type(e).__name__ + ": Cannot create floating_ip "+  str(e), http_code=vimconn.HTTP_Conflict)
 925                         else:
 926                             #Find the external network
 927                             external_nets = list()
 928                             for net in self.neutron.list_networks()['networks']:
 929                                 if net['router:external']:
 930                                         external_nets.append(net)
 931
 932                             if len(external_nets) == 0:
 933                                 raise vimconn.vimconnException("Cannot create floating_ip automatically since no external "
 934                                                                "network is present",
 935                                                                 http_code=vimconn.HTTP_Conflict)
 936                             if len(external_nets) > 1:
 937                                 raise vimconn.vimconnException("Cannot create floating_ip automatically since multiple "
 938                                                                "external networks are present",
 939                                                                http_code=vimconn.HTTP_Conflict)
 940
 941                             pool_id = external_nets[0].get('id')
 942                             param = {'floatingip': {'floating_network_id': pool_id, 'tenant_id': server.tenant_id}}
 943                             try:
 944                                 #self.logger.debug("Creating floating IP")
 945                                 new_floating_ip = self.neutron.create_floatingip(param)
 946                                 free_floating_ip = new_floating_ip['floatingip']['floating_ip_address']
 947                                 fix_ip = floating_network.get('ip')
 948                                 server.add_floating_ip(free_floating_ip, fix_ip)
 949                                 assigned=True
 950                             except Exception as e:
 951                                 raise vimconn.vimconnException(type(e).__name__ + ": Cannot assign floating_ip "+  str(e), http_code=vimconn.HTTP_Conflict)
 952                 except Exception as e:
 953                     if not floating_network['exit_on_floating_ip_error']:
 954                         self.logger.warn("Cannot create floating_ip. %s", str(e))
 955                         continue
 956                     raise
 957
 958             return server.id
 959 #        except nvExceptions.NotFound as e:
 960 #            error_value=-vimconn.HTTP_Not_Found
 961 #            error_text= "vm instance %s not found" % vm_id
 962 #        except TypeError as e:
 963 #            raise vimconn.vimconnException(type(e).__name__ + ": "+  str(e), http_code=vimconn.HTTP_Bad_Request)
 964
 965         except Exception as e:
 966             # delete the volumes we just created
 967             if block_device_mapping:
 968                 for volume_id in block_device_mapping.itervalues():
 969                     self.cinder.volumes.delete(volume_id)
 970
 971             # Delete the VM
 972             if server != None:
 973                 self.delete_vminstance(server.id)
 974             else:
 975                 # delete ports we just created
 976                 for net_item in net_list_vim:
 977                     if 'port-id' in net_item:
 978                         self.neutron.delete_port(net_item['port-id'])
 979
 980             self._format_exception(e)
 981
 982     def get_vminstance(self,vm_id):
 983         '''Returns the VM instance information from VIM'''
 984         #self.logger.debug("Getting VM from VIM")
 985         try:
 986             self._reload_connection()
 987             server = self.nova.servers.find(id=vm_id)
 988             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 989             return server.to_dict()
 990         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
 991             self._format_exception(e)
 992
 993     def get_vminstance_console(self,vm_id, console_type="vnc"):
 994         '''
 995         Get a console for the virtual machine
 996         Params:
 997             vm_id: uuid of the VM
 998             console_type, can be:
 999                 "novnc" (by default), "xvpvnc" for VNC types,
1000                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
1001         Returns dict with the console parameters:
1002                 protocol: ssh, ftp, http, https, ...
1003                 server:   usually ip address
1004                 port:     the http, ssh, ... port
1005                 suffix:   extra text, e.g. the http path and query string
1006         '''
1007         self.logger.debug("Getting VM CONSOLE from VIM")
1008         try:
1009             self._reload_connection()
1010             server = self.nova.servers.find(id=vm_id)
1011             if console_type == None or console_type == "novnc":
1012                 console_dict = server.get_vnc_console("novnc")
1013             elif console_type == "xvpvnc":
1014                 console_dict = server.get_vnc_console(console_type)
1015             elif console_type == "rdp-html5":
1016                 console_dict = server.get_rdp_console(console_type)
1017             elif console_type == "spice-html5":
1018                 console_dict = server.get_spice_console(console_type)
1019             else:
1020                 raise vimconn.vimconnException("console type '{}' not allowed".format(console_type), http_code=vimconn.HTTP_Bad_Request)
1021
1022             console_dict1 = console_dict.get("console")
1023             if console_dict1:
1024                 console_url = console_dict1.get("url")
1025                 if console_url:
1026                     #parse console_url
1027                     protocol_index = console_url.find("//")
1028                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1029                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1030                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1031                         return -vimconn.HTTP_Internal_Server_Error, "Unexpected response from VIM"
1032                     console_dict={"protocol": console_url[0:protocol_index],
1033                                   "server":   console_url[protocol_index+2:port_index],
1034                                   "port":     console_url[port_index:suffix_index],
1035                                   "suffix":   console_url[suffix_index+1:]
1036                                   }
1037                     protocol_index += 2
1038                     return console_dict
1039             raise vimconn.vimconnUnexpectedResponse("Unexpected response from VIM")
1040
1041         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.BadRequest, ConnectionError) as e:
1042             self._format_exception(e)
1043
1044     def delete_vminstance(self, vm_id):
1045         '''Removes a VM instance from VIM. Returns the old identifier
1046         '''
1047         #print "osconnector: Getting VM from VIM"
1048         try:
1049             self._reload_connection()
1050             #delete VM ports attached to this networks before the virtual machine
1051             ports = self.neutron.list_ports(device_id=vm_id)
1052             for p in ports['ports']:
1053                 try:
1054                     self.neutron.delete_port(p["id"])
1055                 except Exception as e:
1056                     self.logger.error("Error deleting port: " + type(e).__name__ + ": "+  str(e))
1057
1058             #commented because detaching the volumes makes the servers.delete not work properly ?!?
1059             #dettach volumes attached
1060             server = self.nova.servers.get(vm_id)
1061             volumes_attached_dict = server._info['os-extended-volumes:volumes_attached']
1062             #for volume in volumes_attached_dict:
1063             #    self.cinder.volumes.detach(volume['id'])
1064
1065             self.nova.servers.delete(vm_id)
1066
1067             #delete volumes.
1068             #Although having detached them should have them  in active status
1069             #we ensure in this loop
1070             keep_waiting = True
1071             elapsed_time = 0
1072             while keep_waiting and elapsed_time < volume_timeout:
1073                 keep_waiting = False
1074                 for volume in volumes_attached_dict:
1075                     if self.cinder.volumes.get(volume['id']).status != 'available':
1076                         keep_waiting = True
1077                     else:
1078                         self.cinder.volumes.delete(volume['id'])
1079                 if keep_waiting:
1080                     time.sleep(1)
1081                     elapsed_time += 1
1082
1083             return vm_id
1084         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
1085             self._format_exception(e)
1086         #TODO insert exception vimconn.HTTP_Unauthorized
1087         #if reaching here is because an exception
1088
1089     def refresh_vms_status(self, vm_list):
1090         '''Get the status of the virtual machines and their interfaces/ports
1091            Params: the list of VM identifiers
1092            Returns a dictionary with:
1093                 vm_id:          #VIM id of this Virtual Machine
1094                     status:     #Mandatory. Text with one of:
1095                                 #  DELETED (not found at vim)
1096                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1097                                 #  OTHER (Vim reported other status not understood)
1098                                 #  ERROR (VIM indicates an ERROR status)
1099                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1100                                 #  CREATING (on building process), ERROR
1101                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
1102                                 #
1103                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
1104                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
1105                     interfaces:
1106                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
1107                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
1108                         vim_net_id:       #network id where this interface is connected
1109                         vim_interface_id: #interface/port VIM id
1110                         ip_address:       #null, or text with IPv4, IPv6 address
1111                         compute_node:     #identification of compute node where PF,VF interface is allocated
1112                         pci:              #PCI address of the NIC that hosts the PF,VF
1113                         vlan:             #physical VLAN used for VF
1114         '''
1115         vm_dict={}
1116         self.logger.debug("refresh_vms status: Getting tenant VM instance information from VIM")
1117         for vm_id in vm_list:
1118             vm={}
1119             try:
1120                 vm_vim = self.get_vminstance(vm_id)
1121                 if vm_vim['status'] in vmStatus2manoFormat:
1122                     vm['status']    =  vmStatus2manoFormat[ vm_vim['status'] ]
1123                 else:
1124                     vm['status']    = "OTHER"
1125                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
1126                 try:
1127                     vm['vim_info']  = yaml.safe_dump(vm_vim, default_flow_style=True, width=256)
1128                 except yaml.representer.RepresenterError:
1129                     vm['vim_info'] = str(vm_vim)
1130                 vm["interfaces"] = []
1131                 if vm_vim.get('fault'):
1132                     vm['error_msg'] = str(vm_vim['fault'])
1133                 #get interfaces
1134                 try:
1135                     self._reload_connection()
1136                     port_dict=self.neutron.list_ports(device_id=vm_id)
1137                     for port in port_dict["ports"]:
1138                         interface={}
1139                         try:
1140                             interface['vim_info'] = yaml.safe_dump(port, default_flow_style=True, width=256)
1141                         except yaml.representer.RepresenterError:
1142                             interface['vim_info'] = str(port)
1143                         interface["mac_address"] = port.get("mac_address")
1144                         interface["vim_net_id"] = port["network_id"]
1145                         interface["vim_interface_id"] = port["id"]
1146                         # check if OS-EXT-SRV-ATTR:host is there,
1147                         # in case of non-admin credentials, it will be missing
1148                         if vm_vim.get('OS-EXT-SRV-ATTR:host'):
1149                             interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host']
1150                         interface["pci"] = None
1151
1152                         # check if binding:profile is there,
1153                         # in case of non-admin credentials, it will be missing
1154                         if port.get('binding:profile'):
1155                             if port['binding:profile'].get('pci_slot'):
1156                                 # TODO: At the moment sr-iov pci addresses are converted to PF pci addresses by setting the slot to 0x00
1157                                 # TODO: This is just a workaround valid for niantinc. Find a better way to do so
1158                                 #   CHANGE DDDD:BB:SS.F to DDDD:BB:00.(F%2)   assuming there are 2 ports per nic
1159                                 pci = port['binding:profile']['pci_slot']
1160                                 # interface["pci"] = pci[:-4] + "00." + str(int(pci[-1]) % 2)
1161                                 interface["pci"] = pci
1162                         interface["vlan"] = None
1163                         #if network is of type vlan and port is of type direct (sr-iov) then set vlan id
1164                         network = self.neutron.show_network(port["network_id"])
1165                         if network['network'].get('provider:network_type') == 'vlan' and \
1166                             port.get("binding:vnic_type") == "direct":
1167                             interface["vlan"] = network['network'].get('provider:segmentation_id')
1168                         ips=[]
1169                         #look for floating ip address
1170                         floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
1171                         if floating_ip_dict.get("floatingips"):
1172                             ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address") )
1173
1174                         for subnet in port["fixed_ips"]:
1175                             ips.append(subnet["ip_address"])
1176                         interface["ip_address"] = ";".join(ips)
1177                         vm["interfaces"].append(interface)
1178                 except Exception as e:
1179                     self.logger.error("Error getting vm interface information " + type(e).__name__ + ": "+  str(e))
1180             except vimconn.vimconnNotFoundException as e:
1181                 self.logger.error("Exception getting vm status: %s", str(e))
1182                 vm['status'] = "DELETED"
1183                 vm['error_msg'] = str(e)
1184             except vimconn.vimconnException as e:
1185                 self.logger.error("Exception getting vm status: %s", str(e))
1186                 vm['status'] = "VIM_ERROR"
1187                 vm['error_msg'] = str(e)
1188             vm_dict[vm_id] = vm
1189         return vm_dict
1190
1191     def action_vminstance(self, vm_id, action_dict):
1192         '''Send and action over a VM instance from VIM
1193         Returns the vm_id if the action was successfully sent to the VIM'''
1194         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
1195         try:
1196             self._reload_connection()
1197             server = self.nova.servers.find(id=vm_id)
1198             if "start" in action_dict:
1199                 if action_dict["start"]=="rebuild":
1200                     server.rebuild()
1201                 else:
1202                     if server.status=="PAUSED":
1203                         server.unpause()
1204                     elif server.status=="SUSPENDED":
1205                         server.resume()
1206                     elif server.status=="SHUTOFF":
1207                         server.start()
1208             elif "pause" in action_dict:
1209                 server.pause()
1210             elif "resume" in action_dict:
1211                 server.resume()
1212             elif "shutoff" in action_dict or "shutdown" in action_dict:
1213                 server.stop()
1214             elif "forceOff" in action_dict:
1215                 server.stop() #TODO
1216             elif "terminate" in action_dict:
1217                 server.delete()
1218             elif "createImage" in action_dict:
1219                 server.create_image()
1220                 #"path":path_schema,
1221                 #"description":description_schema,
1222                 #"name":name_schema,
1223                 #"metadata":metadata_schema,
1224                 #"imageRef": id_schema,
1225                 #"disk": {"oneOf":[{"type": "null"}, {"type":"string"}] },
1226             elif "rebuild" in action_dict:
1227                 server.rebuild(server.image['id'])
1228             elif "reboot" in action_dict:
1229                 server.reboot() #reboot_type='SOFT'
1230             elif "console" in action_dict:
1231                 console_type = action_dict["console"]
1232                 if console_type == None or console_type == "novnc":
1233                     console_dict = server.get_vnc_console("novnc")
1234                 elif console_type == "xvpvnc":
1235                     console_dict = server.get_vnc_console(console_type)
1236                 elif console_type == "rdp-html5":
1237                     console_dict = server.get_rdp_console(console_type)
1238                 elif console_type == "spice-html5":
1239                     console_dict = server.get_spice_console(console_type)
1240                 else:
1241                     raise vimconn.vimconnException("console type '{}' not allowed".format(console_type),
1242                                                    http_code=vimconn.HTTP_Bad_Request)
1243                 try:
1244                     console_url = console_dict["console"]["url"]
1245                     #parse console_url
1246                     protocol_index = console_url.find("//")
1247                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1248                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1249                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1250                         raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1251                     console_dict2={"protocol": console_url[0:protocol_index],
1252                                   "server":   console_url[protocol_index+2 : port_index],
1253                                   "port":     int(console_url[port_index+1 : suffix_index]),
1254                                   "suffix":   console_url[suffix_index+1:]
1255                                   }
1256                     return console_dict2
1257                 except Exception as e:
1258                     raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1259
1260             return vm_id
1261         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1262             self._format_exception(e)
1263         #TODO insert exception vimconn.HTTP_Unauthorized
1264
1265 #NOT USED FUNCTIONS
1266
1267     def new_external_port(self, port_data):
1268         #TODO openstack if needed
1269         '''Adds a external port to VIM'''
1270         '''Returns the port identifier'''
1271         return -vimconn.HTTP_Internal_Server_Error, "osconnector.new_external_port() not implemented"
1272
1273     def connect_port_network(self, port_id, network_id, admin=False):
1274         #TODO openstack if needed
1275         '''Connects a external port to a network'''
1276         '''Returns status code of the VIM response'''
1277         return -vimconn.HTTP_Internal_Server_Error, "osconnector.connect_port_network() not implemented"
1278
1279     def new_user(self, user_name, user_passwd, tenant_id=None):
1280         '''Adds a new user to openstack VIM'''
1281         '''Returns the user identifier'''
1282         self.logger.debug("osconnector: Adding a new user to VIM")
1283         try:
1284             self._reload_connection()
1285             user=self.keystone.users.create(user_name, user_passwd, tenant_id=tenant_id)
1286             #self.keystone.tenants.add_user(self.k_creds["username"], #role)
1287             return user.id
1288         except ksExceptions.ConnectionError as e:
1289             error_value=-vimconn.HTTP_Bad_Request
1290             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1291         except ksExceptions.ClientException as e: #TODO remove
1292             error_value=-vimconn.HTTP_Bad_Request
1293             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1294         #TODO insert exception vimconn.HTTP_Unauthorized
1295         #if reaching here is because an exception
1296         if self.debug:
1297             self.logger.debug("new_user " + error_text)
1298         return error_value, error_text
1299
1300     def delete_user(self, user_id):
1301         '''Delete a user from openstack VIM'''
1302         '''Returns the user identifier'''
1303         if self.debug:
1304             print "osconnector: Deleting  a  user from VIM"
1305         try:
1306             self._reload_connection()
1307             self.keystone.users.delete(user_id)
1308             return 1, user_id
1309         except ksExceptions.ConnectionError as e:
1310             error_value=-vimconn.HTTP_Bad_Request
1311             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1312         except ksExceptions.NotFound as e:
1313             error_value=-vimconn.HTTP_Not_Found
1314             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1315         except ksExceptions.ClientException as e: #TODO remove
1316             error_value=-vimconn.HTTP_Bad_Request
1317             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1318         #TODO insert exception vimconn.HTTP_Unauthorized
1319         #if reaching here is because an exception
1320         if self.debug:
1321             print "delete_tenant " + error_text
1322         return error_value, error_text
1323
1324     def get_hosts_info(self):
1325         '''Get the information of deployed hosts
1326         Returns the hosts content'''
1327         if self.debug:
1328             print "osconnector: Getting Host info from VIM"
1329         try:
1330             h_list=[]
1331             self._reload_connection()
1332             hypervisors = self.nova.hypervisors.list()
1333             for hype in hypervisors:
1334                 h_list.append( hype.to_dict() )
1335             return 1, {"hosts":h_list}
1336         except nvExceptions.NotFound as e:
1337             error_value=-vimconn.HTTP_Not_Found
1338             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1339         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1340             error_value=-vimconn.HTTP_Bad_Request
1341             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1342         #TODO insert exception vimconn.HTTP_Unauthorized
1343         #if reaching here is because an exception
1344         if self.debug:
1345             print "get_hosts_info " + error_text
1346         return error_value, error_text
1347
1348     def get_hosts(self, vim_tenant):
1349         '''Get the hosts and deployed instances
1350         Returns the hosts content'''
1351         r, hype_dict = self.get_hosts_info()
1352         if r<0:
1353             return r, hype_dict
1354         hypervisors = hype_dict["hosts"]
1355         try:
1356             servers = self.nova.servers.list()
1357             for hype in hypervisors:
1358                 for server in servers:
1359                     if server.to_dict()['OS-EXT-SRV-ATTR:hypervisor_hostname']==hype['hypervisor_hostname']:
1360                         if 'vm' in hype:
1361                             hype['vm'].append(server.id)
1362                         else:
1363                             hype['vm'] = [server.id]
1364             return 1, hype_dict
1365         except nvExceptions.NotFound as e:
1366             error_value=-vimconn.HTTP_Not_Found
1367             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1368         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1369             error_value=-vimconn.HTTP_Bad_Request
1370             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1371         #TODO insert exception vimconn.HTTP_Unauthorized
1372         #if reaching here is because an exception
1373         if self.debug:
1374             print "get_hosts " + error_text
1375         return error_value, error_text
1376
1377