osm_ro/vimconn_openstack.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 osconnector implements all the methods to interact with openstack using the python-client.
  26 '''
  27 __author__="Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research"
  28 __date__ ="$22-jun-2014 11:19:29$"
  29
  30 import vimconn
  31 import json
  32 import yaml
  33 import logging
  34 import netaddr
  35 import time
  36 import yaml
  37 import random
  38
  39 from novaclient import client as nClient, exceptions as nvExceptions
  40 from keystoneauth1.identity import v2, v3
  41 from keystoneauth1 import session
  42 import keystoneclient.exceptions as ksExceptions
  43 import keystoneclient.v3.client as ksClient_v3
  44 import keystoneclient.v2_0.client as ksClient_v2
  45 from glanceclient import client as glClient
  46 import glanceclient.client as gl1Client
  47 import glanceclient.exc as gl1Exceptions
  48 from  cinderclient import client as cClient
  49 from httplib import HTTPException
  50 from neutronclient.neutron import client as neClient
  51 from neutronclient.common import exceptions as neExceptions
  52 from requests.exceptions import ConnectionError
  53
  54 '''contain the openstack virtual machine status to openmano status'''
  55 vmStatus2manoFormat={'ACTIVE':'ACTIVE',
  56                      'PAUSED':'PAUSED',
  57                      'SUSPENDED': 'SUSPENDED',
  58                      'SHUTOFF':'INACTIVE',
  59                      'BUILD':'BUILD',
  60                      'ERROR':'ERROR','DELETED':'DELETED'
  61                      }
  62 netStatus2manoFormat={'ACTIVE':'ACTIVE','PAUSED':'PAUSED','INACTIVE':'INACTIVE','BUILD':'BUILD','ERROR':'ERROR','DELETED':'DELETED'
  63                      }
  64
  65 #global var to have a timeout creating and deleting volumes
  66 volume_timeout = 60
  67 server_timeout = 300
  68
  69 class vimconnector(vimconn.vimconnector):
  70     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,
  71                  log_level=None, config={}, persistent_info={}):
  72         '''using common constructor parameters. In this case
  73         'url' is the keystone authorization url,
  74         'url_admin' is not use
  75         '''
  76         api_version = config.get('APIversion')
  77         if api_version and api_version not in ('v3.3', 'v2.0', '2', '3'):
  78             raise vimconn.vimconnException("Invalid value '{}' for config:APIversion. "
  79                                            "Allowed values are 'v3.3', 'v2.0', '2' or '3'".format(api_version))
  80         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
  81                                       config)
  82
  83         self.insecure = self.config.get("insecure", False)
  84         if not url:
  85             raise TypeError, 'url param can not be NoneType'
  86         self.persistent_info = persistent_info
  87         self.session = persistent_info.get('session', {'reload_client': True})
  88         self.nova = self.session.get('nova')
  89         self.neutron = self.session.get('neutron')
  90         self.cinder = self.session.get('cinder')
  91         self.glance = self.session.get('glance')
  92         self.keystone = self.session.get('keystone')
  93         self.api_version3 = self.session.get('api_version3')
  94
  95         self.logger = logging.getLogger('openmano.vim.openstack')
  96         if log_level:
  97             self.logger.setLevel( getattr(logging, log_level) )
  98
  99     def __getitem__(self, index):
 100         """Get individuals parameters.
 101         Throw KeyError"""
 102         if index == 'project_domain_id':
 103             return self.config.get("project_domain_id")
 104         elif index == 'user_domain_id':
 105             return self.config.get("user_domain_id")
 106         else:
 107             return vimconn.vimconnector.__getitem__(self, index)
 108
 109     def __setitem__(self, index, value):
 110         """Set individuals parameters and it is marked as dirty so to force connection reload.
 111         Throw KeyError"""
 112         if index == 'project_domain_id':
 113             self.config["project_domain_id"] = value
 114         elif index == 'user_domain_id':
 115                 self.config["user_domain_id"] = value
 116         else:
 117             vimconn.vimconnector.__setitem__(self, index, value)
 118         self.session['reload_client'] = True
 119
 120     def _reload_connection(self):
 121         '''Called before any operation, it check if credentials has changed
 122         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
 123         '''
 124         #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
 125         if self.session['reload_client']:
 126             if self.config.get('APIversion'):
 127                 self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3'
 128             else:  # get from ending auth_url that end with v3 or with v2.0
 129                 self.api_version3 =  self.url.split("/")[-1] == "v3"
 130             self.session['api_version3'] = self.api_version3
 131             if self.api_version3:
 132                 auth = v3.Password(auth_url=self.url,
 133                                    username=self.user,
 134                                    password=self.passwd,
 135                                    project_name=self.tenant_name,
 136                                    project_id=self.tenant_id,
 137                                    project_domain_id=self.config.get('project_domain_id', 'default'),
 138                                    user_domain_id=self.config.get('user_domain_id', 'default'))
 139             else:
 140                 auth = v2.Password(auth_url=self.url,
 141                                    username=self.user,
 142                                    password=self.passwd,
 143                                    tenant_name=self.tenant_name,
 144                                    tenant_id=self.tenant_id)
 145             sess = session.Session(auth=auth, verify=not self.insecure)
 146             if self.api_version3:
 147                 self.keystone = ksClient_v3.Client(session=sess)
 148             else:
 149                 self.keystone = ksClient_v2.Client(session=sess)
 150             self.session['keystone'] = self.keystone
 151             self.nova = self.session['nova'] = nClient.Client("2.1", session=sess)
 152             self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess)
 153             self.cinder = self.session['cinder'] = cClient.Client(2, session=sess)
 154             self.glance = self.session['glance'] = glClient.Client(2, session=sess)
 155             self.session['reload_client'] = False
 156             self.persistent_info['session'] = self.session
 157
 158     def __net_os2mano(self, net_list_dict):
 159         '''Transform the net openstack format to mano format
 160         net_list_dict can be a list of dict or a single dict'''
 161         if type(net_list_dict) is dict:
 162             net_list_=(net_list_dict,)
 163         elif type(net_list_dict) is list:
 164             net_list_=net_list_dict
 165         else:
 166             raise TypeError("param net_list_dict must be a list or a dictionary")
 167         for net in net_list_:
 168             if net.get('provider:network_type') == "vlan":
 169                 net['type']='data'
 170             else:
 171                 net['type']='bridge'
 172
 173     def _format_exception(self, exception):
 174         '''Transform a keystone, nova, neutron  exception into a vimconn exception'''
 175         if isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
 176                                   ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed
 177                                   )):
 178             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 179         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
 180                                     neExceptions.NeutronException, nvExceptions.BadRequest)):
 181             raise vimconn.vimconnUnexpectedResponse(type(exception).__name__ + ": " + str(exception))
 182         elif isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound)):
 183             raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception))
 184         elif isinstance(exception, nvExceptions.Conflict):
 185             raise vimconn.vimconnConflictException(type(exception).__name__ + ": " + str(exception))
 186         elif isinstance(exception, vimconn.vimconnException):
 187             raise
 188         else:  # ()
 189             self.logger.error("General Exception " + str(exception), exc_info=True)
 190             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 191
 192     def get_tenant_list(self, filter_dict={}):
 193         '''Obtain tenants of VIM
 194         filter_dict can contain the following keys:
 195             name: filter by tenant name
 196             id: filter by tenant uuid/id
 197             <other VIM specific>
 198         Returns the tenant list of dictionaries: [{'name':'<name>, 'id':'<id>, ...}, ...]
 199         '''
 200         self.logger.debug("Getting tenants from VIM filter: '%s'", str(filter_dict))
 201         try:
 202             self._reload_connection()
 203             if self.api_version3:
 204                 project_class_list = self.keystone.projects.list(name=filter_dict.get("name"))
 205             else:
 206                 project_class_list = self.keystone.tenants.findall(**filter_dict)
 207             project_list=[]
 208             for project in project_class_list:
 209                 if filter_dict.get('id') and filter_dict["id"] != project.id:
 210                     continue
 211                 project_list.append(project.to_dict())
 212             return project_list
 213         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e:
 214             self._format_exception(e)
 215
 216     def new_tenant(self, tenant_name, tenant_description):
 217         '''Adds a new tenant to openstack VIM. Returns the tenant identifier'''
 218         self.logger.debug("Adding a new tenant name: %s", tenant_name)
 219         try:
 220             self._reload_connection()
 221             if self.api_version3:
 222                 project = self.keystone.projects.create(tenant_name, self.config.get("project_domain_id", "default"),
 223                                                         description=tenant_description, is_domain=False)
 224             else:
 225                 project = self.keystone.tenants.create(tenant_name, tenant_description)
 226             return project.id
 227         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 228             self._format_exception(e)
 229
 230     def delete_tenant(self, tenant_id):
 231         '''Delete a tenant from openstack VIM. Returns the old tenant identifier'''
 232         self.logger.debug("Deleting tenant %s from VIM", tenant_id)
 233         try:
 234             self._reload_connection()
 235             if self.api_version3:
 236                 self.keystone.projects.delete(tenant_id)
 237             else:
 238                 self.keystone.tenants.delete(tenant_id)
 239             return tenant_id
 240         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 241             self._format_exception(e)
 242
 243     def new_network(self,net_name, net_type, ip_profile=None, shared=False, vlan=None):
 244         '''Adds a tenant network to VIM. Returns the network identifier'''
 245         self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type)
 246         #self.logger.debug(">>>>>>>>>>>>>>>>>> IP profile %s", str(ip_profile))
 247         try:
 248             new_net = None
 249             self._reload_connection()
 250             network_dict = {'name': net_name, 'admin_state_up': True}
 251             if net_type=="data" or net_type=="ptp":
 252                 if self.config.get('dataplane_physical_net') == None:
 253                     raise vimconn.vimconnConflictException("You must provide a 'dataplane_physical_net' at config value before creating sriov network")
 254                 network_dict["provider:physical_network"] = self.config['dataplane_physical_net'] #"physnet_sriov" #TODO physical
 255                 network_dict["provider:network_type"]     = "vlan"
 256                 if vlan!=None:
 257                     network_dict["provider:network_type"] = vlan
 258             network_dict["shared"]=shared
 259             new_net=self.neutron.create_network({'network':network_dict})
 260             #print new_net
 261             #create subnetwork, even if there is no profile
 262             if not ip_profile:
 263                 ip_profile = {}
 264             if 'subnet_address' not in ip_profile:
 265                 #Fake subnet is required
 266                 subnet_rand = random.randint(0, 255)
 267                 ip_profile['subnet_address'] = "192.168.{}.0/24".format(subnet_rand)
 268             if 'ip_version' not in ip_profile:
 269                 ip_profile['ip_version'] = "IPv4"
 270             subnet = {"name":net_name+"-subnet",
 271                     "network_id": new_net["network"]["id"],
 272                     "ip_version": 4 if ip_profile['ip_version']=="IPv4" else 6,
 273                     "cidr": ip_profile['subnet_address']
 274                     }
 275             # Gateway should be set to None if not needed. Otherwise openstack assigns one by default
 276             subnet['gateway_ip'] = ip_profile.get('gateway_address')
 277             if ip_profile.get('dns_address'):
 278                 subnet['dns_nameservers'] = ip_profile['dns_address'].split(";")
 279             if 'dhcp_enabled' in ip_profile:
 280                 subnet['enable_dhcp'] = False if ip_profile['dhcp_enabled']=="false" else True
 281             if 'dhcp_start_address' in ip_profile:
 282                 subnet['allocation_pools'] = []
 283                 subnet['allocation_pools'].append(dict())
 284                 subnet['allocation_pools'][0]['start'] = ip_profile['dhcp_start_address']
 285             if 'dhcp_count' in ip_profile:
 286                 #parts = ip_profile['dhcp_start_address'].split('.')
 287                 #ip_int = (int(parts[0]) << 24) + (int(parts[1]) << 16) + (int(parts[2]) << 8) + int(parts[3])
 288                 ip_int = int(netaddr.IPAddress(ip_profile['dhcp_start_address']))
 289                 ip_int += ip_profile['dhcp_count'] - 1
 290                 ip_str = str(netaddr.IPAddress(ip_int))
 291                 subnet['allocation_pools'][0]['end'] = ip_str
 292             #self.logger.debug(">>>>>>>>>>>>>>>>>> Subnet: %s", str(subnet))
 293             self.neutron.create_subnet({"subnet": subnet} )
 294             return new_net["network"]["id"]
 295         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 296             if new_net:
 297                 self.neutron.delete_network(new_net['network']['id'])
 298             self._format_exception(e)
 299
 300     def get_network_list(self, filter_dict={}):
 301         '''Obtain tenant networks of VIM
 302         Filter_dict can be:
 303             name: network name
 304             id: network uuid
 305             shared: boolean
 306             tenant_id: tenant
 307             admin_state_up: boolean
 308             status: 'ACTIVE'
 309         Returns the network list of dictionaries
 310         '''
 311         self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict))
 312         try:
 313             self._reload_connection()
 314             if self.api_version3 and "tenant_id" in filter_dict:
 315                 filter_dict['project_id'] = filter_dict.pop('tenant_id') #TODO check
 316             net_dict=self.neutron.list_networks(**filter_dict)
 317             net_list=net_dict["networks"]
 318             self.__net_os2mano(net_list)
 319             return net_list
 320         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 321             self._format_exception(e)
 322
 323     def get_network(self, net_id):
 324         '''Obtain details of network from VIM
 325         Returns the network information from a network id'''
 326         self.logger.debug(" Getting tenant network %s from VIM", net_id)
 327         filter_dict={"id": net_id}
 328         net_list = self.get_network_list(filter_dict)
 329         if len(net_list)==0:
 330             raise vimconn.vimconnNotFoundException("Network '{}' not found".format(net_id))
 331         elif len(net_list)>1:
 332             raise vimconn.vimconnConflictException("Found more than one network with this criteria")
 333         net = net_list[0]
 334         subnets=[]
 335         for subnet_id in net.get("subnets", () ):
 336             try:
 337                 subnet = self.neutron.show_subnet(subnet_id)
 338             except Exception as e:
 339                 self.logger.error("osconnector.get_network(): Error getting subnet %s %s" % (net_id, str(e)))
 340                 subnet = {"id": subnet_id, "fault": str(e)}
 341             subnets.append(subnet)
 342         net["subnets"] = subnets
 343         net["encapsulation"] = net.get('provider:network_type')
 344         net["segmentation_id"] = net.get('provider:segmentation_id')
 345         return net
 346
 347     def delete_network(self, net_id):
 348         '''Deletes a tenant network from VIM. Returns the old network identifier'''
 349         self.logger.debug("Deleting network '%s' from VIM", net_id)
 350         try:
 351             self._reload_connection()
 352             #delete VM ports attached to this networks before the network
 353             ports = self.neutron.list_ports(network_id=net_id)
 354             for p in ports['ports']:
 355                 try:
 356                     self.neutron.delete_port(p["id"])
 357                 except Exception as e:
 358                     self.logger.error("Error deleting port %s: %s", p["id"], str(e))
 359             self.neutron.delete_network(net_id)
 360             return net_id
 361         except (neExceptions.ConnectionFailed, neExceptions.NetworkNotFoundClient, neExceptions.NeutronException,
 362                 ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 363             self._format_exception(e)
 364
 365     def refresh_nets_status(self, net_list):
 366         '''Get the status of the networks
 367            Params: the list of network identifiers
 368            Returns a dictionary with:
 369                 net_id:         #VIM id of this network
 370                     status:     #Mandatory. Text with one of:
 371                                 #  DELETED (not found at vim)
 372                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 373                                 #  OTHER (Vim reported other status not understood)
 374                                 #  ERROR (VIM indicates an ERROR status)
 375                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 376                                 #  BUILD (on building process)
 377                                 #
 378                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 379                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 380
 381         '''
 382         net_dict={}
 383         for net_id in net_list:
 384             net = {}
 385             try:
 386                 net_vim = self.get_network(net_id)
 387                 if net_vim['status'] in netStatus2manoFormat:
 388                     net["status"] = netStatus2manoFormat[ net_vim['status'] ]
 389                 else:
 390                     net["status"] = "OTHER"
 391                     net["error_msg"] = "VIM status reported " + net_vim['status']
 392
 393                 if net['status'] == "ACTIVE" and not net_vim['admin_state_up']:
 394                     net['status'] = 'DOWN'
 395                 try:
 396                     net['vim_info'] = yaml.safe_dump(net_vim, default_flow_style=True, width=256)
 397                 except yaml.representer.RepresenterError:
 398                     net['vim_info'] = str(net_vim)
 399                 if net_vim.get('fault'):  #TODO
 400                     net['error_msg'] = str(net_vim['fault'])
 401             except vimconn.vimconnNotFoundException as e:
 402                 self.logger.error("Exception getting net status: %s", str(e))
 403                 net['status'] = "DELETED"
 404                 net['error_msg'] = str(e)
 405             except vimconn.vimconnException as e:
 406                 self.logger.error("Exception getting net status: %s", str(e))
 407                 net['status'] = "VIM_ERROR"
 408                 net['error_msg'] = str(e)
 409             net_dict[net_id] = net
 410         return net_dict
 411
 412     def get_flavor(self, flavor_id):
 413         '''Obtain flavor details from the  VIM. Returns the flavor dict details'''
 414         self.logger.debug("Getting flavor '%s'", flavor_id)
 415         try:
 416             self._reload_connection()
 417             flavor = self.nova.flavors.find(id=flavor_id)
 418             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 419             return flavor.to_dict()
 420         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 421             self._format_exception(e)
 422
 423     def get_flavor_id_from_data(self, flavor_dict):
 424         """Obtain flavor id that match the flavor description
 425            Returns the flavor_id or raises a vimconnNotFoundException
 426            flavor_dict: contains the required ram, vcpus, disk
 427            If 'use_existing_flavors' is set to True at config, the closer flavor that provides same or more ram, vcpus
 428                 and disk is returned. Otherwise a flavor with exactly same ram, vcpus and disk is returned or a
 429                 vimconnNotFoundException is raised
 430         """
 431         exact_match = False if self.config.get('use_existing_flavors') else True
 432         try:
 433             self._reload_connection()
 434             flavor_candidate_id = None
 435             flavor_candidate_data = (10000, 10000, 10000)
 436             flavor_target = (flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"])
 437             # numa=None
 438             numas = flavor_dict.get("extended", {}).get("numas")
 439             if numas:
 440                 #TODO
 441                 raise vimconn.vimconnNotFoundException("Flavor with EPA still not implemted")
 442                 # if len(numas) > 1:
 443                 #     raise vimconn.vimconnNotFoundException("Cannot find any flavor with more than one numa")
 444                 # numa=numas[0]
 445                 # numas = extended.get("numas")
 446             for flavor in self.nova.flavors.list():
 447                 epa = flavor.get_keys()
 448                 if epa:
 449                     continue
 450                     # TODO
 451                 flavor_data = (flavor.ram, flavor.vcpus, flavor.disk)
 452                 if flavor_data == flavor_target:
 453                     return flavor.id
 454                 elif not exact_match and flavor_target < flavor_data < flavor_candidate_data:
 455                     flavor_candidate_id = flavor.id
 456                     flavor_candidate_data = flavor_data
 457             if not exact_match and flavor_candidate_id:
 458                 return flavor_candidate_id
 459             raise vimconn.vimconnNotFoundException("Cannot find any flavor matching '{}'".format(str(flavor_dict)))
 460         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 461             self._format_exception(e)
 462
 463
 464     def new_flavor(self, flavor_data, change_name_if_used=True):
 465         '''Adds a tenant flavor to openstack VIM
 466         if change_name_if_used is True, it will change name in case of conflict, because it is not supported name repetition
 467         Returns the flavor identifier
 468         '''
 469         self.logger.debug("Adding flavor '%s'", str(flavor_data))
 470         retry=0
 471         max_retries=3
 472         name_suffix = 0
 473         name=flavor_data['name']
 474         while retry<max_retries:
 475             retry+=1
 476             try:
 477                 self._reload_connection()
 478                 if change_name_if_used:
 479                     #get used names
 480                     fl_names=[]
 481                     fl=self.nova.flavors.list()
 482                     for f in fl:
 483                         fl_names.append(f.name)
 484                     while name in fl_names:
 485                         name_suffix += 1
 486                         name = flavor_data['name']+"-" + str(name_suffix)
 487
 488                 ram = flavor_data.get('ram',64)
 489                 vcpus = flavor_data.get('vcpus',1)
 490                 numa_properties=None
 491
 492                 extended = flavor_data.get("extended")
 493                 if extended:
 494                     numas=extended.get("numas")
 495                     if numas:
 496                         numa_nodes = len(numas)
 497                         if numa_nodes > 1:
 498                             return -1, "Can not add flavor with more than one numa"
 499                         numa_properties = {"hw:numa_nodes":str(numa_nodes)}
 500                         numa_properties["hw:mem_page_size"] = "large"
 501                         numa_properties["hw:cpu_policy"] = "dedicated"
 502                         numa_properties["hw:numa_mempolicy"] = "strict"
 503                         for numa in numas:
 504                             #overwrite ram and vcpus
 505                             ram = numa['memory']*1024
 506                             #See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/implemented/virt-driver-cpu-thread-pinning.html
 507                             if 'paired-threads' in numa:
 508                                 vcpus = numa['paired-threads']*2
 509                                 #cpu_thread_policy "require" implies that the compute node must have an STM architecture
 510                                 numa_properties["hw:cpu_thread_policy"] = "require"
 511                                 numa_properties["hw:cpu_policy"] = "dedicated"
 512                             elif 'cores' in numa:
 513                                 vcpus = numa['cores']
 514                                 # cpu_thread_policy "prefer" implies that the host must not have an SMT architecture, or a non-SMT architecture will be emulated
 515                                 numa_properties["hw:cpu_thread_policy"] = "isolate"
 516                                 numa_properties["hw:cpu_policy"] = "dedicated"
 517                             elif 'threads' in numa:
 518                                 vcpus = numa['threads']
 519                                 # cpu_thread_policy "prefer" implies that the host may or may not have an SMT architecture
 520                                 numa_properties["hw:cpu_thread_policy"] = "prefer"
 521                                 numa_properties["hw:cpu_policy"] = "dedicated"
 522                             # for interface in numa.get("interfaces",() ):
 523                             #     if interface["dedicated"]=="yes":
 524                             #         raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
 525                             #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"' when a way to connect it is available
 526
 527                 #create flavor
 528                 new_flavor=self.nova.flavors.create(name,
 529                                 ram,
 530                                 vcpus,
 531                                 flavor_data.get('disk',1),
 532                                 is_public=flavor_data.get('is_public', True)
 533                             )
 534                 #add metadata
 535                 if numa_properties:
 536                     new_flavor.set_keys(numa_properties)
 537                 return new_flavor.id
 538             except nvExceptions.Conflict as e:
 539                 if change_name_if_used and retry < max_retries:
 540                     continue
 541                 self._format_exception(e)
 542             #except nvExceptions.BadRequest as e:
 543             except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 544                 self._format_exception(e)
 545
 546     def delete_flavor(self,flavor_id):
 547         '''Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
 548         '''
 549         try:
 550             self._reload_connection()
 551             self.nova.flavors.delete(flavor_id)
 552             return flavor_id
 553         #except nvExceptions.BadRequest as e:
 554         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 555             self._format_exception(e)
 556
 557     def new_image(self,image_dict):
 558         '''
 559         Adds a tenant image to VIM. imge_dict is a dictionary with:
 560             name: name
 561             disk_format: qcow2, vhd, vmdk, raw (by default), ...
 562             location: path or URI
 563             public: "yes" or "no"
 564             metadata: metadata of the image
 565         Returns the image_id
 566         '''
 567         # ALF TODO: revise and change for the new method or session
 568         #using version 1 of glance client
 569         glancev1 = gl1Client.Client('1',self.glance_endpoint, token=self.keystone.auth_token, **self.k_creds)  #TODO check k_creds vs n_creds
 570         retry=0
 571         max_retries=3
 572         while retry<max_retries:
 573             retry+=1
 574             try:
 575                 self._reload_connection()
 576                 #determine format  http://docs.openstack.org/developer/glance/formats.html
 577                 if "disk_format" in image_dict:
 578                     disk_format=image_dict["disk_format"]
 579                 else: #autodiscover based on extension
 580                     if image_dict['location'][-6:]==".qcow2":
 581                         disk_format="qcow2"
 582                     elif image_dict['location'][-4:]==".vhd":
 583                         disk_format="vhd"
 584                     elif image_dict['location'][-5:]==".vmdk":
 585                         disk_format="vmdk"
 586                     elif image_dict['location'][-4:]==".vdi":
 587                         disk_format="vdi"
 588                     elif image_dict['location'][-4:]==".iso":
 589                         disk_format="iso"
 590                     elif image_dict['location'][-4:]==".aki":
 591                         disk_format="aki"
 592                     elif image_dict['location'][-4:]==".ari":
 593                         disk_format="ari"
 594                     elif image_dict['location'][-4:]==".ami":
 595                         disk_format="ami"
 596                     else:
 597                         disk_format="raw"
 598                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
 599                 if image_dict['location'][0:4]=="http":
 600                     new_image = glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 601                             container_format="bare", location=image_dict['location'], disk_format=disk_format)
 602                 else: #local path
 603                     with open(image_dict['location']) as fimage:
 604                         new_image = glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 605                             container_format="bare", data=fimage, disk_format=disk_format)
 606                 #insert metadata. We cannot use 'new_image.properties.setdefault'
 607                 #because nova and glance are "INDEPENDENT" and we are using nova for reading metadata
 608                 new_image_nova=self.nova.images.find(id=new_image.id)
 609                 new_image_nova.metadata.setdefault('location',image_dict['location'])
 610                 metadata_to_load = image_dict.get('metadata')
 611                 if metadata_to_load:
 612                     for k,v in yaml.load(metadata_to_load).iteritems():
 613                         new_image_nova.metadata.setdefault(k,v)
 614                 return new_image.id
 615             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
 616                 self._format_exception(e)
 617             except (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 618                 if retry==max_retries:
 619                     continue
 620                 self._format_exception(e)
 621             except IOError as e:  #can not open the file
 622                 raise vimconn.vimconnConnectionException(type(e).__name__ + ": " + str(e)+ " for " + image_dict['location'],
 623                                                          http_code=vimconn.HTTP_Bad_Request)
 624
 625     def delete_image(self, image_id):
 626         '''Deletes a tenant image from openstack VIM. Returns the old id
 627         '''
 628         try:
 629             self._reload_connection()
 630             self.nova.images.delete(image_id)
 631             return image_id
 632         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e: #TODO remove
 633             self._format_exception(e)
 634
 635     def get_image_id_from_path(self, path):
 636         '''Get the image id from image path in the VIM database. Returns the image_id'''
 637         try:
 638             self._reload_connection()
 639             images = self.nova.images.list()
 640             for image in images:
 641                 if image.metadata.get("location")==path:
 642                     return image.id
 643             raise vimconn.vimconnNotFoundException("image with location '{}' not found".format( path))
 644         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 645             self._format_exception(e)
 646
 647     def get_image_list(self, filter_dict={}):
 648         '''Obtain tenant images from VIM
 649         Filter_dict can be:
 650             id: image id
 651             name: image name
 652             checksum: image checksum
 653         Returns the image list of dictionaries:
 654             [{<the fields at Filter_dict plus some VIM specific>}, ...]
 655             List can be empty
 656         '''
 657         self.logger.debug("Getting image list from VIM filter: '%s'", str(filter_dict))
 658         try:
 659             self._reload_connection()
 660             filter_dict_os=filter_dict.copy()
 661             #First we filter by the available filter fields: name, id. The others are removed.
 662             filter_dict_os.pop('checksum',None)
 663             image_list=self.nova.images.findall(**filter_dict_os)
 664             if len(image_list)==0:
 665                 return []
 666             #Then we filter by the rest of filter fields: checksum
 667             filtered_list = []
 668             for image in image_list:
 669                 image_class=self.glance.images.get(image.id)
 670                 if 'checksum' not in filter_dict or image_class['checksum']==filter_dict.get('checksum'):
 671                     filtered_list.append(image_class.copy())
 672             return filtered_list
 673         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 674             self._format_exception(e)
 675
 676     def __wait_for_vm(self, vm_id, status):
 677         """wait until vm is in the desired status and return True.
 678         If the VM gets in ERROR status, return false.
 679         If the timeout is reached generate an exception"""
 680         elapsed_time = 0
 681         while elapsed_time < server_timeout:
 682             vm_status = self.nova.servers.get(vm_id).status
 683             if vm_status == status:
 684                 return True
 685             if vm_status == 'ERROR':
 686                 return False
 687             time.sleep(1)
 688             elapsed_time += 1
 689
 690         # if we exceeded the timeout rollback
 691         if elapsed_time >= server_timeout:
 692             raise vimconn.vimconnException('Timeout waiting for instance ' + vm_id + ' to get ' + status,
 693                                            http_code=vimconn.HTTP_Request_Timeout)
 694
 695     def new_vminstance(self,name,description,start,image_id,flavor_id,net_list,cloud_config=None,disk_list=None):
 696         '''Adds a VM instance to VIM
 697         Params:
 698             start: indicates if VM must start or boot in pause mode. Ignored
 699             image_id,flavor_id: iamge and flavor uuid
 700             net_list: list of interfaces, each one is a dictionary with:
 701                 name:
 702                 net_id: network uuid to connect
 703                 vpci: virtual vcpi to assign, ignored because openstack lack #TODO
 704                 model: interface model, ignored #TODO
 705                 mac_address: used for  SR-IOV ifaces #TODO for other types
 706                 use: 'data', 'bridge',  'mgmt'
 707                 type: 'virtual', 'PF', 'VF', 'VFnotShared'
 708                 vim_id: filled/added by this function
 709                 floating_ip: True/False (or it can be None)
 710                 #TODO ip, security groups
 711         Returns the instance identifier
 712         '''
 713         self.logger.debug("new_vminstance input: image='%s' flavor='%s' nics='%s'",image_id, flavor_id,str(net_list))
 714         try:
 715             server = None
 716             metadata={}
 717             net_list_vim=[]
 718             external_network=[]     # list of external networks to be connected to instance, later on used to create floating_ip
 719             no_secured_ports = []   # List of port-is with port-security disabled
 720             self._reload_connection()
 721             metadata_vpci={}   # For a specific neutron plugin
 722             block_device_mapping = None
 723             for net in net_list:
 724                 if not net.get("net_id"): #skip non connected iface
 725                     continue
 726
 727                 port_dict={
 728                     "network_id": net["net_id"],
 729                     "name": net.get("name"),
 730                     "admin_state_up": True
 731                 }
 732                 if net["type"]=="virtual":
 733                     if "vpci" in net:
 734                         metadata_vpci[ net["net_id"] ] = [[ net["vpci"], "" ]]
 735                 elif net["type"]=="VF": # for VF
 736                     if "vpci" in net:
 737                         if "VF" not in metadata_vpci:
 738                             metadata_vpci["VF"]=[]
 739                         metadata_vpci["VF"].append([ net["vpci"], "" ])
 740                     port_dict["binding:vnic_type"]="direct"
 741                 else:  # For PT
 742                     if "vpci" in net:
 743                         if "PF" not in metadata_vpci:
 744                             metadata_vpci["PF"]=[]
 745                         metadata_vpci["PF"].append([ net["vpci"], "" ])
 746                     port_dict["binding:vnic_type"]="direct-physical"
 747                 if not port_dict["name"]:
 748                     port_dict["name"]=name
 749                 if net.get("mac_address"):
 750                     port_dict["mac_address"]=net["mac_address"]
 751                 new_port = self.neutron.create_port({"port": port_dict })
 752                 net["mac_adress"] = new_port["port"]["mac_address"]
 753                 net["vim_id"] = new_port["port"]["id"]
 754                 # if try to use a network without subnetwork, it will return a emtpy list
 755                 fixed_ips = new_port["port"].get("fixed_ips")
 756                 if fixed_ips:
 757                     net["ip"] = fixed_ips[0].get("ip_address")
 758                 else:
 759                     net["ip"] = None
 760                 net_list_vim.append({"port-id": new_port["port"]["id"]})
 761
 762                 if net.get('floating_ip', False):
 763                     net['exit_on_floating_ip_error'] = True
 764                     external_network.append(net)
 765                 elif net['use'] == 'mgmt' and self.config.get('use_floating_ip'):
 766                     net['exit_on_floating_ip_error'] = False
 767                     external_network.append(net)
 768
 769                 # If port security is disabled when the port has not yet been attached to the VM, then all vm traffic is dropped.
 770                 # As a workaround we wait until the VM is active and then disable the port-security
 771                 if net.get("port_security") == False:
 772                     no_secured_ports.append(new_port["port"]["id"])
 773
 774             if metadata_vpci:
 775                 metadata = {"pci_assignement": json.dumps(metadata_vpci)}
 776                 if len(metadata["pci_assignement"]) >255:
 777                     #limit the metadata size
 778                     #metadata["pci_assignement"] = metadata["pci_assignement"][0:255]
 779                     self.logger.warn("Metadata deleted since it exceeds the expected length (255) ")
 780                     metadata = {}
 781
 782             self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s' metadata %s",
 783                               name, image_id, flavor_id, str(net_list_vim), description, str(metadata))
 784
 785             security_groups   = self.config.get('security_groups')
 786             if type(security_groups) is str:
 787                 security_groups = ( security_groups, )
 788             #cloud config
 789             userdata=None
 790             config_drive = None
 791             if isinstance(cloud_config, dict):
 792                 if cloud_config.get("user-data"):
 793                     userdata=cloud_config["user-data"]
 794                 if cloud_config.get("boot-data-drive") != None:
 795                     config_drive = cloud_config["boot-data-drive"]
 796                 if cloud_config.get("config-files") or cloud_config.get("users") or cloud_config.get("key-pairs"):
 797                     if userdata:
 798                         raise vimconn.vimconnConflictException("Cloud-config cannot contain both 'userdata' and 'config-files'/'users'/'key-pairs'")
 799                     userdata_dict={}
 800                     #default user
 801                     if cloud_config.get("key-pairs"):
 802                         userdata_dict["ssh-authorized-keys"] = cloud_config["key-pairs"]
 803                         userdata_dict["users"] = [{"default": None, "ssh-authorized-keys": cloud_config["key-pairs"] }]
 804                     if cloud_config.get("users"):
 805                         if "users" not in userdata_dict:
 806                             userdata_dict["users"] = [ "default" ]
 807                         for user in cloud_config["users"]:
 808                             user_info = {
 809                                 "name" : user["name"],
 810                                 "sudo": "ALL = (ALL)NOPASSWD:ALL"
 811                             }
 812                             if "user-info" in user:
 813                                 user_info["gecos"] = user["user-info"]
 814                             if user.get("key-pairs"):
 815                                 user_info["ssh-authorized-keys"] = user["key-pairs"]
 816                             userdata_dict["users"].append(user_info)
 817
 818                     if cloud_config.get("config-files"):
 819                         userdata_dict["write_files"] = []
 820                         for file in cloud_config["config-files"]:
 821                             file_info = {
 822                                 "path" : file["dest"],
 823                                 "content": file["content"]
 824                             }
 825                             if file.get("encoding"):
 826                                 file_info["encoding"] = file["encoding"]
 827                             if file.get("permissions"):
 828                                 file_info["permissions"] = file["permissions"]
 829                             if file.get("owner"):
 830                                 file_info["owner"] = file["owner"]
 831                             userdata_dict["write_files"].append(file_info)
 832                     userdata = "#cloud-config\n"
 833                     userdata += yaml.safe_dump(userdata_dict, indent=4, default_flow_style=False)
 834                 self.logger.debug("userdata: %s", userdata)
 835             elif isinstance(cloud_config, str):
 836                 userdata = cloud_config
 837
 838             #Create additional volumes in case these are present in disk_list
 839             base_disk_index = ord('b')
 840             if disk_list != None:
 841                 block_device_mapping = {}
 842                 for disk in disk_list:
 843                     if 'image_id' in disk:
 844                         volume = self.cinder.volumes.create(size = disk['size'],name = name + '_vd' +
 845                                     chr(base_disk_index), imageRef = disk['image_id'])
 846                     else:
 847                         volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
 848                                     chr(base_disk_index))
 849                     block_device_mapping['_vd' +  chr(base_disk_index)] = volume.id
 850                     base_disk_index += 1
 851
 852                 #wait until volumes are with status available
 853                 keep_waiting = True
 854                 elapsed_time = 0
 855                 while keep_waiting and elapsed_time < volume_timeout:
 856                     keep_waiting = False
 857                     for volume_id in block_device_mapping.itervalues():
 858                         if self.cinder.volumes.get(volume_id).status != 'available':
 859                             keep_waiting = True
 860                     if keep_waiting:
 861                         time.sleep(1)
 862                         elapsed_time += 1
 863
 864                 #if we exceeded the timeout rollback
 865                 if elapsed_time >= volume_timeout:
 866                     #delete the volumes we just created
 867                     for volume_id in block_device_mapping.itervalues():
 868                         self.cinder.volumes.delete(volume_id)
 869
 870                     #delete ports we just created
 871                     for net_item  in net_list_vim:
 872                         if 'port-id' in net_item:
 873                             self.neutron.delete_port(net_item['port-id'])
 874
 875                     raise vimconn.vimconnException('Timeout creating volumes for instance ' + name,
 876                                                    http_code=vimconn.HTTP_Request_Timeout)
 877
 878             self.logger.debug("nova.servers.create({}, {}, {}, nics={}, meta={}, security_groups={}," \
 879                                               "availability_zone={}, key_name={}, userdata={}, config_drive={}, " \
 880                                               "block_device_mapping={})".format(name, image_id, flavor_id, net_list_vim,
 881                                                 metadata, security_groups, self.config.get('availability_zone'),
 882                                                 self.config.get('keypair'), userdata, config_drive, block_device_mapping))
 883             server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim, meta=metadata,
 884                                               security_groups=security_groups,
 885                                               availability_zone=self.config.get('availability_zone'),
 886                                               key_name=self.config.get('keypair'),
 887                                               userdata=userdata,
 888                                               config_drive=config_drive,
 889                                               block_device_mapping=block_device_mapping
 890                                               )  # , description=description)
 891
 892             # Previously mentioned workaround to wait until the VM is active and then disable the port-security
 893             if no_secured_ports:
 894                 self.__wait_for_vm(server.id, 'ACTIVE')
 895
 896             for port_id in no_secured_ports:
 897                 try:
 898                     self.neutron.update_port(port_id, {"port": {"port_security_enabled": False, "security_groups": None} })
 899
 900                 except Exception as e:
 901                     self.logger.error("It was not possible to disable port security for port {}".format(port_id))
 902                     self.delete_vminstance(server.id)
 903                     raise
 904
 905             #print "DONE :-)", server
 906             pool_id = None
 907             floating_ips = self.neutron.list_floatingips().get("floatingips", ())
 908
 909             if external_network:
 910                 self.__wait_for_vm(server.id, 'ACTIVE')
 911
 912             for floating_network in external_network:
 913                 try:
 914                     assigned = False
 915                     while(assigned == False):
 916                         if floating_ips:
 917                             ip = floating_ips.pop(0)
 918                             if not ip.get("port_id", False) and ip.get('tenant_id') == server.tenant_id:
 919                                 free_floating_ip = ip.get("floating_ip_address")
 920                                 try:
 921                                     fix_ip = floating_network.get('ip')
 922                                     server.add_floating_ip(free_floating_ip, fix_ip)
 923                                     assigned = True
 924                                 except Exception as e:
 925                                     raise vimconn.vimconnException(type(e).__name__ + ": Cannot create floating_ip "+  str(e), http_code=vimconn.HTTP_Conflict)
 926                         else:
 927                             #Find the external network
 928                             external_nets = list()
 929                             for net in self.neutron.list_networks()['networks']:
 930                                 if net['router:external']:
 931                                         external_nets.append(net)
 932
 933                             if len(external_nets) == 0:
 934                                 raise vimconn.vimconnException("Cannot create floating_ip automatically since no external "
 935                                                                "network is present",
 936                                                                 http_code=vimconn.HTTP_Conflict)
 937                             if len(external_nets) > 1:
 938                                 raise vimconn.vimconnException("Cannot create floating_ip automatically since multiple "
 939                                                                "external networks are present",
 940                                                                http_code=vimconn.HTTP_Conflict)
 941
 942                             pool_id = external_nets[0].get('id')
 943                             param = {'floatingip': {'floating_network_id': pool_id, 'tenant_id': server.tenant_id}}
 944                             try:
 945                                 #self.logger.debug("Creating floating IP")
 946                                 new_floating_ip = self.neutron.create_floatingip(param)
 947                                 free_floating_ip = new_floating_ip['floatingip']['floating_ip_address']
 948                                 fix_ip = floating_network.get('ip')
 949                                 server.add_floating_ip(free_floating_ip, fix_ip)
 950                                 assigned=True
 951                             except Exception as e:
 952                                 raise vimconn.vimconnException(type(e).__name__ + ": Cannot assign floating_ip "+  str(e), http_code=vimconn.HTTP_Conflict)
 953                 except Exception as e:
 954                     if not floating_network['exit_on_floating_ip_error']:
 955                         self.logger.warn("Cannot create floating_ip. %s", str(e))
 956                         continue
 957                     raise
 958
 959             return server.id
 960 #        except nvExceptions.NotFound as e:
 961 #            error_value=-vimconn.HTTP_Not_Found
 962 #            error_text= "vm instance %s not found" % vm_id
 963 #        except TypeError as e:
 964 #            raise vimconn.vimconnException(type(e).__name__ + ": "+  str(e), http_code=vimconn.HTTP_Bad_Request)
 965
 966         except Exception as e:
 967             # delete the volumes we just created
 968             if block_device_mapping:
 969                 for volume_id in block_device_mapping.itervalues():
 970                     self.cinder.volumes.delete(volume_id)
 971
 972             # Delete the VM
 973             if server != None:
 974                 self.delete_vminstance(server.id)
 975             else:
 976                 # delete ports we just created
 977                 for net_item in net_list_vim:
 978                     if 'port-id' in net_item:
 979                         self.neutron.delete_port(net_item['port-id'])
 980
 981             self._format_exception(e)
 982
 983     def get_vminstance(self,vm_id):
 984         '''Returns the VM instance information from VIM'''
 985         #self.logger.debug("Getting VM from VIM")
 986         try:
 987             self._reload_connection()
 988             server = self.nova.servers.find(id=vm_id)
 989             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 990             return server.to_dict()
 991         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
 992             self._format_exception(e)
 993
 994     def get_vminstance_console(self,vm_id, console_type="vnc"):
 995         '''
 996         Get a console for the virtual machine
 997         Params:
 998             vm_id: uuid of the VM
 999             console_type, can be:
1000                 "novnc" (by default), "xvpvnc" for VNC types,
1001                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
1002         Returns dict with the console parameters:
1003                 protocol: ssh, ftp, http, https, ...
1004                 server:   usually ip address
1005                 port:     the http, ssh, ... port
1006                 suffix:   extra text, e.g. the http path and query string
1007         '''
1008         self.logger.debug("Getting VM CONSOLE from VIM")
1009         try:
1010             self._reload_connection()
1011             server = self.nova.servers.find(id=vm_id)
1012             if console_type == None or console_type == "novnc":
1013                 console_dict = server.get_vnc_console("novnc")
1014             elif console_type == "xvpvnc":
1015                 console_dict = server.get_vnc_console(console_type)
1016             elif console_type == "rdp-html5":
1017                 console_dict = server.get_rdp_console(console_type)
1018             elif console_type == "spice-html5":
1019                 console_dict = server.get_spice_console(console_type)
1020             else:
1021                 raise vimconn.vimconnException("console type '{}' not allowed".format(console_type), http_code=vimconn.HTTP_Bad_Request)
1022
1023             console_dict1 = console_dict.get("console")
1024             if console_dict1:
1025                 console_url = console_dict1.get("url")
1026                 if console_url:
1027                     #parse console_url
1028                     protocol_index = console_url.find("//")
1029                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1030                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1031                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1032                         return -vimconn.HTTP_Internal_Server_Error, "Unexpected response from VIM"
1033                     console_dict={"protocol": console_url[0:protocol_index],
1034                                   "server":   console_url[protocol_index+2:port_index],
1035                                   "port":     console_url[port_index:suffix_index],
1036                                   "suffix":   console_url[suffix_index+1:]
1037                                   }
1038                     protocol_index += 2
1039                     return console_dict
1040             raise vimconn.vimconnUnexpectedResponse("Unexpected response from VIM")
1041
1042         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.BadRequest, ConnectionError) as e:
1043             self._format_exception(e)
1044
1045     def delete_vminstance(self, vm_id):
1046         '''Removes a VM instance from VIM. Returns the old identifier
1047         '''
1048         #print "osconnector: Getting VM from VIM"
1049         try:
1050             self._reload_connection()
1051             #delete VM ports attached to this networks before the virtual machine
1052             ports = self.neutron.list_ports(device_id=vm_id)
1053             for p in ports['ports']:
1054                 try:
1055                     self.neutron.delete_port(p["id"])
1056                 except Exception as e:
1057                     self.logger.error("Error deleting port: " + type(e).__name__ + ": "+  str(e))
1058
1059             #commented because detaching the volumes makes the servers.delete not work properly ?!?
1060             #dettach volumes attached
1061             server = self.nova.servers.get(vm_id)
1062             volumes_attached_dict = server._info['os-extended-volumes:volumes_attached']
1063             #for volume in volumes_attached_dict:
1064             #    self.cinder.volumes.detach(volume['id'])
1065
1066             self.nova.servers.delete(vm_id)
1067
1068             #delete volumes.
1069             #Although having detached them should have them  in active status
1070             #we ensure in this loop
1071             keep_waiting = True
1072             elapsed_time = 0
1073             while keep_waiting and elapsed_time < volume_timeout:
1074                 keep_waiting = False
1075                 for volume in volumes_attached_dict:
1076                     if self.cinder.volumes.get(volume['id']).status != 'available':
1077                         keep_waiting = True
1078                     else:
1079                         self.cinder.volumes.delete(volume['id'])
1080                 if keep_waiting:
1081                     time.sleep(1)
1082                     elapsed_time += 1
1083
1084             return vm_id
1085         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
1086             self._format_exception(e)
1087         #TODO insert exception vimconn.HTTP_Unauthorized
1088         #if reaching here is because an exception
1089
1090     def refresh_vms_status(self, vm_list):
1091         '''Get the status of the virtual machines and their interfaces/ports
1092            Params: the list of VM identifiers
1093            Returns a dictionary with:
1094                 vm_id:          #VIM id of this Virtual Machine
1095                     status:     #Mandatory. Text with one of:
1096                                 #  DELETED (not found at vim)
1097                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1098                                 #  OTHER (Vim reported other status not understood)
1099                                 #  ERROR (VIM indicates an ERROR status)
1100                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1101                                 #  CREATING (on building process), ERROR
1102                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
1103                                 #
1104                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
1105                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
1106                     interfaces:
1107                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
1108                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
1109                         vim_net_id:       #network id where this interface is connected
1110                         vim_interface_id: #interface/port VIM id
1111                         ip_address:       #null, or text with IPv4, IPv6 address
1112                         compute_node:     #identification of compute node where PF,VF interface is allocated
1113                         pci:              #PCI address of the NIC that hosts the PF,VF
1114                         vlan:             #physical VLAN used for VF
1115         '''
1116         vm_dict={}
1117         self.logger.debug("refresh_vms status: Getting tenant VM instance information from VIM")
1118         for vm_id in vm_list:
1119             vm={}
1120             try:
1121                 vm_vim = self.get_vminstance(vm_id)
1122                 if vm_vim['status'] in vmStatus2manoFormat:
1123                     vm['status']    =  vmStatus2manoFormat[ vm_vim['status'] ]
1124                 else:
1125                     vm['status']    = "OTHER"
1126                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
1127                 try:
1128                     vm['vim_info']  = yaml.safe_dump(vm_vim, default_flow_style=True, width=256)
1129                 except yaml.representer.RepresenterError:
1130                     vm['vim_info'] = str(vm_vim)
1131                 vm["interfaces"] = []
1132                 if vm_vim.get('fault'):
1133                     vm['error_msg'] = str(vm_vim['fault'])
1134                 #get interfaces
1135                 try:
1136                     self._reload_connection()
1137                     port_dict=self.neutron.list_ports(device_id=vm_id)
1138                     for port in port_dict["ports"]:
1139                         interface={}
1140                         try:
1141                             interface['vim_info'] = yaml.safe_dump(port, default_flow_style=True, width=256)
1142                         except yaml.representer.RepresenterError:
1143                             interface['vim_info'] = str(port)
1144                         interface["mac_address"] = port.get("mac_address")
1145                         interface["vim_net_id"] = port["network_id"]
1146                         interface["vim_interface_id"] = port["id"]
1147                         # check if OS-EXT-SRV-ATTR:host is there,
1148                         # in case of non-admin credentials, it will be missing
1149                         if vm_vim.get('OS-EXT-SRV-ATTR:host'):
1150                             interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host']
1151                         interface["pci"] = None
1152
1153                         # check if binding:profile is there,
1154                         # in case of non-admin credentials, it will be missing
1155                         if port.get('binding:profile'):
1156                             if port['binding:profile'].get('pci_slot'):
1157                                 # TODO: At the moment sr-iov pci addresses are converted to PF pci addresses by setting the slot to 0x00
1158                                 # TODO: This is just a workaround valid for niantinc. Find a better way to do so
1159                                 #   CHANGE DDDD:BB:SS.F to DDDD:BB:00.(F%2)   assuming there are 2 ports per nic
1160                                 pci = port['binding:profile']['pci_slot']
1161                                 # interface["pci"] = pci[:-4] + "00." + str(int(pci[-1]) % 2)
1162                                 interface["pci"] = pci
1163                         interface["vlan"] = None
1164                         #if network is of type vlan and port is of type direct (sr-iov) then set vlan id
1165                         network = self.neutron.show_network(port["network_id"])
1166                         if network['network'].get('provider:network_type') == 'vlan' and \
1167                             port.get("binding:vnic_type") == "direct":
1168                             interface["vlan"] = network['network'].get('provider:segmentation_id')
1169                         ips=[]
1170                         #look for floating ip address
1171                         floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
1172                         if floating_ip_dict.get("floatingips"):
1173                             ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address") )
1174
1175                         for subnet in port["fixed_ips"]:
1176                             ips.append(subnet["ip_address"])
1177                         interface["ip_address"] = ";".join(ips)
1178                         vm["interfaces"].append(interface)
1179                 except Exception as e:
1180                     self.logger.error("Error getting vm interface information " + type(e).__name__ + ": "+  str(e))
1181             except vimconn.vimconnNotFoundException as e:
1182                 self.logger.error("Exception getting vm status: %s", str(e))
1183                 vm['status'] = "DELETED"
1184                 vm['error_msg'] = str(e)
1185             except vimconn.vimconnException as e:
1186                 self.logger.error("Exception getting vm status: %s", str(e))
1187                 vm['status'] = "VIM_ERROR"
1188                 vm['error_msg'] = str(e)
1189             vm_dict[vm_id] = vm
1190         return vm_dict
1191
1192     def action_vminstance(self, vm_id, action_dict):
1193         '''Send and action over a VM instance from VIM
1194         Returns the vm_id if the action was successfully sent to the VIM'''
1195         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
1196         try:
1197             self._reload_connection()
1198             server = self.nova.servers.find(id=vm_id)
1199             if "start" in action_dict:
1200                 if action_dict["start"]=="rebuild":
1201                     server.rebuild()
1202                 else:
1203                     if server.status=="PAUSED":
1204                         server.unpause()
1205                     elif server.status=="SUSPENDED":
1206                         server.resume()
1207                     elif server.status=="SHUTOFF":
1208                         server.start()
1209             elif "pause" in action_dict:
1210                 server.pause()
1211             elif "resume" in action_dict:
1212                 server.resume()
1213             elif "shutoff" in action_dict or "shutdown" in action_dict:
1214                 server.stop()
1215             elif "forceOff" in action_dict:
1216                 server.stop() #TODO
1217             elif "terminate" in action_dict:
1218                 server.delete()
1219             elif "createImage" in action_dict:
1220                 server.create_image()
1221                 #"path":path_schema,
1222                 #"description":description_schema,
1223                 #"name":name_schema,
1224                 #"metadata":metadata_schema,
1225                 #"imageRef": id_schema,
1226                 #"disk": {"oneOf":[{"type": "null"}, {"type":"string"}] },
1227             elif "rebuild" in action_dict:
1228                 server.rebuild(server.image['id'])
1229             elif "reboot" in action_dict:
1230                 server.reboot() #reboot_type='SOFT'
1231             elif "console" in action_dict:
1232                 console_type = action_dict["console"]
1233                 if console_type == None or console_type == "novnc":
1234                     console_dict = server.get_vnc_console("novnc")
1235                 elif console_type == "xvpvnc":
1236                     console_dict = server.get_vnc_console(console_type)
1237                 elif console_type == "rdp-html5":
1238                     console_dict = server.get_rdp_console(console_type)
1239                 elif console_type == "spice-html5":
1240                     console_dict = server.get_spice_console(console_type)
1241                 else:
1242                     raise vimconn.vimconnException("console type '{}' not allowed".format(console_type),
1243                                                    http_code=vimconn.HTTP_Bad_Request)
1244                 try:
1245                     console_url = console_dict["console"]["url"]
1246                     #parse console_url
1247                     protocol_index = console_url.find("//")
1248                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1249                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1250                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1251                         raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1252                     console_dict2={"protocol": console_url[0:protocol_index],
1253                                   "server":   console_url[protocol_index+2 : port_index],
1254                                   "port":     int(console_url[port_index+1 : suffix_index]),
1255                                   "suffix":   console_url[suffix_index+1:]
1256                                   }
1257                     return console_dict2
1258                 except Exception as e:
1259                     raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1260
1261             return vm_id
1262         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1263             self._format_exception(e)
1264         #TODO insert exception vimconn.HTTP_Unauthorized
1265
1266 #NOT USED FUNCTIONS
1267
1268     def new_external_port(self, port_data):
1269         #TODO openstack if needed
1270         '''Adds a external port to VIM'''
1271         '''Returns the port identifier'''
1272         return -vimconn.HTTP_Internal_Server_Error, "osconnector.new_external_port() not implemented"
1273
1274     def connect_port_network(self, port_id, network_id, admin=False):
1275         #TODO openstack if needed
1276         '''Connects a external port to a network'''
1277         '''Returns status code of the VIM response'''
1278         return -vimconn.HTTP_Internal_Server_Error, "osconnector.connect_port_network() not implemented"
1279
1280     def new_user(self, user_name, user_passwd, tenant_id=None):
1281         '''Adds a new user to openstack VIM'''
1282         '''Returns the user identifier'''
1283         self.logger.debug("osconnector: Adding a new user to VIM")
1284         try:
1285             self._reload_connection()
1286             user=self.keystone.users.create(user_name, user_passwd, tenant_id=tenant_id)
1287             #self.keystone.tenants.add_user(self.k_creds["username"], #role)
1288             return user.id
1289         except ksExceptions.ConnectionError as e:
1290             error_value=-vimconn.HTTP_Bad_Request
1291             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1292         except ksExceptions.ClientException as e: #TODO remove
1293             error_value=-vimconn.HTTP_Bad_Request
1294             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1295         #TODO insert exception vimconn.HTTP_Unauthorized
1296         #if reaching here is because an exception
1297         if self.debug:
1298             self.logger.debug("new_user " + error_text)
1299         return error_value, error_text
1300
1301     def delete_user(self, user_id):
1302         '''Delete a user from openstack VIM'''
1303         '''Returns the user identifier'''
1304         if self.debug:
1305             print "osconnector: Deleting  a  user from VIM"
1306         try:
1307             self._reload_connection()
1308             self.keystone.users.delete(user_id)
1309             return 1, user_id
1310         except ksExceptions.ConnectionError as e:
1311             error_value=-vimconn.HTTP_Bad_Request
1312             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1313         except ksExceptions.NotFound as e:
1314             error_value=-vimconn.HTTP_Not_Found
1315             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1316         except ksExceptions.ClientException as e: #TODO remove
1317             error_value=-vimconn.HTTP_Bad_Request
1318             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1319         #TODO insert exception vimconn.HTTP_Unauthorized
1320         #if reaching here is because an exception
1321         if self.debug:
1322             print "delete_tenant " + error_text
1323         return error_value, error_text
1324
1325     def get_hosts_info(self):
1326         '''Get the information of deployed hosts
1327         Returns the hosts content'''
1328         if self.debug:
1329             print "osconnector: Getting Host info from VIM"
1330         try:
1331             h_list=[]
1332             self._reload_connection()
1333             hypervisors = self.nova.hypervisors.list()
1334             for hype in hypervisors:
1335                 h_list.append( hype.to_dict() )
1336             return 1, {"hosts":h_list}
1337         except nvExceptions.NotFound as e:
1338             error_value=-vimconn.HTTP_Not_Found
1339             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1340         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1341             error_value=-vimconn.HTTP_Bad_Request
1342             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1343         #TODO insert exception vimconn.HTTP_Unauthorized
1344         #if reaching here is because an exception
1345         if self.debug:
1346             print "get_hosts_info " + error_text
1347         return error_value, error_text
1348
1349     def get_hosts(self, vim_tenant):
1350         '''Get the hosts and deployed instances
1351         Returns the hosts content'''
1352         r, hype_dict = self.get_hosts_info()
1353         if r<0:
1354             return r, hype_dict
1355         hypervisors = hype_dict["hosts"]
1356         try:
1357             servers = self.nova.servers.list()
1358             for hype in hypervisors:
1359                 for server in servers:
1360                     if server.to_dict()['OS-EXT-SRV-ATTR:hypervisor_hostname']==hype['hypervisor_hostname']:
1361                         if 'vm' in hype:
1362                             hype['vm'].append(server.id)
1363                         else:
1364                             hype['vm'] = [server.id]
1365             return 1, hype_dict
1366         except nvExceptions.NotFound as e:
1367             error_value=-vimconn.HTTP_Not_Found
1368             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1369         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1370             error_value=-vimconn.HTTP_Bad_Request
1371             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1372         #TODO insert exception vimconn.HTTP_Unauthorized
1373         #if reaching here is because an exception
1374         if self.debug:
1375             print "get_hosts " + error_text
1376         return error_value, error_text
1377
1378