osm_ro/vimconn_openstack.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 osconnector implements all the methods to interact with openstack using the python-client.
  26 '''
  27 __author__="Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research"
  28 __date__ ="$22-jun-2014 11:19:29$"
  29
  30 import vimconn
  31 import json
  32 import yaml
  33 import logging
  34 import netaddr
  35 import time
  36 import yaml
  37 import random
  38
  39 from novaclient import client as nClient, exceptions as nvExceptions
  40 from keystoneauth1.identity import v2, v3
  41 from keystoneauth1 import session
  42 import keystoneclient.exceptions as ksExceptions
  43 import keystoneclient.v3.client as ksClient_v3
  44 import keystoneclient.v2_0.client as ksClient_v2
  45 from glanceclient import client as glClient
  46 import glanceclient.client as gl1Client
  47 import glanceclient.exc as gl1Exceptions
  48 from  cinderclient import client as cClient
  49 from httplib import HTTPException
  50 from neutronclient.neutron import client as neClient
  51 from neutronclient.common import exceptions as neExceptions
  52 from requests.exceptions import ConnectionError
  53
  54 '''contain the openstack virtual machine status to openmano status'''
  55 vmStatus2manoFormat={'ACTIVE':'ACTIVE',
  56                      'PAUSED':'PAUSED',
  57                      'SUSPENDED': 'SUSPENDED',
  58                      'SHUTOFF':'INACTIVE',
  59                      'BUILD':'BUILD',
  60                      'ERROR':'ERROR','DELETED':'DELETED'
  61                      }
  62 netStatus2manoFormat={'ACTIVE':'ACTIVE','PAUSED':'PAUSED','INACTIVE':'INACTIVE','BUILD':'BUILD','ERROR':'ERROR','DELETED':'DELETED'
  63                      }
  64
  65 #global var to have a timeout creating and deleting volumes
  66 volume_timeout = 60
  67 server_timeout = 300
  68
  69 class vimconnector(vimconn.vimconnector):
  70     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,
  71                  log_level=None, config={}, persistent_info={}):
  72         '''using common constructor parameters. In this case
  73         'url' is the keystone authorization url,
  74         'url_admin' is not use
  75         '''
  76         api_version = config.get('APIversion')
  77         if api_version and api_version not in ('v3.3', 'v2.0', '2', '3'):
  78             raise vimconn.vimconnException("Invalid value '{}' for config:APIversion. "
  79                                            "Allowed values are 'v3.3', 'v2.0', '2' or '3'".format(api_version))
  80         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
  81                                       config)
  82
  83         self.insecure = self.config.get("insecure", False)
  84         if not url:
  85             raise TypeError, 'url param can not be NoneType'
  86         self.persistent_info = persistent_info
  87         self.availability_zone = persistent_info.get('availability_zone', None)
  88         self.session = persistent_info.get('session', {'reload_client': True})
  89         self.nova = self.session.get('nova')
  90         self.neutron = self.session.get('neutron')
  91         self.cinder = self.session.get('cinder')
  92         self.glance = self.session.get('glance')
  93         self.glancev1 = self.session.get('glancev1')
  94         self.keystone = self.session.get('keystone')
  95         self.api_version3 = self.session.get('api_version3')
  96
  97         self.logger = logging.getLogger('openmano.vim.openstack')
  98         if log_level:
  99             self.logger.setLevel( getattr(logging, log_level) )
 100
 101     def __getitem__(self, index):
 102         """Get individuals parameters.
 103         Throw KeyError"""
 104         if index == 'project_domain_id':
 105             return self.config.get("project_domain_id")
 106         elif index == 'user_domain_id':
 107             return self.config.get("user_domain_id")
 108         else:
 109             return vimconn.vimconnector.__getitem__(self, index)
 110
 111     def __setitem__(self, index, value):
 112         """Set individuals parameters and it is marked as dirty so to force connection reload.
 113         Throw KeyError"""
 114         if index == 'project_domain_id':
 115             self.config["project_domain_id"] = value
 116         elif index == 'user_domain_id':
 117                 self.config["user_domain_id"] = value
 118         else:
 119             vimconn.vimconnector.__setitem__(self, index, value)
 120         self.session['reload_client'] = True
 121
 122     def _reload_connection(self):
 123         '''Called before any operation, it check if credentials has changed
 124         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
 125         '''
 126         #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
 127         if self.session['reload_client']:
 128             if self.config.get('APIversion'):
 129                 self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3'
 130             else:  # get from ending auth_url that end with v3 or with v2.0
 131                 self.api_version3 =  self.url.split("/")[-1] == "v3"
 132             self.session['api_version3'] = self.api_version3
 133             if self.api_version3:
 134                 auth = v3.Password(auth_url=self.url,
 135                                    username=self.user,
 136                                    password=self.passwd,
 137                                    project_name=self.tenant_name,
 138                                    project_id=self.tenant_id,
 139                                    project_domain_id=self.config.get('project_domain_id', 'default'),
 140                                    user_domain_id=self.config.get('user_domain_id', 'default'))
 141             else:
 142                 auth = v2.Password(auth_url=self.url,
 143                                    username=self.user,
 144                                    password=self.passwd,
 145                                    tenant_name=self.tenant_name,
 146                                    tenant_id=self.tenant_id)
 147             sess = session.Session(auth=auth, verify=not self.insecure)
 148             if self.api_version3:
 149                 self.keystone = ksClient_v3.Client(session=sess)
 150             else:
 151                 self.keystone = ksClient_v2.Client(session=sess)
 152             self.session['keystone'] = self.keystone
 153             # In order to enable microversion functionality an explicit microversion must be specified in 'config'.
 154             # This implementation approach is due to the warning message in
 155             # https://developer.openstack.org/api-guide/compute/microversions.html
 156             # where it is stated that microversion backwards compatibility is not guaranteed and clients should
 157             # always require an specific microversion.
 158             # To be able to use 'device role tagging' functionality define 'microversion: 2.32' in datacenter config
 159             version = self.config.get("microversion")
 160             if not version:
 161                 version = "2.1"
 162             self.nova = self.session['nova'] = nClient.Client(str(version), session=sess)
 163             self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess)
 164             self.cinder = self.session['cinder'] = cClient.Client(2, session=sess)
 165             self.glance = self.session['glance'] = glClient.Client(2, session=sess)
 166             self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess)
 167             self.session['reload_client'] = False
 168             self.persistent_info['session'] = self.session
 169             # add availablity zone info inside  self.persistent_info
 170             self._set_availablity_zones()
 171             self.persistent_info['availability_zone'] = self.availability_zone
 172
 173     def __net_os2mano(self, net_list_dict):
 174         '''Transform the net openstack format to mano format
 175         net_list_dict can be a list of dict or a single dict'''
 176         if type(net_list_dict) is dict:
 177             net_list_=(net_list_dict,)
 178         elif type(net_list_dict) is list:
 179             net_list_=net_list_dict
 180         else:
 181             raise TypeError("param net_list_dict must be a list or a dictionary")
 182         for net in net_list_:
 183             if net.get('provider:network_type') == "vlan":
 184                 net['type']='data'
 185             else:
 186                 net['type']='bridge'
 187
 188     def _format_exception(self, exception):
 189         '''Transform a keystone, nova, neutron  exception into a vimconn exception'''
 190         if isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
 191                                   ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed
 192                                   )):
 193             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 194         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
 195                                     neExceptions.NeutronException, nvExceptions.BadRequest)):
 196             raise vimconn.vimconnUnexpectedResponse(type(exception).__name__ + ": " + str(exception))
 197         elif isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound)):
 198             raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception))
 199         elif isinstance(exception, nvExceptions.Conflict):
 200             raise vimconn.vimconnConflictException(type(exception).__name__ + ": " + str(exception))
 201         elif isinstance(exception, vimconn.vimconnException):
 202             raise
 203         else:  # ()
 204             self.logger.error("General Exception " + str(exception), exc_info=True)
 205             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 206
 207     def get_tenant_list(self, filter_dict={}):
 208         '''Obtain tenants of VIM
 209         filter_dict can contain the following keys:
 210             name: filter by tenant name
 211             id: filter by tenant uuid/id
 212             <other VIM specific>
 213         Returns the tenant list of dictionaries: [{'name':'<name>, 'id':'<id>, ...}, ...]
 214         '''
 215         self.logger.debug("Getting tenants from VIM filter: '%s'", str(filter_dict))
 216         try:
 217             self._reload_connection()
 218             if self.api_version3:
 219                 project_class_list = self.keystone.projects.list(name=filter_dict.get("name"))
 220             else:
 221                 project_class_list = self.keystone.tenants.findall(**filter_dict)
 222             project_list=[]
 223             for project in project_class_list:
 224                 if filter_dict.get('id') and filter_dict["id"] != project.id:
 225                     continue
 226                 project_list.append(project.to_dict())
 227             return project_list
 228         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e:
 229             self._format_exception(e)
 230
 231     def new_tenant(self, tenant_name, tenant_description):
 232         '''Adds a new tenant to openstack VIM. Returns the tenant identifier'''
 233         self.logger.debug("Adding a new tenant name: %s", tenant_name)
 234         try:
 235             self._reload_connection()
 236             if self.api_version3:
 237                 project = self.keystone.projects.create(tenant_name, self.config.get("project_domain_id", "default"),
 238                                                         description=tenant_description, is_domain=False)
 239             else:
 240                 project = self.keystone.tenants.create(tenant_name, tenant_description)
 241             return project.id
 242         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 243             self._format_exception(e)
 244
 245     def delete_tenant(self, tenant_id):
 246         '''Delete a tenant from openstack VIM. Returns the old tenant identifier'''
 247         self.logger.debug("Deleting tenant %s from VIM", tenant_id)
 248         try:
 249             self._reload_connection()
 250             if self.api_version3:
 251                 self.keystone.projects.delete(tenant_id)
 252             else:
 253                 self.keystone.tenants.delete(tenant_id)
 254             return tenant_id
 255         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 256             self._format_exception(e)
 257
 258     def new_network(self,net_name, net_type, ip_profile=None, shared=False, vlan=None):
 259         '''Adds a tenant network to VIM. Returns the network identifier'''
 260         self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type)
 261         #self.logger.debug(">>>>>>>>>>>>>>>>>> IP profile %s", str(ip_profile))
 262         try:
 263             new_net = None
 264             self._reload_connection()
 265             network_dict = {'name': net_name, 'admin_state_up': True}
 266             if net_type=="data" or net_type=="ptp":
 267                 if self.config.get('dataplane_physical_net') == None:
 268                     raise vimconn.vimconnConflictException("You must provide a 'dataplane_physical_net' at config value before creating sriov network")
 269                 network_dict["provider:physical_network"] = self.config['dataplane_physical_net'] #"physnet_sriov" #TODO physical
 270                 network_dict["provider:network_type"]     = "vlan"
 271                 if vlan!=None:
 272                     network_dict["provider:network_type"] = vlan
 273             network_dict["shared"]=shared
 274             new_net=self.neutron.create_network({'network':network_dict})
 275             #print new_net
 276             #create subnetwork, even if there is no profile
 277             if not ip_profile:
 278                 ip_profile = {}
 279             if 'subnet_address' not in ip_profile:
 280                 #Fake subnet is required
 281                 subnet_rand = random.randint(0, 255)
 282                 ip_profile['subnet_address'] = "192.168.{}.0/24".format(subnet_rand)
 283             if 'ip_version' not in ip_profile:
 284                 ip_profile['ip_version'] = "IPv4"
 285             subnet = {"name":net_name+"-subnet",
 286                     "network_id": new_net["network"]["id"],
 287                     "ip_version": 4 if ip_profile['ip_version']=="IPv4" else 6,
 288                     "cidr": ip_profile['subnet_address']
 289                     }
 290             # Gateway should be set to None if not needed. Otherwise openstack assigns one by default
 291             subnet['gateway_ip'] = ip_profile.get('gateway_address')
 292             if ip_profile.get('dns_address'):
 293                 subnet['dns_nameservers'] = ip_profile['dns_address'].split(";")
 294             if 'dhcp_enabled' in ip_profile:
 295                 subnet['enable_dhcp'] = False if ip_profile['dhcp_enabled']=="false" else True
 296             if 'dhcp_start_address' in ip_profile:
 297                 subnet['allocation_pools'] = []
 298                 subnet['allocation_pools'].append(dict())
 299                 subnet['allocation_pools'][0]['start'] = ip_profile['dhcp_start_address']
 300             if 'dhcp_count' in ip_profile:
 301                 #parts = ip_profile['dhcp_start_address'].split('.')
 302                 #ip_int = (int(parts[0]) << 24) + (int(parts[1]) << 16) + (int(parts[2]) << 8) + int(parts[3])
 303                 ip_int = int(netaddr.IPAddress(ip_profile['dhcp_start_address']))
 304                 ip_int += ip_profile['dhcp_count'] - 1
 305                 ip_str = str(netaddr.IPAddress(ip_int))
 306                 subnet['allocation_pools'][0]['end'] = ip_str
 307             #self.logger.debug(">>>>>>>>>>>>>>>>>> Subnet: %s", str(subnet))
 308             self.neutron.create_subnet({"subnet": subnet} )
 309             return new_net["network"]["id"]
 310         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 311             if new_net:
 312                 self.neutron.delete_network(new_net['network']['id'])
 313             self._format_exception(e)
 314
 315     def get_network_list(self, filter_dict={}):
 316         '''Obtain tenant networks of VIM
 317         Filter_dict can be:
 318             name: network name
 319             id: network uuid
 320             shared: boolean
 321             tenant_id: tenant
 322             admin_state_up: boolean
 323             status: 'ACTIVE'
 324         Returns the network list of dictionaries
 325         '''
 326         self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict))
 327         try:
 328             self._reload_connection()
 329             if self.api_version3 and "tenant_id" in filter_dict:
 330                 filter_dict['project_id'] = filter_dict.pop('tenant_id') #TODO check
 331             net_dict=self.neutron.list_networks(**filter_dict)
 332             net_list=net_dict["networks"]
 333             self.__net_os2mano(net_list)
 334             return net_list
 335         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 336             self._format_exception(e)
 337
 338     def get_network(self, net_id):
 339         '''Obtain details of network from VIM
 340         Returns the network information from a network id'''
 341         self.logger.debug(" Getting tenant network %s from VIM", net_id)
 342         filter_dict={"id": net_id}
 343         net_list = self.get_network_list(filter_dict)
 344         if len(net_list)==0:
 345             raise vimconn.vimconnNotFoundException("Network '{}' not found".format(net_id))
 346         elif len(net_list)>1:
 347             raise vimconn.vimconnConflictException("Found more than one network with this criteria")
 348         net = net_list[0]
 349         subnets=[]
 350         for subnet_id in net.get("subnets", () ):
 351             try:
 352                 subnet = self.neutron.show_subnet(subnet_id)
 353             except Exception as e:
 354                 self.logger.error("osconnector.get_network(): Error getting subnet %s %s" % (net_id, str(e)))
 355                 subnet = {"id": subnet_id, "fault": str(e)}
 356             subnets.append(subnet)
 357         net["subnets"] = subnets
 358         net["encapsulation"] = net.get('provider:network_type')
 359         net["segmentation_id"] = net.get('provider:segmentation_id')
 360         return net
 361
 362     def delete_network(self, net_id):
 363         '''Deletes a tenant network from VIM. Returns the old network identifier'''
 364         self.logger.debug("Deleting network '%s' from VIM", net_id)
 365         try:
 366             self._reload_connection()
 367             #delete VM ports attached to this networks before the network
 368             ports = self.neutron.list_ports(network_id=net_id)
 369             for p in ports['ports']:
 370                 try:
 371                     self.neutron.delete_port(p["id"])
 372                 except Exception as e:
 373                     self.logger.error("Error deleting port %s: %s", p["id"], str(e))
 374             self.neutron.delete_network(net_id)
 375             return net_id
 376         except (neExceptions.ConnectionFailed, neExceptions.NetworkNotFoundClient, neExceptions.NeutronException,
 377                 ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 378             self._format_exception(e)
 379
 380     def refresh_nets_status(self, net_list):
 381         '''Get the status of the networks
 382            Params: the list of network identifiers
 383            Returns a dictionary with:
 384                 net_id:         #VIM id of this network
 385                     status:     #Mandatory. Text with one of:
 386                                 #  DELETED (not found at vim)
 387                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 388                                 #  OTHER (Vim reported other status not understood)
 389                                 #  ERROR (VIM indicates an ERROR status)
 390                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 391                                 #  BUILD (on building process)
 392                                 #
 393                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 394                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 395
 396         '''
 397         net_dict={}
 398         for net_id in net_list:
 399             net = {}
 400             try:
 401                 net_vim = self.get_network(net_id)
 402                 if net_vim['status'] in netStatus2manoFormat:
 403                     net["status"] = netStatus2manoFormat[ net_vim['status'] ]
 404                 else:
 405                     net["status"] = "OTHER"
 406                     net["error_msg"] = "VIM status reported " + net_vim['status']
 407
 408                 if net['status'] == "ACTIVE" and not net_vim['admin_state_up']:
 409                     net['status'] = 'DOWN'
 410                 try:
 411                     net['vim_info'] = yaml.safe_dump(net_vim, default_flow_style=True, width=256)
 412                 except yaml.representer.RepresenterError:
 413                     net['vim_info'] = str(net_vim)
 414                 if net_vim.get('fault'):  #TODO
 415                     net['error_msg'] = str(net_vim['fault'])
 416             except vimconn.vimconnNotFoundException as e:
 417                 self.logger.error("Exception getting net status: %s", str(e))
 418                 net['status'] = "DELETED"
 419                 net['error_msg'] = str(e)
 420             except vimconn.vimconnException as e:
 421                 self.logger.error("Exception getting net status: %s", str(e))
 422                 net['status'] = "VIM_ERROR"
 423                 net['error_msg'] = str(e)
 424             net_dict[net_id] = net
 425         return net_dict
 426
 427     def get_flavor(self, flavor_id):
 428         '''Obtain flavor details from the  VIM. Returns the flavor dict details'''
 429         self.logger.debug("Getting flavor '%s'", flavor_id)
 430         try:
 431             self._reload_connection()
 432             flavor = self.nova.flavors.find(id=flavor_id)
 433             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 434             return flavor.to_dict()
 435         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 436             self._format_exception(e)
 437
 438     def get_flavor_id_from_data(self, flavor_dict):
 439         """Obtain flavor id that match the flavor description
 440            Returns the flavor_id or raises a vimconnNotFoundException
 441            flavor_dict: contains the required ram, vcpus, disk
 442            If 'use_existing_flavors' is set to True at config, the closer flavor that provides same or more ram, vcpus
 443                 and disk is returned. Otherwise a flavor with exactly same ram, vcpus and disk is returned or a
 444                 vimconnNotFoundException is raised
 445         """
 446         exact_match = False if self.config.get('use_existing_flavors') else True
 447         try:
 448             self._reload_connection()
 449             flavor_candidate_id = None
 450             flavor_candidate_data = (10000, 10000, 10000)
 451             flavor_target = (flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"])
 452             # numa=None
 453             numas = flavor_dict.get("extended", {}).get("numas")
 454             if numas:
 455                 #TODO
 456                 raise vimconn.vimconnNotFoundException("Flavor with EPA still not implemted")
 457                 # if len(numas) > 1:
 458                 #     raise vimconn.vimconnNotFoundException("Cannot find any flavor with more than one numa")
 459                 # numa=numas[0]
 460                 # numas = extended.get("numas")
 461             for flavor in self.nova.flavors.list():
 462                 epa = flavor.get_keys()
 463                 if epa:
 464                     continue
 465                     # TODO
 466                 flavor_data = (flavor.ram, flavor.vcpus, flavor.disk)
 467                 if flavor_data == flavor_target:
 468                     return flavor.id
 469                 elif not exact_match and flavor_target < flavor_data < flavor_candidate_data:
 470                     flavor_candidate_id = flavor.id
 471                     flavor_candidate_data = flavor_data
 472             if not exact_match and flavor_candidate_id:
 473                 return flavor_candidate_id
 474             raise vimconn.vimconnNotFoundException("Cannot find any flavor matching '{}'".format(str(flavor_dict)))
 475         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 476             self._format_exception(e)
 477
 478
 479     def new_flavor(self, flavor_data, change_name_if_used=True):
 480         '''Adds a tenant flavor to openstack VIM
 481         if change_name_if_used is True, it will change name in case of conflict, because it is not supported name repetition
 482         Returns the flavor identifier
 483         '''
 484         self.logger.debug("Adding flavor '%s'", str(flavor_data))
 485         retry=0
 486         max_retries=3
 487         name_suffix = 0
 488         name=flavor_data['name']
 489         while retry<max_retries:
 490             retry+=1
 491             try:
 492                 self._reload_connection()
 493                 if change_name_if_used:
 494                     #get used names
 495                     fl_names=[]
 496                     fl=self.nova.flavors.list()
 497                     for f in fl:
 498                         fl_names.append(f.name)
 499                     while name in fl_names:
 500                         name_suffix += 1
 501                         name = flavor_data['name']+"-" + str(name_suffix)
 502
 503                 ram = flavor_data.get('ram',64)
 504                 vcpus = flavor_data.get('vcpus',1)
 505                 numa_properties=None
 506
 507                 extended = flavor_data.get("extended")
 508                 if extended:
 509                     numas=extended.get("numas")
 510                     if numas:
 511                         numa_nodes = len(numas)
 512                         if numa_nodes > 1:
 513                             return -1, "Can not add flavor with more than one numa"
 514                         numa_properties = {"hw:numa_nodes":str(numa_nodes)}
 515                         numa_properties["hw:mem_page_size"] = "large"
 516                         numa_properties["hw:cpu_policy"] = "dedicated"
 517                         numa_properties["hw:numa_mempolicy"] = "strict"
 518                         for numa in numas:
 519                             #overwrite ram and vcpus
 520                             ram = numa['memory']*1024
 521                             #See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/implemented/virt-driver-cpu-thread-pinning.html
 522                             if 'paired-threads' in numa:
 523                                 vcpus = numa['paired-threads']*2
 524                                 #cpu_thread_policy "require" implies that the compute node must have an STM architecture
 525                                 numa_properties["hw:cpu_thread_policy"] = "require"
 526                                 numa_properties["hw:cpu_policy"] = "dedicated"
 527                             elif 'cores' in numa:
 528                                 vcpus = numa['cores']
 529                                 # cpu_thread_policy "prefer" implies that the host must not have an SMT architecture, or a non-SMT architecture will be emulated
 530                                 numa_properties["hw:cpu_thread_policy"] = "isolate"
 531                                 numa_properties["hw:cpu_policy"] = "dedicated"
 532                             elif 'threads' in numa:
 533                                 vcpus = numa['threads']
 534                                 # cpu_thread_policy "prefer" implies that the host may or may not have an SMT architecture
 535                                 numa_properties["hw:cpu_thread_policy"] = "prefer"
 536                                 numa_properties["hw:cpu_policy"] = "dedicated"
 537                             # for interface in numa.get("interfaces",() ):
 538                             #     if interface["dedicated"]=="yes":
 539                             #         raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
 540                             #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"' when a way to connect it is available
 541
 542                 #create flavor
 543                 new_flavor=self.nova.flavors.create(name,
 544                                 ram,
 545                                 vcpus,
 546                                 flavor_data.get('disk',1),
 547                                 is_public=flavor_data.get('is_public', True)
 548                             )
 549                 #add metadata
 550                 if numa_properties:
 551                     new_flavor.set_keys(numa_properties)
 552                 return new_flavor.id
 553             except nvExceptions.Conflict as e:
 554                 if change_name_if_used and retry < max_retries:
 555                     continue
 556                 self._format_exception(e)
 557             #except nvExceptions.BadRequest as e:
 558             except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 559                 self._format_exception(e)
 560
 561     def delete_flavor(self,flavor_id):
 562         '''Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
 563         '''
 564         try:
 565             self._reload_connection()
 566             self.nova.flavors.delete(flavor_id)
 567             return flavor_id
 568         #except nvExceptions.BadRequest as e:
 569         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 570             self._format_exception(e)
 571
 572     def new_image(self,image_dict):
 573         '''
 574         Adds a tenant image to VIM. imge_dict is a dictionary with:
 575             name: name
 576             disk_format: qcow2, vhd, vmdk, raw (by default), ...
 577             location: path or URI
 578             public: "yes" or "no"
 579             metadata: metadata of the image
 580         Returns the image_id
 581         '''
 582         retry=0
 583         max_retries=3
 584         while retry<max_retries:
 585             retry+=1
 586             try:
 587                 self._reload_connection()
 588                 #determine format  http://docs.openstack.org/developer/glance/formats.html
 589                 if "disk_format" in image_dict:
 590                     disk_format=image_dict["disk_format"]
 591                 else: #autodiscover based on extension
 592                     if image_dict['location'][-6:]==".qcow2":
 593                         disk_format="qcow2"
 594                     elif image_dict['location'][-4:]==".vhd":
 595                         disk_format="vhd"
 596                     elif image_dict['location'][-5:]==".vmdk":
 597                         disk_format="vmdk"
 598                     elif image_dict['location'][-4:]==".vdi":
 599                         disk_format="vdi"
 600                     elif image_dict['location'][-4:]==".iso":
 601                         disk_format="iso"
 602                     elif image_dict['location'][-4:]==".aki":
 603                         disk_format="aki"
 604                     elif image_dict['location'][-4:]==".ari":
 605                         disk_format="ari"
 606                     elif image_dict['location'][-4:]==".ami":
 607                         disk_format="ami"
 608                     else:
 609                         disk_format="raw"
 610                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
 611                 if image_dict['location'][0:4]=="http":
 612                     new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 613                             container_format="bare", location=image_dict['location'], disk_format=disk_format)
 614                 else: #local path
 615                     with open(image_dict['location']) as fimage:
 616                         new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 617                             container_format="bare", data=fimage, disk_format=disk_format)
 618                 #insert metadata. We cannot use 'new_image.properties.setdefault'
 619                 #because nova and glance are "INDEPENDENT" and we are using nova for reading metadata
 620                 new_image_nova=self.nova.images.find(id=new_image.id)
 621                 new_image_nova.metadata.setdefault('location',image_dict['location'])
 622                 metadata_to_load = image_dict.get('metadata')
 623                 if metadata_to_load:
 624                     for k,v in yaml.load(metadata_to_load).iteritems():
 625                         new_image_nova.metadata.setdefault(k,v)
 626                 return new_image.id
 627             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
 628                 self._format_exception(e)
 629             except (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 630                 if retry==max_retries:
 631                     continue
 632                 self._format_exception(e)
 633             except IOError as e:  #can not open the file
 634                 raise vimconn.vimconnConnectionException(type(e).__name__ + ": " + str(e)+ " for " + image_dict['location'],
 635                                                          http_code=vimconn.HTTP_Bad_Request)
 636
 637     def delete_image(self, image_id):
 638         '''Deletes a tenant image from openstack VIM. Returns the old id
 639         '''
 640         try:
 641             self._reload_connection()
 642             self.nova.images.delete(image_id)
 643             return image_id
 644         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e: #TODO remove
 645             self._format_exception(e)
 646
 647     def get_image_id_from_path(self, path):
 648         '''Get the image id from image path in the VIM database. Returns the image_id'''
 649         try:
 650             self._reload_connection()
 651             images = self.nova.images.list()
 652             for image in images:
 653                 if image.metadata.get("location")==path:
 654                     return image.id
 655             raise vimconn.vimconnNotFoundException("image with location '{}' not found".format( path))
 656         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 657             self._format_exception(e)
 658
 659     def get_image_list(self, filter_dict={}):
 660         '''Obtain tenant images from VIM
 661         Filter_dict can be:
 662             id: image id
 663             name: image name
 664             checksum: image checksum
 665         Returns the image list of dictionaries:
 666             [{<the fields at Filter_dict plus some VIM specific>}, ...]
 667             List can be empty
 668         '''
 669         self.logger.debug("Getting image list from VIM filter: '%s'", str(filter_dict))
 670         try:
 671             self._reload_connection()
 672             filter_dict_os=filter_dict.copy()
 673             #First we filter by the available filter fields: name, id. The others are removed.
 674             filter_dict_os.pop('checksum',None)
 675             image_list=self.nova.images.findall(**filter_dict_os)
 676             if len(image_list)==0:
 677                 return []
 678             #Then we filter by the rest of filter fields: checksum
 679             filtered_list = []
 680             for image in image_list:
 681                 image_class=self.glance.images.get(image.id)
 682                 if 'checksum' not in filter_dict or image_class['checksum']==filter_dict.get('checksum'):
 683                     filtered_list.append(image_class.copy())
 684             return filtered_list
 685         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 686             self._format_exception(e)
 687
 688     def __wait_for_vm(self, vm_id, status):
 689         """wait until vm is in the desired status and return True.
 690         If the VM gets in ERROR status, return false.
 691         If the timeout is reached generate an exception"""
 692         elapsed_time = 0
 693         while elapsed_time < server_timeout:
 694             vm_status = self.nova.servers.get(vm_id).status
 695             if vm_status == status:
 696                 return True
 697             if vm_status == 'ERROR':
 698                 return False
 699             time.sleep(1)
 700             elapsed_time += 1
 701
 702         # if we exceeded the timeout rollback
 703         if elapsed_time >= server_timeout:
 704             raise vimconn.vimconnException('Timeout waiting for instance ' + vm_id + ' to get ' + status,
 705                                            http_code=vimconn.HTTP_Request_Timeout)
 706
 707     def _get_openstack_availablity_zones(self):
 708         """
 709         Get from openstack availability zones available
 710         :return:
 711         """
 712         try:
 713             openstack_availability_zone = self.nova.availability_zones.list()
 714             openstack_availability_zone = [str(zone.zoneName) for zone in openstack_availability_zone
 715                                            if zone.zoneName != 'internal']
 716             return openstack_availability_zone
 717         except Exception as e:
 718             return None
 719
 720     def _set_availablity_zones(self):
 721         """
 722         Set vim availablity zone
 723         :return:
 724         """
 725
 726         if 'availability_zone' in self.config:
 727             vim_availability_zones = self.config.get('availability_zone')
 728             if isinstance(vim_availability_zones, str):
 729                 self.availability_zone = [vim_availability_zones]
 730             elif isinstance(vim_availability_zones, list):
 731                 self.availability_zone = vim_availability_zones
 732         else:
 733             self.availability_zone = self._get_openstack_availablity_zones()
 734
 735     def _get_vm_availability_zone(self, availability_zone_index, availability_zone_list):
 736         """
 737         Return thge availability zone to be used by the created VM.
 738         :return: The VIM availability zone to be used or None
 739         """
 740         if availability_zone_index is None:
 741             if not self.config.get('availability_zone'):
 742                 return None
 743             elif isinstance(self.config.get('availability_zone'), str):
 744                 return self.config['availability_zone']
 745             else:
 746                 # TODO consider using a different parameter at config for default AV and AV list match
 747                 return self.config['availability_zone'][0]
 748
 749         vim_availability_zones = self.availability_zone
 750         # check if VIM offer enough availability zones describe in the VNFD
 751         if vim_availability_zones and len(availability_zone_list) <= len(vim_availability_zones):
 752             # check if all the names of NFV AV match VIM AV names
 753             match_by_index = False
 754             for av in availability_zone_list:
 755                 if av not in vim_availability_zones:
 756                     match_by_index = True
 757                     break
 758             if match_by_index:
 759                 return vim_availability_zones[availability_zone_index]
 760             else:
 761                 return availability_zone_list[availability_zone_index]
 762         else:
 763             raise vimconn.vimconnConflictException("No enough availability zones at VIM for this deployment")
 764
 765     def new_vminstance(self, name, description, start, image_id, flavor_id, net_list, cloud_config=None, disk_list=None,
 766                        availability_zone_index=None, availability_zone_list=None):
 767         '''Adds a VM instance to VIM
 768         Params:
 769             start: indicates if VM must start or boot in pause mode. Ignored
 770             image_id,flavor_id: iamge and flavor uuid
 771             net_list: list of interfaces, each one is a dictionary with:
 772                 name:
 773                 net_id: network uuid to connect
 774                 vpci: virtual vcpi to assign, ignored because openstack lack #TODO
 775                 model: interface model, ignored #TODO
 776                 mac_address: used for  SR-IOV ifaces #TODO for other types
 777                 use: 'data', 'bridge',  'mgmt'
 778                 type: 'virtual', 'PF', 'VF', 'VFnotShared'
 779                 vim_id: filled/added by this function
 780                 floating_ip: True/False (or it can be None)
 781                 'cloud_config': (optional) dictionary with:
 782                 'key-pairs': (optional) list of strings with the public key to be inserted to the default user
 783                 'users': (optional) list of users to be inserted, each item is a dict with:
 784                     'name': (mandatory) user name,
 785                     'key-pairs': (optional) list of strings with the public key to be inserted to the user
 786                 'user-data': (optional) string is a text script to be passed directly to cloud-init
 787                 'config-files': (optional). List of files to be transferred. Each item is a dict with:
 788                     'dest': (mandatory) string with the destination absolute path
 789                     'encoding': (optional, by default text). Can be one of:
 790                         'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
 791                     'content' (mandatory): string with the content of the file
 792                     'permissions': (optional) string with file permissions, typically octal notation '0644'
 793                     'owner': (optional) file owner, string with the format 'owner:group'
 794                 'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk)
 795             'disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
 796                 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
 797                 'size': (mandatory) string with the size of the disk in GB
 798             availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required
 799             availability_zone_list: list of availability zones given by user in the VNFD descriptor.  Ignore if
 800                 availability_zone_index is None
 801                 #TODO ip, security groups
 802         Returns the instance identifier
 803         '''
 804         self.logger.debug("new_vminstance input: image='%s' flavor='%s' nics='%s'",image_id, flavor_id,str(net_list))
 805         try:
 806             server = None
 807             metadata={}
 808             net_list_vim=[]
 809             external_network=[]     # list of external networks to be connected to instance, later on used to create floating_ip
 810             no_secured_ports = []   # List of port-is with port-security disabled
 811             self._reload_connection()
 812             metadata_vpci={}   # For a specific neutron plugin
 813             block_device_mapping = None
 814             for net in net_list:
 815                 if not net.get("net_id"): #skip non connected iface
 816                     continue
 817
 818                 port_dict={
 819                     "network_id": net["net_id"],
 820                     "name": net.get("name"),
 821                     "admin_state_up": True
 822                 }
 823                 if net["type"]=="virtual":
 824                     if "vpci" in net:
 825                         metadata_vpci[ net["net_id"] ] = [[ net["vpci"], "" ]]
 826                 elif net["type"]=="VF": # for VF
 827                     if "vpci" in net:
 828                         if "VF" not in metadata_vpci:
 829                             metadata_vpci["VF"]=[]
 830                         metadata_vpci["VF"].append([ net["vpci"], "" ])
 831                     port_dict["binding:vnic_type"]="direct"
 832                 else:  # For PT
 833                     if "vpci" in net:
 834                         if "PF" not in metadata_vpci:
 835                             metadata_vpci["PF"]=[]
 836                         metadata_vpci["PF"].append([ net["vpci"], "" ])
 837                     port_dict["binding:vnic_type"]="direct-physical"
 838                 if not port_dict["name"]:
 839                     port_dict["name"]=name
 840                 if net.get("mac_address"):
 841                     port_dict["mac_address"]=net["mac_address"]
 842                 new_port = self.neutron.create_port({"port": port_dict })
 843                 net["mac_adress"] = new_port["port"]["mac_address"]
 844                 net["vim_id"] = new_port["port"]["id"]
 845                 # if try to use a network without subnetwork, it will return a emtpy list
 846                 fixed_ips = new_port["port"].get("fixed_ips")
 847                 if fixed_ips:
 848                     net["ip"] = fixed_ips[0].get("ip_address")
 849                 else:
 850                     net["ip"] = None
 851
 852                 port = {"port-id": new_port["port"]["id"]}
 853                 if float(self.nova.api_version.get_string()) >= 2.32:
 854                     port["tag"] = new_port["port"]["name"]
 855                 net_list_vim.append(port)
 856
 857                 if net.get('floating_ip', False):
 858                     net['exit_on_floating_ip_error'] = True
 859                     external_network.append(net)
 860                 elif net['use'] == 'mgmt' and self.config.get('use_floating_ip'):
 861                     net['exit_on_floating_ip_error'] = False
 862                     external_network.append(net)
 863
 864                 # If port security is disabled when the port has not yet been attached to the VM, then all vm traffic is dropped.
 865                 # As a workaround we wait until the VM is active and then disable the port-security
 866                 if net.get("port_security") == False:
 867                     no_secured_ports.append(new_port["port"]["id"])
 868
 869             if metadata_vpci:
 870                 metadata = {"pci_assignement": json.dumps(metadata_vpci)}
 871                 if len(metadata["pci_assignement"]) >255:
 872                     #limit the metadata size
 873                     #metadata["pci_assignement"] = metadata["pci_assignement"][0:255]
 874                     self.logger.warn("Metadata deleted since it exceeds the expected length (255) ")
 875                     metadata = {}
 876
 877             self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s' metadata %s",
 878                               name, image_id, flavor_id, str(net_list_vim), description, str(metadata))
 879
 880             security_groups   = self.config.get('security_groups')
 881             if type(security_groups) is str:
 882                 security_groups = ( security_groups, )
 883             #cloud config
 884             userdata=None
 885             config_drive = None
 886             if isinstance(cloud_config, dict):
 887                 if cloud_config.get("user-data"):
 888                     userdata=cloud_config["user-data"]
 889                 if cloud_config.get("boot-data-drive") != None:
 890                     config_drive = cloud_config["boot-data-drive"]
 891                 if cloud_config.get("config-files") or cloud_config.get("users") or cloud_config.get("key-pairs"):
 892                     if userdata:
 893                         raise vimconn.vimconnConflictException("Cloud-config cannot contain both 'userdata' and 'config-files'/'users'/'key-pairs'")
 894                     userdata_dict={}
 895                     #default user
 896                     if cloud_config.get("key-pairs"):
 897                         userdata_dict["ssh-authorized-keys"] = cloud_config["key-pairs"]
 898                         userdata_dict["users"] = [{"default": None, "ssh-authorized-keys": cloud_config["key-pairs"] }]
 899                     if cloud_config.get("users"):
 900                         if "users" not in userdata_dict:
 901                             userdata_dict["users"] = [ "default" ]
 902                         for user in cloud_config["users"]:
 903                             user_info = {
 904                                 "name" : user["name"],
 905                                 "sudo": "ALL = (ALL)NOPASSWD:ALL"
 906                             }
 907                             if "user-info" in user:
 908                                 user_info["gecos"] = user["user-info"]
 909                             if user.get("key-pairs"):
 910                                 user_info["ssh-authorized-keys"] = user["key-pairs"]
 911                             userdata_dict["users"].append(user_info)
 912
 913                     if cloud_config.get("config-files"):
 914                         userdata_dict["write_files"] = []
 915                         for file in cloud_config["config-files"]:
 916                             file_info = {
 917                                 "path" : file["dest"],
 918                                 "content": file["content"]
 919                             }
 920                             if file.get("encoding"):
 921                                 file_info["encoding"] = file["encoding"]
 922                             if file.get("permissions"):
 923                                 file_info["permissions"] = file["permissions"]
 924                             if file.get("owner"):
 925                                 file_info["owner"] = file["owner"]
 926                             userdata_dict["write_files"].append(file_info)
 927                     userdata = "#cloud-config\n"
 928                     userdata += yaml.safe_dump(userdata_dict, indent=4, default_flow_style=False)
 929                 self.logger.debug("userdata: %s", userdata)
 930             elif isinstance(cloud_config, str):
 931                 userdata = cloud_config
 932
 933             #Create additional volumes in case these are present in disk_list
 934             base_disk_index = ord('b')
 935             if disk_list != None:
 936                 block_device_mapping = {}
 937                 for disk in disk_list:
 938                     if 'image_id' in disk:
 939                         volume = self.cinder.volumes.create(size = disk['size'],name = name + '_vd' +
 940                                     chr(base_disk_index), imageRef = disk['image_id'])
 941                     else:
 942                         volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
 943                                     chr(base_disk_index))
 944                     block_device_mapping['_vd' +  chr(base_disk_index)] = volume.id
 945                     base_disk_index += 1
 946
 947                 #wait until volumes are with status available
 948                 keep_waiting = True
 949                 elapsed_time = 0
 950                 while keep_waiting and elapsed_time < volume_timeout:
 951                     keep_waiting = False
 952                     for volume_id in block_device_mapping.itervalues():
 953                         if self.cinder.volumes.get(volume_id).status != 'available':
 954                             keep_waiting = True
 955                     if keep_waiting:
 956                         time.sleep(1)
 957                         elapsed_time += 1
 958
 959                 #if we exceeded the timeout rollback
 960                 if elapsed_time >= volume_timeout:
 961                     #delete the volumes we just created
 962                     for volume_id in block_device_mapping.itervalues():
 963                         self.cinder.volumes.delete(volume_id)
 964
 965                     #delete ports we just created
 966                     for net_item  in net_list_vim:
 967                         if 'port-id' in net_item:
 968                             self.neutron.delete_port(net_item['port-id'])
 969
 970                     raise vimconn.vimconnException('Timeout creating volumes for instance ' + name,
 971                                                    http_code=vimconn.HTTP_Request_Timeout)
 972             # get availability Zone
 973             vm_av_zone = self._get_vm_availability_zone(availability_zone_index, availability_zone_list)
 974
 975             self.logger.debug("nova.servers.create({}, {}, {}, nics={}, meta={}, security_groups={}, "
 976                               "availability_zone={}, key_name={}, userdata={}, config_drive={}, "
 977                               "block_device_mapping={})".format(name, image_id, flavor_id, net_list_vim, metadata,
 978                                                                 security_groups, vm_av_zone, self.config.get('keypair'),
 979                               userdata, config_drive, block_device_mapping))
 980             server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim, meta=metadata,
 981                                               security_groups=security_groups,
 982                                               availability_zone=vm_av_zone,
 983                                               key_name=self.config.get('keypair'),
 984                                               userdata=userdata,
 985                                               config_drive=config_drive,
 986                                               block_device_mapping=block_device_mapping
 987                                               )  # , description=description)
 988
 989             # Previously mentioned workaround to wait until the VM is active and then disable the port-security
 990             if no_secured_ports:
 991                 self.__wait_for_vm(server.id, 'ACTIVE')
 992
 993             for port_id in no_secured_ports:
 994                 try:
 995                     self.neutron.update_port(port_id, {"port": {"port_security_enabled": False, "security_groups": None} })
 996
 997                 except Exception as e:
 998                     self.logger.error("It was not possible to disable port security for port {}".format(port_id))
 999                     self.delete_vminstance(server.id)
1000                     raise
1001
1002             #print "DONE :-)", server
1003             pool_id = None
1004             floating_ips = self.neutron.list_floatingips().get("floatingips", ())
1005
1006             if external_network:
1007                 self.__wait_for_vm(server.id, 'ACTIVE')
1008
1009             for floating_network in external_network:
1010                 try:
1011                     assigned = False
1012                     while(assigned == False):
1013                         if floating_ips:
1014                             ip = floating_ips.pop(0)
1015                             if not ip.get("port_id", False) and ip.get('tenant_id') == server.tenant_id:
1016                                 free_floating_ip = ip.get("floating_ip_address")
1017                                 try:
1018                                     fix_ip = floating_network.get('ip')
1019                                     server.add_floating_ip(free_floating_ip, fix_ip)
1020                                     assigned = True
1021                                 except Exception as e:
1022                                     raise vimconn.vimconnException(type(e).__name__ + ": Cannot create floating_ip "+  str(e), http_code=vimconn.HTTP_Conflict)
1023                         else:
1024                             #Find the external network
1025                             external_nets = list()
1026                             for net in self.neutron.list_networks()['networks']:
1027                                 if net['router:external']:
1028                                         external_nets.append(net)
1029
1030                             if len(external_nets) == 0:
1031                                 raise vimconn.vimconnException("Cannot create floating_ip automatically since no external "
1032                                                                "network is present",
1033                                                                 http_code=vimconn.HTTP_Conflict)
1034                             if len(external_nets) > 1:
1035                                 raise vimconn.vimconnException("Cannot create floating_ip automatically since multiple "
1036                                                                "external networks are present",
1037                                                                http_code=vimconn.HTTP_Conflict)
1038
1039                             pool_id = external_nets[0].get('id')
1040                             param = {'floatingip': {'floating_network_id': pool_id, 'tenant_id': server.tenant_id}}
1041                             try:
1042                                 #self.logger.debug("Creating floating IP")
1043                                 new_floating_ip = self.neutron.create_floatingip(param)
1044                                 free_floating_ip = new_floating_ip['floatingip']['floating_ip_address']
1045                                 fix_ip = floating_network.get('ip')
1046                                 server.add_floating_ip(free_floating_ip, fix_ip)
1047                                 assigned=True
1048                             except Exception as e:
1049                                 raise vimconn.vimconnException(type(e).__name__ + ": Cannot assign floating_ip "+  str(e), http_code=vimconn.HTTP_Conflict)
1050                 except Exception as e:
1051                     if not floating_network['exit_on_floating_ip_error']:
1052                         self.logger.warn("Cannot create floating_ip. %s", str(e))
1053                         continue
1054                     raise
1055
1056             return server.id
1057 #        except nvExceptions.NotFound as e:
1058 #            error_value=-vimconn.HTTP_Not_Found
1059 #            error_text= "vm instance %s not found" % vm_id
1060 #        except TypeError as e:
1061 #            raise vimconn.vimconnException(type(e).__name__ + ": "+  str(e), http_code=vimconn.HTTP_Bad_Request)
1062
1063         except Exception as e:
1064             # delete the volumes we just created
1065             if block_device_mapping:
1066                 for volume_id in block_device_mapping.itervalues():
1067                     self.cinder.volumes.delete(volume_id)
1068
1069             # Delete the VM
1070             if server != None:
1071                 self.delete_vminstance(server.id)
1072             else:
1073                 # delete ports we just created
1074                 for net_item in net_list_vim:
1075                     if 'port-id' in net_item:
1076                         self.neutron.delete_port(net_item['port-id'])
1077
1078             self._format_exception(e)
1079
1080     def get_vminstance(self,vm_id):
1081         '''Returns the VM instance information from VIM'''
1082         #self.logger.debug("Getting VM from VIM")
1083         try:
1084             self._reload_connection()
1085             server = self.nova.servers.find(id=vm_id)
1086             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
1087             return server.to_dict()
1088         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1089             self._format_exception(e)
1090
1091     def get_vminstance_console(self,vm_id, console_type="vnc"):
1092         '''
1093         Get a console for the virtual machine
1094         Params:
1095             vm_id: uuid of the VM
1096             console_type, can be:
1097                 "novnc" (by default), "xvpvnc" for VNC types,
1098                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
1099         Returns dict with the console parameters:
1100                 protocol: ssh, ftp, http, https, ...
1101                 server:   usually ip address
1102                 port:     the http, ssh, ... port
1103                 suffix:   extra text, e.g. the http path and query string
1104         '''
1105         self.logger.debug("Getting VM CONSOLE from VIM")
1106         try:
1107             self._reload_connection()
1108             server = self.nova.servers.find(id=vm_id)
1109             if console_type == None or console_type == "novnc":
1110                 console_dict = server.get_vnc_console("novnc")
1111             elif console_type == "xvpvnc":
1112                 console_dict = server.get_vnc_console(console_type)
1113             elif console_type == "rdp-html5":
1114                 console_dict = server.get_rdp_console(console_type)
1115             elif console_type == "spice-html5":
1116                 console_dict = server.get_spice_console(console_type)
1117             else:
1118                 raise vimconn.vimconnException("console type '{}' not allowed".format(console_type), http_code=vimconn.HTTP_Bad_Request)
1119
1120             console_dict1 = console_dict.get("console")
1121             if console_dict1:
1122                 console_url = console_dict1.get("url")
1123                 if console_url:
1124                     #parse console_url
1125                     protocol_index = console_url.find("//")
1126                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1127                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1128                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1129                         return -vimconn.HTTP_Internal_Server_Error, "Unexpected response from VIM"
1130                     console_dict={"protocol": console_url[0:protocol_index],
1131                                   "server":   console_url[protocol_index+2:port_index],
1132                                   "port":     console_url[port_index:suffix_index],
1133                                   "suffix":   console_url[suffix_index+1:]
1134                                   }
1135                     protocol_index += 2
1136                     return console_dict
1137             raise vimconn.vimconnUnexpectedResponse("Unexpected response from VIM")
1138
1139         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.BadRequest, ConnectionError) as e:
1140             self._format_exception(e)
1141
1142     def delete_vminstance(self, vm_id):
1143         '''Removes a VM instance from VIM. Returns the old identifier
1144         '''
1145         #print "osconnector: Getting VM from VIM"
1146         try:
1147             self._reload_connection()
1148             #delete VM ports attached to this networks before the virtual machine
1149             ports = self.neutron.list_ports(device_id=vm_id)
1150             for p in ports['ports']:
1151                 try:
1152                     self.neutron.delete_port(p["id"])
1153                 except Exception as e:
1154                     self.logger.error("Error deleting port: " + type(e).__name__ + ": "+  str(e))
1155
1156             #commented because detaching the volumes makes the servers.delete not work properly ?!?
1157             #dettach volumes attached
1158             server = self.nova.servers.get(vm_id)
1159             volumes_attached_dict = server._info['os-extended-volumes:volumes_attached']
1160             #for volume in volumes_attached_dict:
1161             #    self.cinder.volumes.detach(volume['id'])
1162
1163             self.nova.servers.delete(vm_id)
1164
1165             #delete volumes.
1166             #Although having detached them should have them  in active status
1167             #we ensure in this loop
1168             keep_waiting = True
1169             elapsed_time = 0
1170             while keep_waiting and elapsed_time < volume_timeout:
1171                 keep_waiting = False
1172                 for volume in volumes_attached_dict:
1173                     if self.cinder.volumes.get(volume['id']).status != 'available':
1174                         keep_waiting = True
1175                     else:
1176                         self.cinder.volumes.delete(volume['id'])
1177                 if keep_waiting:
1178                     time.sleep(1)
1179                     elapsed_time += 1
1180
1181             return vm_id
1182         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
1183             self._format_exception(e)
1184         #TODO insert exception vimconn.HTTP_Unauthorized
1185         #if reaching here is because an exception
1186
1187     def refresh_vms_status(self, vm_list):
1188         '''Get the status of the virtual machines and their interfaces/ports
1189            Params: the list of VM identifiers
1190            Returns a dictionary with:
1191                 vm_id:          #VIM id of this Virtual Machine
1192                     status:     #Mandatory. Text with one of:
1193                                 #  DELETED (not found at vim)
1194                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1195                                 #  OTHER (Vim reported other status not understood)
1196                                 #  ERROR (VIM indicates an ERROR status)
1197                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1198                                 #  CREATING (on building process), ERROR
1199                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
1200                                 #
1201                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
1202                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
1203                     interfaces:
1204                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
1205                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
1206                         vim_net_id:       #network id where this interface is connected
1207                         vim_interface_id: #interface/port VIM id
1208                         ip_address:       #null, or text with IPv4, IPv6 address
1209                         compute_node:     #identification of compute node where PF,VF interface is allocated
1210                         pci:              #PCI address of the NIC that hosts the PF,VF
1211                         vlan:             #physical VLAN used for VF
1212         '''
1213         vm_dict={}
1214         self.logger.debug("refresh_vms status: Getting tenant VM instance information from VIM")
1215         for vm_id in vm_list:
1216             vm={}
1217             try:
1218                 vm_vim = self.get_vminstance(vm_id)
1219                 if vm_vim['status'] in vmStatus2manoFormat:
1220                     vm['status']    =  vmStatus2manoFormat[ vm_vim['status'] ]
1221                 else:
1222                     vm['status']    = "OTHER"
1223                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
1224                 try:
1225                     vm['vim_info']  = yaml.safe_dump(vm_vim, default_flow_style=True, width=256)
1226                 except yaml.representer.RepresenterError:
1227                     vm['vim_info'] = str(vm_vim)
1228                 vm["interfaces"] = []
1229                 if vm_vim.get('fault'):
1230                     vm['error_msg'] = str(vm_vim['fault'])
1231                 #get interfaces
1232                 try:
1233                     self._reload_connection()
1234                     port_dict=self.neutron.list_ports(device_id=vm_id)
1235                     for port in port_dict["ports"]:
1236                         interface={}
1237                         try:
1238                             interface['vim_info'] = yaml.safe_dump(port, default_flow_style=True, width=256)
1239                         except yaml.representer.RepresenterError:
1240                             interface['vim_info'] = str(port)
1241                         interface["mac_address"] = port.get("mac_address")
1242                         interface["vim_net_id"] = port["network_id"]
1243                         interface["vim_interface_id"] = port["id"]
1244                         # check if OS-EXT-SRV-ATTR:host is there,
1245                         # in case of non-admin credentials, it will be missing
1246                         if vm_vim.get('OS-EXT-SRV-ATTR:host'):
1247                             interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host']
1248                         interface["pci"] = None
1249
1250                         # check if binding:profile is there,
1251                         # in case of non-admin credentials, it will be missing
1252                         if port.get('binding:profile'):
1253                             if port['binding:profile'].get('pci_slot'):
1254                                 # TODO: At the moment sr-iov pci addresses are converted to PF pci addresses by setting the slot to 0x00
1255                                 # TODO: This is just a workaround valid for niantinc. Find a better way to do so
1256                                 #   CHANGE DDDD:BB:SS.F to DDDD:BB:00.(F%2)   assuming there are 2 ports per nic
1257                                 pci = port['binding:profile']['pci_slot']
1258                                 # interface["pci"] = pci[:-4] + "00." + str(int(pci[-1]) % 2)
1259                                 interface["pci"] = pci
1260                         interface["vlan"] = None
1261                         #if network is of type vlan and port is of type direct (sr-iov) then set vlan id
1262                         network = self.neutron.show_network(port["network_id"])
1263                         if network['network'].get('provider:network_type') == 'vlan' and \
1264                             port.get("binding:vnic_type") == "direct":
1265                             interface["vlan"] = network['network'].get('provider:segmentation_id')
1266                         ips=[]
1267                         #look for floating ip address
1268                         floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
1269                         if floating_ip_dict.get("floatingips"):
1270                             ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address") )
1271
1272                         for subnet in port["fixed_ips"]:
1273                             ips.append(subnet["ip_address"])
1274                         interface["ip_address"] = ";".join(ips)
1275                         vm["interfaces"].append(interface)
1276                 except Exception as e:
1277                     self.logger.error("Error getting vm interface information " + type(e).__name__ + ": "+  str(e))
1278             except vimconn.vimconnNotFoundException as e:
1279                 self.logger.error("Exception getting vm status: %s", str(e))
1280                 vm['status'] = "DELETED"
1281                 vm['error_msg'] = str(e)
1282             except vimconn.vimconnException as e:
1283                 self.logger.error("Exception getting vm status: %s", str(e))
1284                 vm['status'] = "VIM_ERROR"
1285                 vm['error_msg'] = str(e)
1286             vm_dict[vm_id] = vm
1287         return vm_dict
1288
1289     def action_vminstance(self, vm_id, action_dict):
1290         '''Send and action over a VM instance from VIM
1291         Returns the vm_id if the action was successfully sent to the VIM'''
1292         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
1293         try:
1294             self._reload_connection()
1295             server = self.nova.servers.find(id=vm_id)
1296             if "start" in action_dict:
1297                 if action_dict["start"]=="rebuild":
1298                     server.rebuild()
1299                 else:
1300                     if server.status=="PAUSED":
1301                         server.unpause()
1302                     elif server.status=="SUSPENDED":
1303                         server.resume()
1304                     elif server.status=="SHUTOFF":
1305                         server.start()
1306             elif "pause" in action_dict:
1307                 server.pause()
1308             elif "resume" in action_dict:
1309                 server.resume()
1310             elif "shutoff" in action_dict or "shutdown" in action_dict:
1311                 server.stop()
1312             elif "forceOff" in action_dict:
1313                 server.stop() #TODO
1314             elif "terminate" in action_dict:
1315                 server.delete()
1316             elif "createImage" in action_dict:
1317                 server.create_image()
1318                 #"path":path_schema,
1319                 #"description":description_schema,
1320                 #"name":name_schema,
1321                 #"metadata":metadata_schema,
1322                 #"imageRef": id_schema,
1323                 #"disk": {"oneOf":[{"type": "null"}, {"type":"string"}] },
1324             elif "rebuild" in action_dict:
1325                 server.rebuild(server.image['id'])
1326             elif "reboot" in action_dict:
1327                 server.reboot() #reboot_type='SOFT'
1328             elif "console" in action_dict:
1329                 console_type = action_dict["console"]
1330                 if console_type == None or console_type == "novnc":
1331                     console_dict = server.get_vnc_console("novnc")
1332                 elif console_type == "xvpvnc":
1333                     console_dict = server.get_vnc_console(console_type)
1334                 elif console_type == "rdp-html5":
1335                     console_dict = server.get_rdp_console(console_type)
1336                 elif console_type == "spice-html5":
1337                     console_dict = server.get_spice_console(console_type)
1338                 else:
1339                     raise vimconn.vimconnException("console type '{}' not allowed".format(console_type),
1340                                                    http_code=vimconn.HTTP_Bad_Request)
1341                 try:
1342                     console_url = console_dict["console"]["url"]
1343                     #parse console_url
1344                     protocol_index = console_url.find("//")
1345                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1346                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1347                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1348                         raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1349                     console_dict2={"protocol": console_url[0:protocol_index],
1350                                   "server":   console_url[protocol_index+2 : port_index],
1351                                   "port":     int(console_url[port_index+1 : suffix_index]),
1352                                   "suffix":   console_url[suffix_index+1:]
1353                                   }
1354                     return console_dict2
1355                 except Exception as e:
1356                     raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1357
1358             return vm_id
1359         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1360             self._format_exception(e)
1361         #TODO insert exception vimconn.HTTP_Unauthorized
1362
1363 #NOT USED FUNCTIONS
1364
1365     def new_external_port(self, port_data):
1366         #TODO openstack if needed
1367         '''Adds a external port to VIM'''
1368         '''Returns the port identifier'''
1369         return -vimconn.HTTP_Internal_Server_Error, "osconnector.new_external_port() not implemented"
1370
1371     def connect_port_network(self, port_id, network_id, admin=False):
1372         #TODO openstack if needed
1373         '''Connects a external port to a network'''
1374         '''Returns status code of the VIM response'''
1375         return -vimconn.HTTP_Internal_Server_Error, "osconnector.connect_port_network() not implemented"
1376
1377     def new_user(self, user_name, user_passwd, tenant_id=None):
1378         '''Adds a new user to openstack VIM'''
1379         '''Returns the user identifier'''
1380         self.logger.debug("osconnector: Adding a new user to VIM")
1381         try:
1382             self._reload_connection()
1383             user=self.keystone.users.create(user_name, user_passwd, tenant_id=tenant_id)
1384             #self.keystone.tenants.add_user(self.k_creds["username"], #role)
1385             return user.id
1386         except ksExceptions.ConnectionError as e:
1387             error_value=-vimconn.HTTP_Bad_Request
1388             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1389         except ksExceptions.ClientException as e: #TODO remove
1390             error_value=-vimconn.HTTP_Bad_Request
1391             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1392         #TODO insert exception vimconn.HTTP_Unauthorized
1393         #if reaching here is because an exception
1394         if self.debug:
1395             self.logger.debug("new_user " + error_text)
1396         return error_value, error_text
1397
1398     def delete_user(self, user_id):
1399         '''Delete a user from openstack VIM'''
1400         '''Returns the user identifier'''
1401         if self.debug:
1402             print "osconnector: Deleting  a  user from VIM"
1403         try:
1404             self._reload_connection()
1405             self.keystone.users.delete(user_id)
1406             return 1, user_id
1407         except ksExceptions.ConnectionError as e:
1408             error_value=-vimconn.HTTP_Bad_Request
1409             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1410         except ksExceptions.NotFound as e:
1411             error_value=-vimconn.HTTP_Not_Found
1412             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1413         except ksExceptions.ClientException as e: #TODO remove
1414             error_value=-vimconn.HTTP_Bad_Request
1415             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1416         #TODO insert exception vimconn.HTTP_Unauthorized
1417         #if reaching here is because an exception
1418         if self.debug:
1419             print "delete_tenant " + error_text
1420         return error_value, error_text
1421
1422     def get_hosts_info(self):
1423         '''Get the information of deployed hosts
1424         Returns the hosts content'''
1425         if self.debug:
1426             print "osconnector: Getting Host info from VIM"
1427         try:
1428             h_list=[]
1429             self._reload_connection()
1430             hypervisors = self.nova.hypervisors.list()
1431             for hype in hypervisors:
1432                 h_list.append( hype.to_dict() )
1433             return 1, {"hosts":h_list}
1434         except nvExceptions.NotFound as e:
1435             error_value=-vimconn.HTTP_Not_Found
1436             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1437         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1438             error_value=-vimconn.HTTP_Bad_Request
1439             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1440         #TODO insert exception vimconn.HTTP_Unauthorized
1441         #if reaching here is because an exception
1442         if self.debug:
1443             print "get_hosts_info " + error_text
1444         return error_value, error_text
1445
1446     def get_hosts(self, vim_tenant):
1447         '''Get the hosts and deployed instances
1448         Returns the hosts content'''
1449         r, hype_dict = self.get_hosts_info()
1450         if r<0:
1451             return r, hype_dict
1452         hypervisors = hype_dict["hosts"]
1453         try:
1454             servers = self.nova.servers.list()
1455             for hype in hypervisors:
1456                 for server in servers:
1457                     if server.to_dict()['OS-EXT-SRV-ATTR:hypervisor_hostname']==hype['hypervisor_hostname']:
1458                         if 'vm' in hype:
1459                             hype['vm'].append(server.id)
1460                         else:
1461                             hype['vm'] = [server.id]
1462             return 1, hype_dict
1463         except nvExceptions.NotFound as e:
1464             error_value=-vimconn.HTTP_Not_Found
1465             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1466         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1467             error_value=-vimconn.HTTP_Bad_Request
1468             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1469         #TODO insert exception vimconn.HTTP_Unauthorized
1470         #if reaching here is because an exception
1471         if self.debug:
1472             print "get_hosts " + error_text
1473         return error_value, error_text
1474
1475