osm_ro/vimconn_openstack.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 osconnector implements all the methods to interact with openstack using the python-client.
  26 '''
  27 __author__="Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research"
  28 __date__ ="$22-jun-2014 11:19:29$"
  29
  30 import vimconn
  31 import json
  32 import yaml
  33 import logging
  34 import netaddr
  35 import time
  36 import yaml
  37 import random
  38
  39 from novaclient import client as nClient, exceptions as nvExceptions
  40 from keystoneauth1.identity import v2, v3
  41 from keystoneauth1 import session
  42 import keystoneclient.exceptions as ksExceptions
  43 import keystoneclient.v3.client as ksClient_v3
  44 import keystoneclient.v2_0.client as ksClient_v2
  45 from glanceclient import client as glClient
  46 import glanceclient.client as gl1Client
  47 import glanceclient.exc as gl1Exceptions
  48 from  cinderclient import client as cClient
  49 from httplib import HTTPException
  50 from neutronclient.neutron import client as neClient
  51 from neutronclient.common import exceptions as neExceptions
  52 from requests.exceptions import ConnectionError
  53
  54 '''contain the openstack virtual machine status to openmano status'''
  55 vmStatus2manoFormat={'ACTIVE':'ACTIVE',
  56                      'PAUSED':'PAUSED',
  57                      'SUSPENDED': 'SUSPENDED',
  58                      'SHUTOFF':'INACTIVE',
  59                      'BUILD':'BUILD',
  60                      'ERROR':'ERROR','DELETED':'DELETED'
  61                      }
  62 netStatus2manoFormat={'ACTIVE':'ACTIVE','PAUSED':'PAUSED','INACTIVE':'INACTIVE','BUILD':'BUILD','ERROR':'ERROR','DELETED':'DELETED'
  63                      }
  64
  65 #global var to have a timeout creating and deleting volumes
  66 volume_timeout = 60
  67 server_timeout = 300
  68
  69 class vimconnector(vimconn.vimconnector):
  70     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,
  71                  log_level=None, config={}, persistent_info={}):
  72         '''using common constructor parameters. In this case
  73         'url' is the keystone authorization url,
  74         'url_admin' is not use
  75         '''
  76         api_version = config.get('APIversion')
  77         if api_version and api_version not in ('v3.3', 'v2.0', '2', '3'):
  78             raise vimconn.vimconnException("Invalid value '{}' for config:APIversion. "
  79                                            "Allowed values are 'v3.3', 'v2.0', '2' or '3'".format(api_version))
  80         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
  81                                       config)
  82
  83         self.insecure = self.config.get("insecure", False)
  84         if not url:
  85             raise TypeError, 'url param can not be NoneType'
  86         self.persistent_info = persistent_info
  87         self.availability_zone = persistent_info.get('availability_zone', None)
  88         self.session = persistent_info.get('session', {'reload_client': True})
  89         self.nova = self.session.get('nova')
  90         self.neutron = self.session.get('neutron')
  91         self.cinder = self.session.get('cinder')
  92         self.glance = self.session.get('glance')
  93         self.glancev1 = self.session.get('glancev1')
  94         self.keystone = self.session.get('keystone')
  95         self.api_version3 = self.session.get('api_version3')
  96
  97         self.logger = logging.getLogger('openmano.vim.openstack')
  98         if log_level:
  99             self.logger.setLevel( getattr(logging, log_level) )
 100
 101     def __getitem__(self, index):
 102         """Get individuals parameters.
 103         Throw KeyError"""
 104         if index == 'project_domain_id':
 105             return self.config.get("project_domain_id")
 106         elif index == 'user_domain_id':
 107             return self.config.get("user_domain_id")
 108         else:
 109             return vimconn.vimconnector.__getitem__(self, index)
 110
 111     def __setitem__(self, index, value):
 112         """Set individuals parameters and it is marked as dirty so to force connection reload.
 113         Throw KeyError"""
 114         if index == 'project_domain_id':
 115             self.config["project_domain_id"] = value
 116         elif index == 'user_domain_id':
 117                 self.config["user_domain_id"] = value
 118         else:
 119             vimconn.vimconnector.__setitem__(self, index, value)
 120         self.session['reload_client'] = True
 121
 122     def _reload_connection(self):
 123         '''Called before any operation, it check if credentials has changed
 124         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
 125         '''
 126         #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
 127         if self.session['reload_client']:
 128             if self.config.get('APIversion'):
 129                 self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3'
 130             else:  # get from ending auth_url that end with v3 or with v2.0
 131                 self.api_version3 =  self.url.split("/")[-1] == "v3"
 132             self.session['api_version3'] = self.api_version3
 133             if self.api_version3:
 134                 auth = v3.Password(auth_url=self.url,
 135                                    username=self.user,
 136                                    password=self.passwd,
 137                                    project_name=self.tenant_name,
 138                                    project_id=self.tenant_id,
 139                                    project_domain_id=self.config.get('project_domain_id', 'default'),
 140                                    user_domain_id=self.config.get('user_domain_id', 'default'))
 141             else:
 142                 auth = v2.Password(auth_url=self.url,
 143                                    username=self.user,
 144                                    password=self.passwd,
 145                                    tenant_name=self.tenant_name,
 146                                    tenant_id=self.tenant_id)
 147             sess = session.Session(auth=auth, verify=not self.insecure)
 148             if self.api_version3:
 149                 self.keystone = ksClient_v3.Client(session=sess)
 150             else:
 151                 self.keystone = ksClient_v2.Client(session=sess)
 152             self.session['keystone'] = self.keystone
 153             # In order to enable microversion functionality an explicit microversion must be specified in 'config'.
 154             # This implementation approach is due to the warning message in
 155             # https://developer.openstack.org/api-guide/compute/microversions.html
 156             # where it is stated that microversion backwards compatibility is not guaranteed and clients should
 157             # always require an specific microversion.
 158             # To be able to use 'device role tagging' functionality define 'microversion: 2.32' in datacenter config
 159             version = self.config.get("microversion")
 160             if not version:
 161                 version = "2.1"
 162             self.nova = self.session['nova'] = nClient.Client(str(version), session=sess)
 163             self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess)
 164             self.cinder = self.session['cinder'] = cClient.Client(2, session=sess)
 165             self.glance = self.session['glance'] = glClient.Client(2, session=sess)
 166             self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess)
 167             self.session['reload_client'] = False
 168             self.persistent_info['session'] = self.session
 169             # add availablity zone info inside  self.persistent_info
 170             self._set_availablity_zones()
 171             self.persistent_info['availability_zone'] = self.availability_zone
 172
 173     def __net_os2mano(self, net_list_dict):
 174         '''Transform the net openstack format to mano format
 175         net_list_dict can be a list of dict or a single dict'''
 176         if type(net_list_dict) is dict:
 177             net_list_=(net_list_dict,)
 178         elif type(net_list_dict) is list:
 179             net_list_=net_list_dict
 180         else:
 181             raise TypeError("param net_list_dict must be a list or a dictionary")
 182         for net in net_list_:
 183             if net.get('provider:network_type') == "vlan":
 184                 net['type']='data'
 185             else:
 186                 net['type']='bridge'
 187
 188     def _format_exception(self, exception):
 189         '''Transform a keystone, nova, neutron  exception into a vimconn exception'''
 190         if isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
 191                                   ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed
 192                                   )):
 193             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 194         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
 195                                     neExceptions.NeutronException, nvExceptions.BadRequest)):
 196             raise vimconn.vimconnUnexpectedResponse(type(exception).__name__ + ": " + str(exception))
 197         elif isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound)):
 198             raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception))
 199         elif isinstance(exception, nvExceptions.Conflict):
 200             raise vimconn.vimconnConflictException(type(exception).__name__ + ": " + str(exception))
 201         elif isinstance(exception, vimconn.vimconnException):
 202             raise
 203         else:  # ()
 204             self.logger.error("General Exception " + str(exception), exc_info=True)
 205             raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception))
 206
 207     def get_tenant_list(self, filter_dict={}):
 208         '''Obtain tenants of VIM
 209         filter_dict can contain the following keys:
 210             name: filter by tenant name
 211             id: filter by tenant uuid/id
 212             <other VIM specific>
 213         Returns the tenant list of dictionaries: [{'name':'<name>, 'id':'<id>, ...}, ...]
 214         '''
 215         self.logger.debug("Getting tenants from VIM filter: '%s'", str(filter_dict))
 216         try:
 217             self._reload_connection()
 218             if self.api_version3:
 219                 project_class_list = self.keystone.projects.list(name=filter_dict.get("name"))
 220             else:
 221                 project_class_list = self.keystone.tenants.findall(**filter_dict)
 222             project_list=[]
 223             for project in project_class_list:
 224                 if filter_dict.get('id') and filter_dict["id"] != project.id:
 225                     continue
 226                 project_list.append(project.to_dict())
 227             return project_list
 228         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e:
 229             self._format_exception(e)
 230
 231     def new_tenant(self, tenant_name, tenant_description):
 232         '''Adds a new tenant to openstack VIM. Returns the tenant identifier'''
 233         self.logger.debug("Adding a new tenant name: %s", tenant_name)
 234         try:
 235             self._reload_connection()
 236             if self.api_version3:
 237                 project = self.keystone.projects.create(tenant_name, self.config.get("project_domain_id", "default"),
 238                                                         description=tenant_description, is_domain=False)
 239             else:
 240                 project = self.keystone.tenants.create(tenant_name, tenant_description)
 241             return project.id
 242         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 243             self._format_exception(e)
 244
 245     def delete_tenant(self, tenant_id):
 246         '''Delete a tenant from openstack VIM. Returns the old tenant identifier'''
 247         self.logger.debug("Deleting tenant %s from VIM", tenant_id)
 248         try:
 249             self._reload_connection()
 250             if self.api_version3:
 251                 self.keystone.projects.delete(tenant_id)
 252             else:
 253                 self.keystone.tenants.delete(tenant_id)
 254             return tenant_id
 255         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError)  as e:
 256             self._format_exception(e)
 257
 258     def new_network(self,net_name, net_type, ip_profile=None, shared=False, vlan=None):
 259         '''Adds a tenant network to VIM. Returns the network identifier'''
 260         self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type)
 261         #self.logger.debug(">>>>>>>>>>>>>>>>>> IP profile %s", str(ip_profile))
 262         try:
 263             new_net = None
 264             self._reload_connection()
 265             network_dict = {'name': net_name, 'admin_state_up': True}
 266             if net_type=="data" or net_type=="ptp":
 267                 if self.config.get('dataplane_physical_net') == None:
 268                     raise vimconn.vimconnConflictException("You must provide a 'dataplane_physical_net' at config value before creating sriov network")
 269                 network_dict["provider:physical_network"] = self.config['dataplane_physical_net'] #"physnet_sriov" #TODO physical
 270                 network_dict["provider:network_type"]     = "vlan"
 271                 if vlan!=None:
 272                     network_dict["provider:network_type"] = vlan
 273             network_dict["shared"]=shared
 274             new_net=self.neutron.create_network({'network':network_dict})
 275             #print new_net
 276             #create subnetwork, even if there is no profile
 277             if not ip_profile:
 278                 ip_profile = {}
 279             if 'subnet_address' not in ip_profile:
 280                 #Fake subnet is required
 281                 subnet_rand = random.randint(0, 255)
 282                 ip_profile['subnet_address'] = "192.168.{}.0/24".format(subnet_rand)
 283             if 'ip_version' not in ip_profile:
 284                 ip_profile['ip_version'] = "IPv4"
 285             subnet = {"name":net_name+"-subnet",
 286                     "network_id": new_net["network"]["id"],
 287                     "ip_version": 4 if ip_profile['ip_version']=="IPv4" else 6,
 288                     "cidr": ip_profile['subnet_address']
 289                     }
 290             # Gateway should be set to None if not needed. Otherwise openstack assigns one by default
 291             subnet['gateway_ip'] = ip_profile.get('gateway_address')
 292             if ip_profile.get('dns_address'):
 293                 subnet['dns_nameservers'] = ip_profile['dns_address'].split(";")
 294             if 'dhcp_enabled' in ip_profile:
 295                 subnet['enable_dhcp'] = False if ip_profile['dhcp_enabled']=="false" else True
 296             if 'dhcp_start_address' in ip_profile:
 297                 subnet['allocation_pools'] = []
 298                 subnet['allocation_pools'].append(dict())
 299                 subnet['allocation_pools'][0]['start'] = ip_profile['dhcp_start_address']
 300             if 'dhcp_count' in ip_profile:
 301                 #parts = ip_profile['dhcp_start_address'].split('.')
 302                 #ip_int = (int(parts[0]) << 24) + (int(parts[1]) << 16) + (int(parts[2]) << 8) + int(parts[3])
 303                 ip_int = int(netaddr.IPAddress(ip_profile['dhcp_start_address']))
 304                 ip_int += ip_profile['dhcp_count'] - 1
 305                 ip_str = str(netaddr.IPAddress(ip_int))
 306                 subnet['allocation_pools'][0]['end'] = ip_str
 307             #self.logger.debug(">>>>>>>>>>>>>>>>>> Subnet: %s", str(subnet))
 308             self.neutron.create_subnet({"subnet": subnet} )
 309             return new_net["network"]["id"]
 310         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 311             if new_net:
 312                 self.neutron.delete_network(new_net['network']['id'])
 313             self._format_exception(e)
 314
 315     def get_network_list(self, filter_dict={}):
 316         '''Obtain tenant networks of VIM
 317         Filter_dict can be:
 318             name: network name
 319             id: network uuid
 320             shared: boolean
 321             tenant_id: tenant
 322             admin_state_up: boolean
 323             status: 'ACTIVE'
 324         Returns the network list of dictionaries
 325         '''
 326         self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict))
 327         try:
 328             self._reload_connection()
 329             if self.api_version3 and "tenant_id" in filter_dict:
 330                 filter_dict['project_id'] = filter_dict.pop('tenant_id') #TODO check
 331             net_dict=self.neutron.list_networks(**filter_dict)
 332             net_list=net_dict["networks"]
 333             self.__net_os2mano(net_list)
 334             return net_list
 335         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 336             self._format_exception(e)
 337
 338     def get_network(self, net_id):
 339         '''Obtain details of network from VIM
 340         Returns the network information from a network id'''
 341         self.logger.debug(" Getting tenant network %s from VIM", net_id)
 342         filter_dict={"id": net_id}
 343         net_list = self.get_network_list(filter_dict)
 344         if len(net_list)==0:
 345             raise vimconn.vimconnNotFoundException("Network '{}' not found".format(net_id))
 346         elif len(net_list)>1:
 347             raise vimconn.vimconnConflictException("Found more than one network with this criteria")
 348         net = net_list[0]
 349         subnets=[]
 350         for subnet_id in net.get("subnets", () ):
 351             try:
 352                 subnet = self.neutron.show_subnet(subnet_id)
 353             except Exception as e:
 354                 self.logger.error("osconnector.get_network(): Error getting subnet %s %s" % (net_id, str(e)))
 355                 subnet = {"id": subnet_id, "fault": str(e)}
 356             subnets.append(subnet)
 357         net["subnets"] = subnets
 358         net["encapsulation"] = net.get('provider:network_type')
 359         net["segmentation_id"] = net.get('provider:segmentation_id')
 360         return net
 361
 362     def delete_network(self, net_id):
 363         '''Deletes a tenant network from VIM. Returns the old network identifier'''
 364         self.logger.debug("Deleting network '%s' from VIM", net_id)
 365         try:
 366             self._reload_connection()
 367             #delete VM ports attached to this networks before the network
 368             ports = self.neutron.list_ports(network_id=net_id)
 369             for p in ports['ports']:
 370                 try:
 371                     self.neutron.delete_port(p["id"])
 372                 except Exception as e:
 373                     self.logger.error("Error deleting port %s: %s", p["id"], str(e))
 374             self.neutron.delete_network(net_id)
 375             return net_id
 376         except (neExceptions.ConnectionFailed, neExceptions.NetworkNotFoundClient, neExceptions.NeutronException,
 377                 ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 378             self._format_exception(e)
 379
 380     def refresh_nets_status(self, net_list):
 381         '''Get the status of the networks
 382            Params: the list of network identifiers
 383            Returns a dictionary with:
 384                 net_id:         #VIM id of this network
 385                     status:     #Mandatory. Text with one of:
 386                                 #  DELETED (not found at vim)
 387                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 388                                 #  OTHER (Vim reported other status not understood)
 389                                 #  ERROR (VIM indicates an ERROR status)
 390                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 391                                 #  BUILD (on building process)
 392                                 #
 393                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 394                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 395
 396         '''
 397         net_dict={}
 398         for net_id in net_list:
 399             net = {}
 400             try:
 401                 net_vim = self.get_network(net_id)
 402                 if net_vim['status'] in netStatus2manoFormat:
 403                     net["status"] = netStatus2manoFormat[ net_vim['status'] ]
 404                 else:
 405                     net["status"] = "OTHER"
 406                     net["error_msg"] = "VIM status reported " + net_vim['status']
 407
 408                 if net['status'] == "ACTIVE" and not net_vim['admin_state_up']:
 409                     net['status'] = 'DOWN'
 410                 try:
 411                     net['vim_info'] = yaml.safe_dump(net_vim, default_flow_style=True, width=256)
 412                 except yaml.representer.RepresenterError:
 413                     net['vim_info'] = str(net_vim)
 414                 if net_vim.get('fault'):  #TODO
 415                     net['error_msg'] = str(net_vim['fault'])
 416             except vimconn.vimconnNotFoundException as e:
 417                 self.logger.error("Exception getting net status: %s", str(e))
 418                 net['status'] = "DELETED"
 419                 net['error_msg'] = str(e)
 420             except vimconn.vimconnException as e:
 421                 self.logger.error("Exception getting net status: %s", str(e))
 422                 net['status'] = "VIM_ERROR"
 423                 net['error_msg'] = str(e)
 424             net_dict[net_id] = net
 425         return net_dict
 426
 427     def get_flavor(self, flavor_id):
 428         '''Obtain flavor details from the  VIM. Returns the flavor dict details'''
 429         self.logger.debug("Getting flavor '%s'", flavor_id)
 430         try:
 431             self._reload_connection()
 432             flavor = self.nova.flavors.find(id=flavor_id)
 433             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 434             return flavor.to_dict()
 435         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 436             self._format_exception(e)
 437
 438     def get_flavor_id_from_data(self, flavor_dict):
 439         """Obtain flavor id that match the flavor description
 440            Returns the flavor_id or raises a vimconnNotFoundException
 441            flavor_dict: contains the required ram, vcpus, disk
 442            If 'use_existing_flavors' is set to True at config, the closer flavor that provides same or more ram, vcpus
 443                 and disk is returned. Otherwise a flavor with exactly same ram, vcpus and disk is returned or a
 444                 vimconnNotFoundException is raised
 445         """
 446         exact_match = False if self.config.get('use_existing_flavors') else True
 447         try:
 448             self._reload_connection()
 449             flavor_candidate_id = None
 450             flavor_candidate_data = (10000, 10000, 10000)
 451             flavor_target = (flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"])
 452             # numa=None
 453             numas = flavor_dict.get("extended", {}).get("numas")
 454             if numas:
 455                 #TODO
 456                 raise vimconn.vimconnNotFoundException("Flavor with EPA still not implemted")
 457                 # if len(numas) > 1:
 458                 #     raise vimconn.vimconnNotFoundException("Cannot find any flavor with more than one numa")
 459                 # numa=numas[0]
 460                 # numas = extended.get("numas")
 461             for flavor in self.nova.flavors.list():
 462                 epa = flavor.get_keys()
 463                 if epa:
 464                     continue
 465                     # TODO
 466                 flavor_data = (flavor.ram, flavor.vcpus, flavor.disk)
 467                 if flavor_data == flavor_target:
 468                     return flavor.id
 469                 elif not exact_match and flavor_target < flavor_data < flavor_candidate_data:
 470                     flavor_candidate_id = flavor.id
 471                     flavor_candidate_data = flavor_data
 472             if not exact_match and flavor_candidate_id:
 473                 return flavor_candidate_id
 474             raise vimconn.vimconnNotFoundException("Cannot find any flavor matching '{}'".format(str(flavor_dict)))
 475         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e:
 476             self._format_exception(e)
 477
 478
 479     def new_flavor(self, flavor_data, change_name_if_used=True):
 480         '''Adds a tenant flavor to openstack VIM
 481         if change_name_if_used is True, it will change name in case of conflict, because it is not supported name repetition
 482         Returns the flavor identifier
 483         '''
 484         self.logger.debug("Adding flavor '%s'", str(flavor_data))
 485         retry=0
 486         max_retries=3
 487         name_suffix = 0
 488         name=flavor_data['name']
 489         while retry<max_retries:
 490             retry+=1
 491             try:
 492                 self._reload_connection()
 493                 if change_name_if_used:
 494                     #get used names
 495                     fl_names=[]
 496                     fl=self.nova.flavors.list()
 497                     for f in fl:
 498                         fl_names.append(f.name)
 499                     while name in fl_names:
 500                         name_suffix += 1
 501                         name = flavor_data['name']+"-" + str(name_suffix)
 502
 503                 ram = flavor_data.get('ram',64)
 504                 vcpus = flavor_data.get('vcpus',1)
 505                 numa_properties=None
 506
 507                 extended = flavor_data.get("extended")
 508                 if extended:
 509                     numas=extended.get("numas")
 510                     if numas:
 511                         numa_nodes = len(numas)
 512                         if numa_nodes > 1:
 513                             return -1, "Can not add flavor with more than one numa"
 514                         numa_properties = {"hw:numa_nodes":str(numa_nodes)}
 515                         numa_properties["hw:mem_page_size"] = "large"
 516                         numa_properties["hw:cpu_policy"] = "dedicated"
 517                         numa_properties["hw:numa_mempolicy"] = "strict"
 518                         for numa in numas:
 519                             #overwrite ram and vcpus
 520                             ram = numa['memory']*1024
 521                             #See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/implemented/virt-driver-cpu-thread-pinning.html
 522                             if 'paired-threads' in numa:
 523                                 vcpus = numa['paired-threads']*2
 524                                 #cpu_thread_policy "require" implies that the compute node must have an STM architecture
 525                                 numa_properties["hw:cpu_thread_policy"] = "require"
 526                                 numa_properties["hw:cpu_policy"] = "dedicated"
 527                             elif 'cores' in numa:
 528                                 vcpus = numa['cores']
 529                                 # cpu_thread_policy "prefer" implies that the host must not have an SMT architecture, or a non-SMT architecture will be emulated
 530                                 numa_properties["hw:cpu_thread_policy"] = "isolate"
 531                                 numa_properties["hw:cpu_policy"] = "dedicated"
 532                             elif 'threads' in numa:
 533                                 vcpus = numa['threads']
 534                                 # cpu_thread_policy "prefer" implies that the host may or may not have an SMT architecture
 535                                 numa_properties["hw:cpu_thread_policy"] = "prefer"
 536                                 numa_properties["hw:cpu_policy"] = "dedicated"
 537                             # for interface in numa.get("interfaces",() ):
 538                             #     if interface["dedicated"]=="yes":
 539                             #         raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
 540                             #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"' when a way to connect it is available
 541
 542                 #create flavor
 543                 new_flavor=self.nova.flavors.create(name,
 544                                 ram,
 545                                 vcpus,
 546                                 flavor_data.get('disk',1),
 547                                 is_public=flavor_data.get('is_public', True)
 548                             )
 549                 #add metadata
 550                 if numa_properties:
 551                     new_flavor.set_keys(numa_properties)
 552                 return new_flavor.id
 553             except nvExceptions.Conflict as e:
 554                 if change_name_if_used and retry < max_retries:
 555                     continue
 556                 self._format_exception(e)
 557             #except nvExceptions.BadRequest as e:
 558             except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 559                 self._format_exception(e)
 560
 561     def delete_flavor(self,flavor_id):
 562         '''Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
 563         '''
 564         try:
 565             self._reload_connection()
 566             self.nova.flavors.delete(flavor_id)
 567             return flavor_id
 568         #except nvExceptions.BadRequest as e:
 569         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
 570             self._format_exception(e)
 571
 572     def new_image(self,image_dict):
 573         '''
 574         Adds a tenant image to VIM. imge_dict is a dictionary with:
 575             name: name
 576             disk_format: qcow2, vhd, vmdk, raw (by default), ...
 577             location: path or URI
 578             public: "yes" or "no"
 579             metadata: metadata of the image
 580         Returns the image_id
 581         '''
 582         retry=0
 583         max_retries=3
 584         while retry<max_retries:
 585             retry+=1
 586             try:
 587                 self._reload_connection()
 588                 #determine format  http://docs.openstack.org/developer/glance/formats.html
 589                 if "disk_format" in image_dict:
 590                     disk_format=image_dict["disk_format"]
 591                 else: #autodiscover based on extension
 592                     if image_dict['location'][-6:]==".qcow2":
 593                         disk_format="qcow2"
 594                     elif image_dict['location'][-4:]==".vhd":
 595                         disk_format="vhd"
 596                     elif image_dict['location'][-5:]==".vmdk":
 597                         disk_format="vmdk"
 598                     elif image_dict['location'][-4:]==".vdi":
 599                         disk_format="vdi"
 600                     elif image_dict['location'][-4:]==".iso":
 601                         disk_format="iso"
 602                     elif image_dict['location'][-4:]==".aki":
 603                         disk_format="aki"
 604                     elif image_dict['location'][-4:]==".ari":
 605                         disk_format="ari"
 606                     elif image_dict['location'][-4:]==".ami":
 607                         disk_format="ami"
 608                     else:
 609                         disk_format="raw"
 610                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
 611                 if image_dict['location'][0:4]=="http":
 612                     new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 613                             container_format="bare", location=image_dict['location'], disk_format=disk_format)
 614                 else: #local path
 615                     with open(image_dict['location']) as fimage:
 616                         new_image = self.glancev1.images.create(name=image_dict['name'], is_public=image_dict.get('public',"yes")=="yes",
 617                             container_format="bare", data=fimage, disk_format=disk_format)
 618                 #insert metadata. We cannot use 'new_image.properties.setdefault'
 619                 #because nova and glance are "INDEPENDENT" and we are using nova for reading metadata
 620                 new_image_nova=self.nova.images.find(id=new_image.id)
 621                 new_image_nova.metadata.setdefault('location',image_dict['location'])
 622                 metadata_to_load = image_dict.get('metadata')
 623                 if metadata_to_load:
 624                     for k,v in yaml.load(metadata_to_load).iteritems():
 625                         new_image_nova.metadata.setdefault(k,v)
 626                 return new_image.id
 627             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
 628                 self._format_exception(e)
 629             except (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 630                 if retry==max_retries:
 631                     continue
 632                 self._format_exception(e)
 633             except IOError as e:  #can not open the file
 634                 raise vimconn.vimconnConnectionException(type(e).__name__ + ": " + str(e)+ " for " + image_dict['location'],
 635                                                          http_code=vimconn.HTTP_Bad_Request)
 636
 637     def delete_image(self, image_id):
 638         '''Deletes a tenant image from openstack VIM. Returns the old id
 639         '''
 640         try:
 641             self._reload_connection()
 642             self.nova.images.delete(image_id)
 643             return image_id
 644         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e: #TODO remove
 645             self._format_exception(e)
 646
 647     def get_image_id_from_path(self, path):
 648         '''Get the image id from image path in the VIM database. Returns the image_id'''
 649         try:
 650             self._reload_connection()
 651             images = self.nova.images.list()
 652             for image in images:
 653                 if image.metadata.get("location")==path:
 654                     return image.id
 655             raise vimconn.vimconnNotFoundException("image with location '{}' not found".format( path))
 656         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 657             self._format_exception(e)
 658
 659     def get_image_list(self, filter_dict={}):
 660         '''Obtain tenant images from VIM
 661         Filter_dict can be:
 662             id: image id
 663             name: image name
 664             checksum: image checksum
 665         Returns the image list of dictionaries:
 666             [{<the fields at Filter_dict plus some VIM specific>}, ...]
 667             List can be empty
 668         '''
 669         self.logger.debug("Getting image list from VIM filter: '%s'", str(filter_dict))
 670         try:
 671             self._reload_connection()
 672             filter_dict_os=filter_dict.copy()
 673             #First we filter by the available filter fields: name, id. The others are removed.
 674             filter_dict_os.pop('checksum',None)
 675             image_list=self.nova.images.findall(**filter_dict_os)
 676             if len(image_list)==0:
 677                 return []
 678             #Then we filter by the rest of filter fields: checksum
 679             filtered_list = []
 680             for image in image_list:
 681                 image_class=self.glance.images.get(image.id)
 682                 if 'checksum' not in filter_dict or image_class['checksum']==filter_dict.get('checksum'):
 683                     filtered_list.append(image_class.copy())
 684             return filtered_list
 685         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError, ConnectionError) as e:
 686             self._format_exception(e)
 687
 688     def __wait_for_vm(self, vm_id, status):
 689         """wait until vm is in the desired status and return True.
 690         If the VM gets in ERROR status, return false.
 691         If the timeout is reached generate an exception"""
 692         elapsed_time = 0
 693         while elapsed_time < server_timeout:
 694             vm_status = self.nova.servers.get(vm_id).status
 695             if vm_status == status:
 696                 return True
 697             if vm_status == 'ERROR':
 698                 return False
 699             time.sleep(1)
 700             elapsed_time += 1
 701
 702         # if we exceeded the timeout rollback
 703         if elapsed_time >= server_timeout:
 704             raise vimconn.vimconnException('Timeout waiting for instance ' + vm_id + ' to get ' + status,
 705                                            http_code=vimconn.HTTP_Request_Timeout)
 706
 707     def _get_openstack_availablity_zones(self):
 708         """
 709         Get from openstack availability zones available
 710         :return:
 711         """
 712         try:
 713             openstack_availability_zone = self.nova.availability_zones.list()
 714             openstack_availability_zone = [str(zone.zoneName) for zone in openstack_availability_zone
 715                                            if zone.zoneName != 'internal']
 716             return openstack_availability_zone
 717         except Exception as e:
 718             return None
 719
 720     def _set_availablity_zones(self):
 721         """
 722         Set vim availablity zone
 723         :return:
 724         """
 725
 726         if 'availability_zone' in self.config:
 727             vim_availability_zones = self.config.get('availability_zone')
 728             if isinstance(vim_availability_zones, str):
 729                 self.availability_zone = [vim_availability_zones]
 730             elif isinstance(vim_availability_zones, list):
 731                 self.availability_zone = vim_availability_zones
 732         else:
 733             self.availability_zone = self._get_openstack_availablity_zones()
 734
 735     def _get_vm_availavility_zone(self, availavility_zone_index, nfv_availability_zones):
 736         """
 737         Return a list with all availability zones create during datacenter attach.
 738         :return: List with availability zones
 739         """
 740         openstack_avilability_zone = self.availability_zone
 741
 742         # check if VIM offer enough availability zones describe in the VNFC
 743         if self.availability_zone and availavility_zone_index is not None \
 744                 and 0 <= len(nfv_availability_zones) <= len(self.availability_zone):
 745
 746             if nfv_availability_zones:
 747                 vnf_azone = nfv_availability_zones[availavility_zone_index]
 748                 zones_available = []
 749
 750                 for nfv_zone in nfv_availability_zones:
 751                     for vim_zone in openstack_avilability_zone:
 752                         if nfv_zone is vim_zone:
 753                             zones_available.append(nfv_zone)
 754
 755                 if len(zones_available) == len(openstack_avilability_zone) and vnf_azone in openstack_avilability_zone:
 756                     return vnf_azone
 757                 else:
 758                     return openstack_avilability_zone[availavility_zone_index]
 759         else:
 760             raise vimconn.vimconnConflictException("No enough availablity zones for this deployment")
 761         return None
 762
 763     def new_vminstance(self, name, description, start, image_id, flavor_id, net_list,cloud_config=None,disk_list=None,
 764                        availavility_zone_index=None, nfv_availability_zones=None):
 765         '''Adds a VM instance to VIM
 766         Params:
 767             start: indicates if VM must start or boot in pause mode. Ignored
 768             image_id,flavor_id: iamge and flavor uuid
 769             net_list: list of interfaces, each one is a dictionary with:
 770                 name:
 771                 net_id: network uuid to connect
 772                 vpci: virtual vcpi to assign, ignored because openstack lack #TODO
 773                 model: interface model, ignored #TODO
 774                 mac_address: used for  SR-IOV ifaces #TODO for other types
 775                 use: 'data', 'bridge',  'mgmt'
 776                 type: 'virtual', 'PF', 'VF', 'VFnotShared'
 777                 vim_id: filled/added by this function
 778                 floating_ip: True/False (or it can be None)
 779                 'cloud_config': (optional) dictionary with:
 780                 'key-pairs': (optional) list of strings with the public key to be inserted to the default user
 781                 'users': (optional) list of users to be inserted, each item is a dict with:
 782                     'name': (mandatory) user name,
 783                     'key-pairs': (optional) list of strings with the public key to be inserted to the user
 784                 'user-data': (optional) string is a text script to be passed directly to cloud-init
 785                 'config-files': (optional). List of files to be transferred. Each item is a dict with:
 786                     'dest': (mandatory) string with the destination absolute path
 787                     'encoding': (optional, by default text). Can be one of:
 788                         'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
 789                     'content' (mandatory): string with the content of the file
 790                     'permissions': (optional) string with file permissions, typically octal notation '0644'
 791                     'owner': (optional) file owner, string with the format 'owner:group'
 792                 'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk)
 793             'disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
 794                 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
 795                 'size': (mandatory) string with the size of the disk in GB
 796             availavility_zone_index:counter for instance order in vim availability_zones availables
 797             nfv_availability_zones: Lost given by user in the VNFC descriptor.
 798                 #TODO ip, security groups
 799         Returns the instance identifier
 800         '''
 801         self.logger.debug("new_vminstance input: image='%s' flavor='%s' nics='%s'",image_id, flavor_id,str(net_list))
 802         try:
 803             server = None
 804             metadata={}
 805             net_list_vim=[]
 806             external_network=[]     # list of external networks to be connected to instance, later on used to create floating_ip
 807             no_secured_ports = []   # List of port-is with port-security disabled
 808             self._reload_connection()
 809             metadata_vpci={}   # For a specific neutron plugin
 810             block_device_mapping = None
 811             for net in net_list:
 812                 if not net.get("net_id"): #skip non connected iface
 813                     continue
 814
 815                 port_dict={
 816                     "network_id": net["net_id"],
 817                     "name": net.get("name"),
 818                     "admin_state_up": True
 819                 }
 820                 if net["type"]=="virtual":
 821                     if "vpci" in net:
 822                         metadata_vpci[ net["net_id"] ] = [[ net["vpci"], "" ]]
 823                 elif net["type"]=="VF": # for VF
 824                     if "vpci" in net:
 825                         if "VF" not in metadata_vpci:
 826                             metadata_vpci["VF"]=[]
 827                         metadata_vpci["VF"].append([ net["vpci"], "" ])
 828                     port_dict["binding:vnic_type"]="direct"
 829                 else:  # For PT
 830                     if "vpci" in net:
 831                         if "PF" not in metadata_vpci:
 832                             metadata_vpci["PF"]=[]
 833                         metadata_vpci["PF"].append([ net["vpci"], "" ])
 834                     port_dict["binding:vnic_type"]="direct-physical"
 835                 if not port_dict["name"]:
 836                     port_dict["name"]=name
 837                 if net.get("mac_address"):
 838                     port_dict["mac_address"]=net["mac_address"]
 839                 new_port = self.neutron.create_port({"port": port_dict })
 840                 net["mac_adress"] = new_port["port"]["mac_address"]
 841                 net["vim_id"] = new_port["port"]["id"]
 842                 # if try to use a network without subnetwork, it will return a emtpy list
 843                 fixed_ips = new_port["port"].get("fixed_ips")
 844                 if fixed_ips:
 845                     net["ip"] = fixed_ips[0].get("ip_address")
 846                 else:
 847                     net["ip"] = None
 848
 849                 port = {"port-id": new_port["port"]["id"]}
 850                 if float(self.nova.api_version.get_string()) >= 2.32:
 851                     port["tag"] = new_port["port"]["name"]
 852                 net_list_vim.append(port)
 853
 854                 if net.get('floating_ip', False):
 855                     net['exit_on_floating_ip_error'] = True
 856                     external_network.append(net)
 857                 elif net['use'] == 'mgmt' and self.config.get('use_floating_ip'):
 858                     net['exit_on_floating_ip_error'] = False
 859                     external_network.append(net)
 860
 861                 # If port security is disabled when the port has not yet been attached to the VM, then all vm traffic is dropped.
 862                 # As a workaround we wait until the VM is active and then disable the port-security
 863                 if net.get("port_security") == False:
 864                     no_secured_ports.append(new_port["port"]["id"])
 865
 866             if metadata_vpci:
 867                 metadata = {"pci_assignement": json.dumps(metadata_vpci)}
 868                 if len(metadata["pci_assignement"]) >255:
 869                     #limit the metadata size
 870                     #metadata["pci_assignement"] = metadata["pci_assignement"][0:255]
 871                     self.logger.warn("Metadata deleted since it exceeds the expected length (255) ")
 872                     metadata = {}
 873
 874             self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s' metadata %s",
 875                               name, image_id, flavor_id, str(net_list_vim), description, str(metadata))
 876
 877             security_groups   = self.config.get('security_groups')
 878             if type(security_groups) is str:
 879                 security_groups = ( security_groups, )
 880             #cloud config
 881             userdata=None
 882             config_drive = None
 883             if isinstance(cloud_config, dict):
 884                 if cloud_config.get("user-data"):
 885                     userdata=cloud_config["user-data"]
 886                 if cloud_config.get("boot-data-drive") != None:
 887                     config_drive = cloud_config["boot-data-drive"]
 888                 if cloud_config.get("config-files") or cloud_config.get("users") or cloud_config.get("key-pairs"):
 889                     if userdata:
 890                         raise vimconn.vimconnConflictException("Cloud-config cannot contain both 'userdata' and 'config-files'/'users'/'key-pairs'")
 891                     userdata_dict={}
 892                     #default user
 893                     if cloud_config.get("key-pairs"):
 894                         userdata_dict["ssh-authorized-keys"] = cloud_config["key-pairs"]
 895                         userdata_dict["users"] = [{"default": None, "ssh-authorized-keys": cloud_config["key-pairs"] }]
 896                     if cloud_config.get("users"):
 897                         if "users" not in userdata_dict:
 898                             userdata_dict["users"] = [ "default" ]
 899                         for user in cloud_config["users"]:
 900                             user_info = {
 901                                 "name" : user["name"],
 902                                 "sudo": "ALL = (ALL)NOPASSWD:ALL"
 903                             }
 904                             if "user-info" in user:
 905                                 user_info["gecos"] = user["user-info"]
 906                             if user.get("key-pairs"):
 907                                 user_info["ssh-authorized-keys"] = user["key-pairs"]
 908                             userdata_dict["users"].append(user_info)
 909
 910                     if cloud_config.get("config-files"):
 911                         userdata_dict["write_files"] = []
 912                         for file in cloud_config["config-files"]:
 913                             file_info = {
 914                                 "path" : file["dest"],
 915                                 "content": file["content"]
 916                             }
 917                             if file.get("encoding"):
 918                                 file_info["encoding"] = file["encoding"]
 919                             if file.get("permissions"):
 920                                 file_info["permissions"] = file["permissions"]
 921                             if file.get("owner"):
 922                                 file_info["owner"] = file["owner"]
 923                             userdata_dict["write_files"].append(file_info)
 924                     userdata = "#cloud-config\n"
 925                     userdata += yaml.safe_dump(userdata_dict, indent=4, default_flow_style=False)
 926                 self.logger.debug("userdata: %s", userdata)
 927             elif isinstance(cloud_config, str):
 928                 userdata = cloud_config
 929
 930             #Create additional volumes in case these are present in disk_list
 931             base_disk_index = ord('b')
 932             if disk_list != None:
 933                 block_device_mapping = {}
 934                 for disk in disk_list:
 935                     if 'image_id' in disk:
 936                         volume = self.cinder.volumes.create(size = disk['size'],name = name + '_vd' +
 937                                     chr(base_disk_index), imageRef = disk['image_id'])
 938                     else:
 939                         volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
 940                                     chr(base_disk_index))
 941                     block_device_mapping['_vd' +  chr(base_disk_index)] = volume.id
 942                     base_disk_index += 1
 943
 944                 #wait until volumes are with status available
 945                 keep_waiting = True
 946                 elapsed_time = 0
 947                 while keep_waiting and elapsed_time < volume_timeout:
 948                     keep_waiting = False
 949                     for volume_id in block_device_mapping.itervalues():
 950                         if self.cinder.volumes.get(volume_id).status != 'available':
 951                             keep_waiting = True
 952                     if keep_waiting:
 953                         time.sleep(1)
 954                         elapsed_time += 1
 955
 956                 #if we exceeded the timeout rollback
 957                 if elapsed_time >= volume_timeout:
 958                     #delete the volumes we just created
 959                     for volume_id in block_device_mapping.itervalues():
 960                         self.cinder.volumes.delete(volume_id)
 961
 962                     #delete ports we just created
 963                     for net_item  in net_list_vim:
 964                         if 'port-id' in net_item:
 965                             self.neutron.delete_port(net_item['port-id'])
 966
 967                     raise vimconn.vimconnException('Timeout creating volumes for instance ' + name,
 968                                                    http_code=vimconn.HTTP_Request_Timeout)
 969             # get availability Zone
 970             vm_av_zone = self._get_vm_availavility_zone(availavility_zone_index, nfv_availability_zones)
 971
 972             self.logger.debug("nova.servers.create({}, {}, {}, nics={}, meta={}, security_groups={}, "
 973                               "availability_zone={}, key_name={}, userdata={}, config_drive={}, "
 974                               "block_device_mapping={})".format(name, image_id, flavor_id, net_list_vim, metadata,
 975                                                                 security_groups, vm_av_zone, self.config.get('keypair'),
 976                               userdata, config_drive, block_device_mapping))
 977             server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim, meta=metadata,
 978                                               security_groups=security_groups,
 979                                               availability_zone=vm_av_zone,
 980                                               key_name=self.config.get('keypair'),
 981                                               userdata=userdata,
 982                                               config_drive=config_drive,
 983                                               block_device_mapping=block_device_mapping
 984                                               )  # , description=description)
 985
 986             # Previously mentioned workaround to wait until the VM is active and then disable the port-security
 987             if no_secured_ports:
 988                 self.__wait_for_vm(server.id, 'ACTIVE')
 989
 990             for port_id in no_secured_ports:
 991                 try:
 992                     self.neutron.update_port(port_id, {"port": {"port_security_enabled": False, "security_groups": None} })
 993
 994                 except Exception as e:
 995                     self.logger.error("It was not possible to disable port security for port {}".format(port_id))
 996                     self.delete_vminstance(server.id)
 997                     raise
 998
 999             #print "DONE :-)", server
1000             pool_id = None
1001             floating_ips = self.neutron.list_floatingips().get("floatingips", ())
1002
1003             if external_network:
1004                 self.__wait_for_vm(server.id, 'ACTIVE')
1005
1006             for floating_network in external_network:
1007                 try:
1008                     assigned = False
1009                     while(assigned == False):
1010                         if floating_ips:
1011                             ip = floating_ips.pop(0)
1012                             if not ip.get("port_id", False) and ip.get('tenant_id') == server.tenant_id:
1013                                 free_floating_ip = ip.get("floating_ip_address")
1014                                 try:
1015                                     fix_ip = floating_network.get('ip')
1016                                     server.add_floating_ip(free_floating_ip, fix_ip)
1017                                     assigned = True
1018                                 except Exception as e:
1019                                     raise vimconn.vimconnException(type(e).__name__ + ": Cannot create floating_ip "+  str(e), http_code=vimconn.HTTP_Conflict)
1020                         else:
1021                             #Find the external network
1022                             external_nets = list()
1023                             for net in self.neutron.list_networks()['networks']:
1024                                 if net['router:external']:
1025                                         external_nets.append(net)
1026
1027                             if len(external_nets) == 0:
1028                                 raise vimconn.vimconnException("Cannot create floating_ip automatically since no external "
1029                                                                "network is present",
1030                                                                 http_code=vimconn.HTTP_Conflict)
1031                             if len(external_nets) > 1:
1032                                 raise vimconn.vimconnException("Cannot create floating_ip automatically since multiple "
1033                                                                "external networks are present",
1034                                                                http_code=vimconn.HTTP_Conflict)
1035
1036                             pool_id = external_nets[0].get('id')
1037                             param = {'floatingip': {'floating_network_id': pool_id, 'tenant_id': server.tenant_id}}
1038                             try:
1039                                 #self.logger.debug("Creating floating IP")
1040                                 new_floating_ip = self.neutron.create_floatingip(param)
1041                                 free_floating_ip = new_floating_ip['floatingip']['floating_ip_address']
1042                                 fix_ip = floating_network.get('ip')
1043                                 server.add_floating_ip(free_floating_ip, fix_ip)
1044                                 assigned=True
1045                             except Exception as e:
1046                                 raise vimconn.vimconnException(type(e).__name__ + ": Cannot assign floating_ip "+  str(e), http_code=vimconn.HTTP_Conflict)
1047                 except Exception as e:
1048                     if not floating_network['exit_on_floating_ip_error']:
1049                         self.logger.warn("Cannot create floating_ip. %s", str(e))
1050                         continue
1051                     raise
1052
1053             return server.id
1054 #        except nvExceptions.NotFound as e:
1055 #            error_value=-vimconn.HTTP_Not_Found
1056 #            error_text= "vm instance %s not found" % vm_id
1057 #        except TypeError as e:
1058 #            raise vimconn.vimconnException(type(e).__name__ + ": "+  str(e), http_code=vimconn.HTTP_Bad_Request)
1059
1060         except Exception as e:
1061             # delete the volumes we just created
1062             if block_device_mapping:
1063                 for volume_id in block_device_mapping.itervalues():
1064                     self.cinder.volumes.delete(volume_id)
1065
1066             # Delete the VM
1067             if server != None:
1068                 self.delete_vminstance(server.id)
1069             else:
1070                 # delete ports we just created
1071                 for net_item in net_list_vim:
1072                     if 'port-id' in net_item:
1073                         self.neutron.delete_port(net_item['port-id'])
1074
1075             self._format_exception(e)
1076
1077     def get_vminstance(self,vm_id):
1078         '''Returns the VM instance information from VIM'''
1079         #self.logger.debug("Getting VM from VIM")
1080         try:
1081             self._reload_connection()
1082             server = self.nova.servers.find(id=vm_id)
1083             #TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
1084             return server.to_dict()
1085         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1086             self._format_exception(e)
1087
1088     def get_vminstance_console(self,vm_id, console_type="vnc"):
1089         '''
1090         Get a console for the virtual machine
1091         Params:
1092             vm_id: uuid of the VM
1093             console_type, can be:
1094                 "novnc" (by default), "xvpvnc" for VNC types,
1095                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
1096         Returns dict with the console parameters:
1097                 protocol: ssh, ftp, http, https, ...
1098                 server:   usually ip address
1099                 port:     the http, ssh, ... port
1100                 suffix:   extra text, e.g. the http path and query string
1101         '''
1102         self.logger.debug("Getting VM CONSOLE from VIM")
1103         try:
1104             self._reload_connection()
1105             server = self.nova.servers.find(id=vm_id)
1106             if console_type == None or console_type == "novnc":
1107                 console_dict = server.get_vnc_console("novnc")
1108             elif console_type == "xvpvnc":
1109                 console_dict = server.get_vnc_console(console_type)
1110             elif console_type == "rdp-html5":
1111                 console_dict = server.get_rdp_console(console_type)
1112             elif console_type == "spice-html5":
1113                 console_dict = server.get_spice_console(console_type)
1114             else:
1115                 raise vimconn.vimconnException("console type '{}' not allowed".format(console_type), http_code=vimconn.HTTP_Bad_Request)
1116
1117             console_dict1 = console_dict.get("console")
1118             if console_dict1:
1119                 console_url = console_dict1.get("url")
1120                 if console_url:
1121                     #parse console_url
1122                     protocol_index = console_url.find("//")
1123                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1124                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1125                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1126                         return -vimconn.HTTP_Internal_Server_Error, "Unexpected response from VIM"
1127                     console_dict={"protocol": console_url[0:protocol_index],
1128                                   "server":   console_url[protocol_index+2:port_index],
1129                                   "port":     console_url[port_index:suffix_index],
1130                                   "suffix":   console_url[suffix_index+1:]
1131                                   }
1132                     protocol_index += 2
1133                     return console_dict
1134             raise vimconn.vimconnUnexpectedResponse("Unexpected response from VIM")
1135
1136         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.BadRequest, ConnectionError) as e:
1137             self._format_exception(e)
1138
1139     def delete_vminstance(self, vm_id):
1140         '''Removes a VM instance from VIM. Returns the old identifier
1141         '''
1142         #print "osconnector: Getting VM from VIM"
1143         try:
1144             self._reload_connection()
1145             #delete VM ports attached to this networks before the virtual machine
1146             ports = self.neutron.list_ports(device_id=vm_id)
1147             for p in ports['ports']:
1148                 try:
1149                     self.neutron.delete_port(p["id"])
1150                 except Exception as e:
1151                     self.logger.error("Error deleting port: " + type(e).__name__ + ": "+  str(e))
1152
1153             #commented because detaching the volumes makes the servers.delete not work properly ?!?
1154             #dettach volumes attached
1155             server = self.nova.servers.get(vm_id)
1156             volumes_attached_dict = server._info['os-extended-volumes:volumes_attached']
1157             #for volume in volumes_attached_dict:
1158             #    self.cinder.volumes.detach(volume['id'])
1159
1160             self.nova.servers.delete(vm_id)
1161
1162             #delete volumes.
1163             #Although having detached them should have them  in active status
1164             #we ensure in this loop
1165             keep_waiting = True
1166             elapsed_time = 0
1167             while keep_waiting and elapsed_time < volume_timeout:
1168                 keep_waiting = False
1169                 for volume in volumes_attached_dict:
1170                     if self.cinder.volumes.get(volume['id']).status != 'available':
1171                         keep_waiting = True
1172                     else:
1173                         self.cinder.volumes.delete(volume['id'])
1174                 if keep_waiting:
1175                     time.sleep(1)
1176                     elapsed_time += 1
1177
1178             return vm_id
1179         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError) as e:
1180             self._format_exception(e)
1181         #TODO insert exception vimconn.HTTP_Unauthorized
1182         #if reaching here is because an exception
1183
1184     def refresh_vms_status(self, vm_list):
1185         '''Get the status of the virtual machines and their interfaces/ports
1186            Params: the list of VM identifiers
1187            Returns a dictionary with:
1188                 vm_id:          #VIM id of this Virtual Machine
1189                     status:     #Mandatory. Text with one of:
1190                                 #  DELETED (not found at vim)
1191                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1192                                 #  OTHER (Vim reported other status not understood)
1193                                 #  ERROR (VIM indicates an ERROR status)
1194                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1195                                 #  CREATING (on building process), ERROR
1196                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
1197                                 #
1198                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
1199                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
1200                     interfaces:
1201                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
1202                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
1203                         vim_net_id:       #network id where this interface is connected
1204                         vim_interface_id: #interface/port VIM id
1205                         ip_address:       #null, or text with IPv4, IPv6 address
1206                         compute_node:     #identification of compute node where PF,VF interface is allocated
1207                         pci:              #PCI address of the NIC that hosts the PF,VF
1208                         vlan:             #physical VLAN used for VF
1209         '''
1210         vm_dict={}
1211         self.logger.debug("refresh_vms status: Getting tenant VM instance information from VIM")
1212         for vm_id in vm_list:
1213             vm={}
1214             try:
1215                 vm_vim = self.get_vminstance(vm_id)
1216                 if vm_vim['status'] in vmStatus2manoFormat:
1217                     vm['status']    =  vmStatus2manoFormat[ vm_vim['status'] ]
1218                 else:
1219                     vm['status']    = "OTHER"
1220                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
1221                 try:
1222                     vm['vim_info']  = yaml.safe_dump(vm_vim, default_flow_style=True, width=256)
1223                 except yaml.representer.RepresenterError:
1224                     vm['vim_info'] = str(vm_vim)
1225                 vm["interfaces"] = []
1226                 if vm_vim.get('fault'):
1227                     vm['error_msg'] = str(vm_vim['fault'])
1228                 #get interfaces
1229                 try:
1230                     self._reload_connection()
1231                     port_dict=self.neutron.list_ports(device_id=vm_id)
1232                     for port in port_dict["ports"]:
1233                         interface={}
1234                         try:
1235                             interface['vim_info'] = yaml.safe_dump(port, default_flow_style=True, width=256)
1236                         except yaml.representer.RepresenterError:
1237                             interface['vim_info'] = str(port)
1238                         interface["mac_address"] = port.get("mac_address")
1239                         interface["vim_net_id"] = port["network_id"]
1240                         interface["vim_interface_id"] = port["id"]
1241                         # check if OS-EXT-SRV-ATTR:host is there,
1242                         # in case of non-admin credentials, it will be missing
1243                         if vm_vim.get('OS-EXT-SRV-ATTR:host'):
1244                             interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host']
1245                         interface["pci"] = None
1246
1247                         # check if binding:profile is there,
1248                         # in case of non-admin credentials, it will be missing
1249                         if port.get('binding:profile'):
1250                             if port['binding:profile'].get('pci_slot'):
1251                                 # TODO: At the moment sr-iov pci addresses are converted to PF pci addresses by setting the slot to 0x00
1252                                 # TODO: This is just a workaround valid for niantinc. Find a better way to do so
1253                                 #   CHANGE DDDD:BB:SS.F to DDDD:BB:00.(F%2)   assuming there are 2 ports per nic
1254                                 pci = port['binding:profile']['pci_slot']
1255                                 # interface["pci"] = pci[:-4] + "00." + str(int(pci[-1]) % 2)
1256                                 interface["pci"] = pci
1257                         interface["vlan"] = None
1258                         #if network is of type vlan and port is of type direct (sr-iov) then set vlan id
1259                         network = self.neutron.show_network(port["network_id"])
1260                         if network['network'].get('provider:network_type') == 'vlan' and \
1261                             port.get("binding:vnic_type") == "direct":
1262                             interface["vlan"] = network['network'].get('provider:segmentation_id')
1263                         ips=[]
1264                         #look for floating ip address
1265                         floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
1266                         if floating_ip_dict.get("floatingips"):
1267                             ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address") )
1268
1269                         for subnet in port["fixed_ips"]:
1270                             ips.append(subnet["ip_address"])
1271                         interface["ip_address"] = ";".join(ips)
1272                         vm["interfaces"].append(interface)
1273                 except Exception as e:
1274                     self.logger.error("Error getting vm interface information " + type(e).__name__ + ": "+  str(e))
1275             except vimconn.vimconnNotFoundException as e:
1276                 self.logger.error("Exception getting vm status: %s", str(e))
1277                 vm['status'] = "DELETED"
1278                 vm['error_msg'] = str(e)
1279             except vimconn.vimconnException as e:
1280                 self.logger.error("Exception getting vm status: %s", str(e))
1281                 vm['status'] = "VIM_ERROR"
1282                 vm['error_msg'] = str(e)
1283             vm_dict[vm_id] = vm
1284         return vm_dict
1285
1286     def action_vminstance(self, vm_id, action_dict):
1287         '''Send and action over a VM instance from VIM
1288         Returns the vm_id if the action was successfully sent to the VIM'''
1289         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
1290         try:
1291             self._reload_connection()
1292             server = self.nova.servers.find(id=vm_id)
1293             if "start" in action_dict:
1294                 if action_dict["start"]=="rebuild":
1295                     server.rebuild()
1296                 else:
1297                     if server.status=="PAUSED":
1298                         server.unpause()
1299                     elif server.status=="SUSPENDED":
1300                         server.resume()
1301                     elif server.status=="SHUTOFF":
1302                         server.start()
1303             elif "pause" in action_dict:
1304                 server.pause()
1305             elif "resume" in action_dict:
1306                 server.resume()
1307             elif "shutoff" in action_dict or "shutdown" in action_dict:
1308                 server.stop()
1309             elif "forceOff" in action_dict:
1310                 server.stop() #TODO
1311             elif "terminate" in action_dict:
1312                 server.delete()
1313             elif "createImage" in action_dict:
1314                 server.create_image()
1315                 #"path":path_schema,
1316                 #"description":description_schema,
1317                 #"name":name_schema,
1318                 #"metadata":metadata_schema,
1319                 #"imageRef": id_schema,
1320                 #"disk": {"oneOf":[{"type": "null"}, {"type":"string"}] },
1321             elif "rebuild" in action_dict:
1322                 server.rebuild(server.image['id'])
1323             elif "reboot" in action_dict:
1324                 server.reboot() #reboot_type='SOFT'
1325             elif "console" in action_dict:
1326                 console_type = action_dict["console"]
1327                 if console_type == None or console_type == "novnc":
1328                     console_dict = server.get_vnc_console("novnc")
1329                 elif console_type == "xvpvnc":
1330                     console_dict = server.get_vnc_console(console_type)
1331                 elif console_type == "rdp-html5":
1332                     console_dict = server.get_rdp_console(console_type)
1333                 elif console_type == "spice-html5":
1334                     console_dict = server.get_spice_console(console_type)
1335                 else:
1336                     raise vimconn.vimconnException("console type '{}' not allowed".format(console_type),
1337                                                    http_code=vimconn.HTTP_Bad_Request)
1338                 try:
1339                     console_url = console_dict["console"]["url"]
1340                     #parse console_url
1341                     protocol_index = console_url.find("//")
1342                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1343                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1344                     if protocol_index < 0 or port_index<0 or suffix_index<0:
1345                         raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1346                     console_dict2={"protocol": console_url[0:protocol_index],
1347                                   "server":   console_url[protocol_index+2 : port_index],
1348                                   "port":     int(console_url[port_index+1 : suffix_index]),
1349                                   "suffix":   console_url[suffix_index+1:]
1350                                   }
1351                     return console_dict2
1352                 except Exception as e:
1353                     raise vimconn.vimconnException("Unexpected response from VIM " + str(console_dict))
1354
1355             return vm_id
1356         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound, ConnectionError) as e:
1357             self._format_exception(e)
1358         #TODO insert exception vimconn.HTTP_Unauthorized
1359
1360 #NOT USED FUNCTIONS
1361
1362     def new_external_port(self, port_data):
1363         #TODO openstack if needed
1364         '''Adds a external port to VIM'''
1365         '''Returns the port identifier'''
1366         return -vimconn.HTTP_Internal_Server_Error, "osconnector.new_external_port() not implemented"
1367
1368     def connect_port_network(self, port_id, network_id, admin=False):
1369         #TODO openstack if needed
1370         '''Connects a external port to a network'''
1371         '''Returns status code of the VIM response'''
1372         return -vimconn.HTTP_Internal_Server_Error, "osconnector.connect_port_network() not implemented"
1373
1374     def new_user(self, user_name, user_passwd, tenant_id=None):
1375         '''Adds a new user to openstack VIM'''
1376         '''Returns the user identifier'''
1377         self.logger.debug("osconnector: Adding a new user to VIM")
1378         try:
1379             self._reload_connection()
1380             user=self.keystone.users.create(user_name, user_passwd, tenant_id=tenant_id)
1381             #self.keystone.tenants.add_user(self.k_creds["username"], #role)
1382             return user.id
1383         except ksExceptions.ConnectionError as e:
1384             error_value=-vimconn.HTTP_Bad_Request
1385             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1386         except ksExceptions.ClientException as e: #TODO remove
1387             error_value=-vimconn.HTTP_Bad_Request
1388             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1389         #TODO insert exception vimconn.HTTP_Unauthorized
1390         #if reaching here is because an exception
1391         if self.debug:
1392             self.logger.debug("new_user " + error_text)
1393         return error_value, error_text
1394
1395     def delete_user(self, user_id):
1396         '''Delete a user from openstack VIM'''
1397         '''Returns the user identifier'''
1398         if self.debug:
1399             print "osconnector: Deleting  a  user from VIM"
1400         try:
1401             self._reload_connection()
1402             self.keystone.users.delete(user_id)
1403             return 1, user_id
1404         except ksExceptions.ConnectionError as e:
1405             error_value=-vimconn.HTTP_Bad_Request
1406             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1407         except ksExceptions.NotFound as e:
1408             error_value=-vimconn.HTTP_Not_Found
1409             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1410         except ksExceptions.ClientException as e: #TODO remove
1411             error_value=-vimconn.HTTP_Bad_Request
1412             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1413         #TODO insert exception vimconn.HTTP_Unauthorized
1414         #if reaching here is because an exception
1415         if self.debug:
1416             print "delete_tenant " + error_text
1417         return error_value, error_text
1418
1419     def get_hosts_info(self):
1420         '''Get the information of deployed hosts
1421         Returns the hosts content'''
1422         if self.debug:
1423             print "osconnector: Getting Host info from VIM"
1424         try:
1425             h_list=[]
1426             self._reload_connection()
1427             hypervisors = self.nova.hypervisors.list()
1428             for hype in hypervisors:
1429                 h_list.append( hype.to_dict() )
1430             return 1, {"hosts":h_list}
1431         except nvExceptions.NotFound as e:
1432             error_value=-vimconn.HTTP_Not_Found
1433             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1434         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1435             error_value=-vimconn.HTTP_Bad_Request
1436             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1437         #TODO insert exception vimconn.HTTP_Unauthorized
1438         #if reaching here is because an exception
1439         if self.debug:
1440             print "get_hosts_info " + error_text
1441         return error_value, error_text
1442
1443     def get_hosts(self, vim_tenant):
1444         '''Get the hosts and deployed instances
1445         Returns the hosts content'''
1446         r, hype_dict = self.get_hosts_info()
1447         if r<0:
1448             return r, hype_dict
1449         hypervisors = hype_dict["hosts"]
1450         try:
1451             servers = self.nova.servers.list()
1452             for hype in hypervisors:
1453                 for server in servers:
1454                     if server.to_dict()['OS-EXT-SRV-ATTR:hypervisor_hostname']==hype['hypervisor_hostname']:
1455                         if 'vm' in hype:
1456                             hype['vm'].append(server.id)
1457                         else:
1458                             hype['vm'] = [server.id]
1459             return 1, hype_dict
1460         except nvExceptions.NotFound as e:
1461             error_value=-vimconn.HTTP_Not_Found
1462             error_text= (str(e) if len(e.args)==0 else str(e.args[0]))
1463         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1464             error_value=-vimconn.HTTP_Bad_Request
1465             error_text= type(e).__name__ + ": "+  (str(e) if len(e.args)==0 else str(e.args[0]))
1466         #TODO insert exception vimconn.HTTP_Unauthorized
1467         #if reaching here is because an exception
1468         if self.debug:
1469             print "get_hosts " + error_text
1470         return error_value, error_text
1471
1472